diff --git a/.gitignore b/.gitignore index 8a7309999b8c..5eddd0d76f29 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,5 @@ build/ .libs/ m4/ *.nc - - +*.log +*.gz diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..24fd6e268e1f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,58 @@ +language: c +dist: trusty +sudo: false + +branches: + only: + - master + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - pkg-config netcdf-bin libnetcdf-dev openmpi-bin libopenmpi-dev gfortran doxygen graphviz + +before_install: + - test -n $CC && unset CC + - test -n $FC && unset FC + - test -n $CPPFLAGS && unset CPPFLAGS + - test -n FCFLAGS && unset FCFLAGS + +before_script: + - export CC=mpicc + - export FC=mpif90 + - export CPPFLAGS='-I/usr/include' + - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz + - tar -xzvf pnetcdf-1.11.0.tar.gz + - ls -l + - pushd pnetcdf-1.11.0 + - ./configure --prefix=/usr --enable-shared + - make + - sudo make install + - popd +env: + global: + - CC=mpicc + - FC=mpif90 + - CPPFLAGS='-I/usr/include' + - CFLAGS='-std=c99' + - LDFLAGS='-L/usr/lib' + +script: + - ls -l /usr/include + - autoreconf -i + - export CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' + - export FFLAGS='-fsanitize=address -fno-omit-frame-pointer' + - export FCFLAGS='-fsanitize=address -fno-omit-frame-pointer' + - export DISTCHECK_CONFIGURE_FLAGS='--enable-fortran' + - ./configure --enable-fortran --enable-developer-docs + - make + - make -j distcheck + - rm -rf build + - mkdir build + - cd build + - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. + - make VERBOSE=1 + - make tests VERBOSE=1 + - make test VERBOSE=1 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index c50087ad740a..846218ffa9a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,8 @@ project (PIO C Fortran) # The project version number. set(VERSION_MAJOR 2 CACHE STRING "Project major version number.") -set(VERSION_MINOR 3 CACHE STRING "Project minor version number.") -set(VERSION_PATCH 0 CACHE STRING "Project patch version number.") +set(VERSION_MINOR 4 CACHE STRING "Project minor version number.") +set(VERSION_PATCH 4 CACHE STRING "Project patch version number.") mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) # The size of the data buffer for write/read_darray(). @@ -26,9 +26,17 @@ option (PIO_INTERNAL_DOC "Enable PIO developer documentation" OFF) option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) -option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" OFF) +option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" ON) +option (PIO_USE_PNETCDF_VARD "Use pnetcdf put_vard " OFF) option (WITH_PNETCDF "Require the use of PnetCDF" ON) +# Set a variable that appears in the config.h.in file. +if(PIO_USE_PNETCDF_VARD) + set(USE_VARD 1) +else() + set(USE_VARD 0) +endif() + # Set a variable that appears in the config.h.in file. if(PIO_USE_MALLOC) set(USE_MALLOC 1) diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 000000000000..8652d01dda1b --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,16 @@ +/****************************************************************************** + * + * + * + * Copyright (C) 2009-2019 + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ \ No newline at end of file diff --git a/Makefile.am b/Makefile.am index 990da3e474e2..b114d46e0415 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,16 @@ +# This is part of PIO. It creates the main Makefile. -SUBDIRS = src tests -#Recommended by libtoolize +# Ed Hartnett + +# Look in the m4 directory for autotools stuff. ACLOCAL_AMFLAGS= -I m4 + +# Does the user want to build fortran? +if BUILD_DOCS +DOC = doc +endif + +SUBDIRS = src tests examples ${DOC} scripts + +EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT + diff --git a/README.md b/README.md index e03c2f9bd959..6e8b263e018f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,14 @@ A high-level Parallel I/O Library for structured grid applications ## Website -For complete documentation, see our website at [http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). +For complete documentation, see our website at +[http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). + +## Mailing List + +The (low-traffic) PIO mailing list is at +https://groups.google.com/forum/#!forum/parallelio, send email to the +list at parallelio@googlegroups.com. ## Nightly Tests @@ -13,128 +20,52 @@ cdash site at [http://my.cdash.org/index.php?project=PIO](http://my.cdash.org/in ## Dependencies -PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.0+) for I/O. +PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) +for I/O. NetCDF may be built with or without netCDF-4 features. NetCDF +is required for PIO, PnetCDF is optional. + Ideally, the NetCDF version should be built with MPI, which requires that it be linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be built with DAP support, which introduces a dependency on CURL. Additionally, HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP. -## Configuring with CMake +## Building PIO -To configure the build, PIO requires CMake version 2.8.12+. The typical -configuration with CMake can be done as follows: +To build PIO, unpack the distribution tarball and do: ``` -CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source +CC=mpicc FC=mpif90 ./configure --enable-fortran && make check install ``` -where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers -for your system. - -The `OPTIONS` section typically should consist of pointers to the install -locations for various dependencies, assuming these dependencies are not -located in *canonical* search locations. - -For each dependency `XXX`, one can specify the location of its -installation path with the CMake variable `XXX_PATH`. If the `C` and -`Fortran` libraries for the dependency are installed in different locations -(such as can be done with NetCDF), then you can specify individually -`XXX_C_PATH` and `XXX_Fortran_PATH`. Hence, you can specify the locations -of both NetCDF-C and NetCDF-Fortran, as well as PnetCDF, with the following -CMake configuration line: - +For a full description of the available options and flags, try: ``` -CC=mpicc FC=mpif90 cmake -DNetCDF_C_PATH=/path/to/netcdf-c \ - -DNetCDF_Fortran_PATH=/path/to/netcdf-fortran \ - -DPnetCDF_PATH=/path/to/pnetcdf \ - /path/to/pio/source +./configure --help ``` -This works for the dependencies: `NetCDF`, `PnetCDF`, `HDF5`, `LIBZ`, `SZIP`. - -### Additional CMake Options - -Additional configuration options can be specified on the command line. - -The `PIO_ENABLE_TIMING` option can be set to `ON` or `OFF` to enable or -disable the use of GPTL timing in the PIO libraries. This feature requires -the GPTL C library for the PIO `C` library and the GPTL Fortran library with -the `perf_mod.mod` and `perf_utils.mod` interface modules. If these GPTL -libraries are already installed on the system, the user can point PIO to the -location of these libraries with the `GPTL_PATH` variable (or, individually, -`GPTL_C_PATH` and `GPTL_Fortran_Perf_PATH` variables). However, if these -GPTL libraries are not installed on the system, and GPTL cannot be found, -then PIO will build its own internal version of GPTL. - -If PnetCDF is not installed on the system, the user can disable its use by -setting `-DWITH_PNETCDF=OFF`. This will disable the search for PnetCDF on the -system and disable the use of PnetCDF from within PIO. - -If the user wishes to disable the PIO tests, then the user can set the -variable `-DPIO_ENABLE_TESTS=OFF`. This will entirely disable the CTest -testing suite, as well as remove all of the test build targets. - -If you wish to install PIO in a safe location for use later with other -software, you may set the `CMAKE_INSTALL_PREFIX` variable to point to the -desired install location. - -## Building - -Once you have successfully configured PIO with CMake in a build directory. -From within the build directory, build PIO with: - -``` -make -``` - -This will build the `pioc` and `piof` libraries. - -## Testing - -If you desire to do testing, and `PIO_ENABLE_TESTS=ON` (which is the default -setting), you may build the test executables with: - -``` -make tests -``` - -Once the tests have been built, you may run tests with: - -``` -ctest -``` - -If you have not run `make tests` before you run `ctest`, then you will see -all of the tests fail. - -Alternatively, you may build the test executables and then run tests -immediately with: +Note that environment variables CC and FC may need to be set to the +MPI versions of the C and Fortran compiler. Also CPPFLAGS and LDFLAGS +may need to be set to indicate the locations of one or more of the +dependent libraries. (If using MPI compilers, the entire set of +dependent libraries should be built with the same compilers.) For +example: ``` +export CC=mpicc +export FC=mpifort +export CPPFLAGS='-I/usr/local/netcdf-fortran-4.4.5_c_4.6.3_mpich-3.2/include -I/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/include -I/usr/local/pnetcdf-1.11.0_shared/include' +export LDFLAGS='-L/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/lib -L/usr/local/pnetcdf-1.11.0_shared/lib' +./configure --prefix=/usr/local/pio-2.4.2 --enable-fortran make check +make install ``` -(similar to the typical `make check` Autotools target). +## Building with CMake -**NOTE:** It is important to note that these tests are designed to run in parallel. -If you are on one of the supported supercomputing platforms (i.e., NERSC, NWSC, ALCF, -etc.), then the `ctest` command will assume that the tests will be run in an appropriately -configured and scheduled parallel job. This can be done by requesting an interactive -session from the login nodes and then running `ctest` from within the interactive -terminal. Alternatively, this can be done by running the `ctest` command from a -job submission script. It is important to understand, however, that `ctest` itself -will preface all of the test executable commands with the appropriate `mpirun`/`mpiexec`/`runjob`/etc. -Hence, you should not further preface the `ctest` command with these MPI launchers. - -## Installing - -Once you have built the PIO libraries, you may install them in the location -specified by the `CMAKE_INSTALL_PREFIX`. To do this, simply type: +The typical configuration with CMake can be done as follows: ``` -make install +CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source ``` -If the internal GPTL libraries were built (because GPTL could not be found -and the `PIO_ENABLE_TIMING` variable is set to `ON`), then these libraries -will be installed with PIO. +Full instructions for the cmake build can be found in the installation +documentation. diff --git a/cmake/LibFind.cmake b/cmake/LibFind.cmake index 7da13e325965..61cd93aa37d4 100644 --- a/cmake/LibFind.cmake +++ b/cmake/LibFind.cmake @@ -224,11 +224,15 @@ function (find_package_component PKG) if (${PKG}_PATHS) list (APPEND SEARCH_DIRS ${${PKG}_PATHS}) endif () - - # Start the search for the include file and library file - set (${PKGCOMP}_PREFIX ${PKGCOMP}_PREFIX-NOTFOUND) - set (${PKGCOMP}_INCLUDE_DIR ${PKGCOMP}_INCLUDE_DIR-NOTFOUND) - set (${PKGCOMP}_LIBRARY ${PKGCOMP}_LIBRARY-NOTFOUND) + + # Start the search for the include file and library file. Only overload + # if the variable is not defined. + foreach (suffix PREFIX LIBRARY INCLUDE_DIR) + if (NOT DEFINED ${PKGCOMP}_${suffix}) + set (${PKGCOMP}_${suffix} ${PKGCOMP}_${suffix}-NOTFOUND) + endif () + endforeach () + foreach (dir IN LISTS SEARCH_DIRS) # Search for include file names in current dirrectory diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake index 2dd9a7d27a8d..f1116724831f 100644 --- a/cmake/LibMPI.cmake +++ b/cmake/LibMPI.cmake @@ -16,7 +16,7 @@ endif () # #============================================================================== -# - Get the machine platform-specific +# - Get the machine platform-specific # # Syntax: platform_name (RETURN_VARIABLE) # @@ -25,30 +25,25 @@ function (platform_name RETURN_VARIABLE) # Determine platform name from site name... site_name (SITENAME) - # UCAR/NWSC Machines - if (SITENAME MATCHES "^yslogin" OR - SITENAME MATCHES "^geyser" OR - SITENAME MATCHES "^caldera") - - set (${RETURN_VARIABLE} "nwsc" PARENT_SCOPE) - # New NWSC SGI machine - elseif (SITENAME MATCHES "^laramie") - + if (SITENAME MATCHES "^laramie" OR + SITENAME MATCHES "^cheyenne" OR + SITENAME MATCHES "^chadmin") + set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE) - + # ALCF/Argonne Machines elseif (SITENAME MATCHES "^mira" OR SITENAME MATCHES "^cetus" OR SITENAME MATCHES "^vesta" OR SITENAME MATCHES "^cooley") - + set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE) - + # NERSC Machines elseif (SITENAME MATCHES "^edison" OR SITENAME MATCHES "^cori") - + set (${RETURN_VARIABLE} "nersc" PARENT_SCOPE) # NCSA Machine (Blue Waters) @@ -61,11 +56,11 @@ function (platform_name RETURN_VARIABLE) SITENAME MATCHES "^titan") set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE) - + else () set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE) - + endif () endfunction () @@ -85,35 +80,35 @@ function (add_mpi_test TESTNAME) set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE) set (multiValueArgs ARGUMENTS) cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - + # Store parsed arguments for convenience set (exec_file ${${TESTNAME}_EXECUTABLE}) set (exec_args ${${TESTNAME}_ARGUMENTS}) set (num_procs ${${TESTNAME}_NUMPROCS}) set (timeout ${${TESTNAME}_TIMEOUT}) - + # Get the platform name platform_name (PLATFORM) - + # Default ("unknown" platform) execution if (PLATFORM STREQUAL "unknown") # Run tests directly from the command line - set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} - ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} + set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} + ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${MPIEXEC_POSTFLAGS} ${exec_args}) else () - + # Run tests from the platform-specific executable - set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} + set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args}) - + endif () - + # Add the test to CTest add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD}) - + # Adjust the test timeout set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout}) diff --git a/cmake/mpiexec.nwsc b/cmake/mpiexec.nwsc deleted file mode 100755 index a6242b66712e..000000000000 --- a/cmake/mpiexec.nwsc +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Arguments: -# -# $1 - Number of MPI Tasks -# $2+ - Executable and its arguments -# - -NP=$1 -shift - -mpirun.lsf $@ -n$NP diff --git a/cmake/mpiexec.nwscla b/cmake/mpiexec.nwscla index bb7018bf92ce..9aea7be13e77 100755 --- a/cmake/mpiexec.nwscla +++ b/cmake/mpiexec.nwscla @@ -8,5 +8,4 @@ NP=$1 shift - -mpiexec_mpt -n $NP $@ +mpirun -np $NP $@ diff --git a/cmake_config.h.in b/cmake_config.h.in index ba817fe5d38f..cfd26d776010 100644 --- a/cmake_config.h.in +++ b/cmake_config.h.in @@ -1,10 +1,12 @@ -/** @file +/** @file * * This is the template for the config.h file, which is created at * build-time by cmake. */ #ifndef _PIO_CONFIG_ #define _PIO_CONFIG_ +/* Set to avoid warning in intel19 compiler wrt strnlen */ +#define _GNU_SOURCE /** The major part of the version number. */ #define PIO_VERSION_MAJOR @VERSION_MAJOR@ @@ -28,4 +30,6 @@ /* buffer size for darray data. */ #define PIO_BUFFER_SIZE @PIO_BUFFER_SIZE@ +#define USE_VARD @USE_VARD@ + #endif /* _PIO_CONFIG_ */ diff --git a/configure.ac b/configure.ac index be47a262a60b..d18bf60bb58d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,19 +1,33 @@ ## This is the autoconf file for the PIO library. ## Ed Hartnett 8/16/17 + # Initialize autoconf and automake. -AC_INIT(pio, 2.3.0) +AC_INIT(pio, 2.4.4-development) AC_CONFIG_SRCDIR(src/clib/pio_darray.c) AM_INIT_AUTOMAKE([foreign serial-tests]) +# The PIO version, again. +AC_DEFINE([PIO_VERSION_MAJOR], [2], [PIO major version]) +AC_DEFINE([PIO_VERSION_MINOR], [4], [PIO minor version]) +AC_DEFINE([PIO_VERSION_PATCH], [4], [PIO patch version]) + +# Once more for the documentation. +AC_SUBST([VERSION_MAJOR], [2]) +AC_SUBST([VERSION_MINOR], [4]) +AC_SUBST([VERSION_PATCH], [4]) + # The m4 directory holds macros for autoconf. AC_CONFIG_MACRO_DIR([m4]) +# Libtool initialisation. +LT_INIT + # Find and learn about the C compiler. AC_PROG_CC -# Libtool initialisation. -AC_PROG_LIBTOOL +# Find and learn about the Fortran compiler. +AC_PROG_FC # Always use malloc in autotools builds. AC_DEFINE([PIO_USE_MALLOC], [1], [use malloc for memory]) @@ -38,18 +52,115 @@ if test "x$enable_logging" = xyes; then AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.]) fi -# NetCDF (at least classic) is required for PIO to build. -AC_DEFINE([_NETCDF], [1], [netCDF classic library available]) +# Does the user want to use MPE library? +AC_MSG_CHECKING([whether use of MPE library is enabled]) +AC_ARG_ENABLE([mpe], + [AS_HELP_STRING([--enable-mpe], + [enable use of MPE library for timing and diagnostic info (may negatively impact performance).])]) +test "x$enable_mpe" = xyes || enable_mpe=no +AC_MSG_RESULT([$enable_mpe]) +if test "x$enable_mpe" = xyes; then + AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [-lpthread -lm]) + AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no]) + if test "x$HAVE_LIBMPE" = xno -o "x$HAVE_MPE" = xno; then + AC_MSG_ERROR([MPE not found but --enable-mpe used.]) + fi + AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.]) +fi -# Is parallel-netcdf library available? -#AC_DEFINE([_PNETCDF], [1], [parallel-netcdf library available]) +# Does the user want to enable timing? +AC_MSG_CHECKING([whether GPTL timing library is used]) +AC_ARG_ENABLE([timing], + [AS_HELP_STRING([--enable-timing], + [enable use of the GPTL timing library.])]) +test "x$enable_timing" = xyes || enable_timing=no +AC_MSG_RESULT([$enable_timing]) +if test "x$enable_timing" = xyes; then + AC_DEFINE([TIMING], 1, [If true, use GPTL timing library.]) +fi +AM_CONDITIONAL(USE_GPTL, [test "x$enable_timing" = xyes]) -# The PIO version, again. -AC_DEFINE([PIO_VERSION_MAJOR], [2], [PIO major version]) -AC_DEFINE([PIO_VERSION_MINOR], [3], [PIO minor version]) -AC_DEFINE([PIO_VERSION_PATCH], [0], [PIO patch version]) +# Does the user want to disable papi? +AC_MSG_CHECKING([whether PAPI should be enabled (if enable-timing is used)]) +AC_ARG_ENABLE([papi], [AS_HELP_STRING([--disable-papi], + [disable PAPI library use])]) +test "x$enable_papi" = xno || enable_papi=yes +AC_MSG_RESULT($enable_papi) -# ??? +# Does the user want to disable test runs? +AC_MSG_CHECKING([whether test runs should be enabled for make check]) +AC_ARG_ENABLE([test-runs], [AS_HELP_STRING([--disable-test-runs], + [disable running run_test.sh test scripts for make check. Tests will still be built.])]) +test "x$enable_test_runs" = xno || enable_test_runs=yes +AC_MSG_RESULT($enable_test_runs) +AM_CONDITIONAL(RUN_TESTS, [test "x$enable_test_runs" = xyes]) + +# Does the user want to enable Fortran library? +AC_MSG_CHECKING([whether Fortran library should be built]) +AC_ARG_ENABLE([fortran], + [AS_HELP_STRING([--enable-fortran], + [build the PIO Fortran library.])]) +test "x$enable_fortran" = xyes || enable_fortran=no +AC_MSG_RESULT([$enable_fortran]) +AM_CONDITIONAL(BUILD_FORTRAN, [test "x$enable_fortran" = xyes]) + +# Does the user want to disable pnetcdf? +AC_MSG_CHECKING([whether pnetcdf is to be used]) +AC_ARG_ENABLE([pnetcdf], + [AS_HELP_STRING([--disable-pnetcdf], + [Disable pnetcdf use.])]) +test "x$enable_pnetcdf" = xno || enable_pnetcdf=yes +AC_MSG_RESULT([$enable_pnetcdf]) +AM_CONDITIONAL(BUILD_PNETCDF, [test "x$enable_pnetcdf" = xyes]) + +# Does the user want to build documentation? +AC_MSG_CHECKING([whether documentation should be build (requires doxygen)]) +AC_ARG_ENABLE([docs], + [AS_HELP_STRING([--enable-docs], + [enable building of documentation with doxygen.])]) +test "x$enable_docs" = xyes || enable_docs=no +AC_MSG_RESULT([$enable_docs]) + +# Does the user want to developer documentation? +AC_MSG_CHECKING([whether PIO developer documentation should be build (only for PIO developers)]) +AC_ARG_ENABLE([developer-docs], + [AS_HELP_STRING([--enable-developer-docs], + [enable building of PIO developer documentation with doxygen.])]) +test "x$enable_developer_docs" = xyes || enable_developer_docs=no +AC_MSG_RESULT([$enable_developer_docs]) + +# Developer docs enables docs. +if test "x$enable_developer_docs" = xyes; then + enable_docs=yes +fi +AM_CONDITIONAL(BUILD_DOCS, [test "x$enable_docs" = xyes]) + +# Is doxygen installed? +AC_CHECK_PROGS([DOXYGEN], [doxygen]) +if test -z "$DOXYGEN" -a "x$enable_docs" = xyes; then + AC_MSG_ERROR([Doxygen not found but --enable-docs used.]) +fi + +# If building docs, process Doxyfile.in into Doxyfile. +if test "x$enable_docs" = xyes; then + AC_SUBST([CMAKE_CURRENT_SOURCE_DIR], ["."]) + AC_SUBST([CMAKE_BINARY_DIR], [".."]) + if test "x$enable_fortran" = xno; then + AC_MSG_ERROR([--enable-fortran is required for documentation builds.]) + fi + AC_SUBST([FORTRAN_SRC_FILES], ["../src/flib/piodarray.f90 ../src/flib/pio.F90 ../src/flib/pio_kinds.F90 ../src/flib/piolib_mod.f90 ../src/flib/pionfatt_mod_2.f90 ../src/flib/pio_nf.F90 ../src/flib/pionfget_mod_2.f90 ../src/flib/pionfput_mod.f90 ../src/flib/pio_support.F90 ../src/flib/pio_types.F90"]) + if test "x$enable_developer_docs" = xyes; then + AC_SUBST([C_SRC_FILES], ["../src/clib"]) + else + AC_SUBST([C_SRC_FILES], ["../src/clib/pio_nc.c ../src/clib/pio_nc4.c ../src/clib/pio_darray.c ../src/clib/pio_get_nc.c ../src/clib/pio_put_nc.c ../src/clib/pioc_support.c ../src/clib/pioc.c ../src/clib/pio_file.c ../src/clib/pio.h ../src/clib/pio_get_vard.c ../src/clib/pio_put_vard.c"]) + fi + AC_CONFIG_FILES([doc/Doxyfile]) +fi + +# NetCDF (at least classic) is required for PIO to build. +AC_DEFINE([_NETCDF], [1], [netCDF classic library available]) + +# ???? AC_DEFINE([CPRGNU], [1], [defined by CMake build]) # We must have MPI to build PIO. @@ -61,11 +172,17 @@ AC_DEFINE([INCLUDE_CMAKE_FCI], [1], [defined by CMake build]) # All builds are on LINUX. AC_DEFINE([LINUX], [1], [defined by CMake build]) +# Define to solve intel compiler warning. +AC_DEFINE([_GNU_SOURCE], [1], [solve strnlen declared implicitly warning on intel compiler]) + # Check for netCDF library. AC_CHECK_LIB([netcdf], [nc_create], [], [AC_MSG_ERROR([Can't find or link to the netcdf library.])]) # Check for pnetcdf library. AC_CHECK_LIB([pnetcdf], [ncmpi_create], [], []) +if test "x$ac_cv_lib_pnetcdf_ncmpi_create" = xno -a $enable_pnetcdf = yes; then + AC_MSG_ERROR([Pnetcdf not found. Set CPPFLAGS/LDFLAGS or use --disable-pnetcdf.]) +fi # If we have parallel-netcdf, then set these as well. if test x$ac_cv_lib_pnetcdf_ncmpi_create = xyes; then @@ -86,6 +203,7 @@ AC_MSG_RESULT([${have_netcdf_par}]) if test x$have_netcdf_par = xyes; then AC_DEFINE([_NETCDF4],[1],[Does netCDF library provide netCDF-4 with parallel access]) fi +AM_CONDITIONAL(BUILD_NETCDF4, [test "x$have_netcdf_par" = xyes]) # Not working for some reason, so I will just set it... AC_CHECK_TYPE([MPI_Offset], [], [], [#include ]) @@ -98,6 +216,45 @@ fi #AC_CHECK_SIZEOF([MPI_Offset], [], [[#include ]]) #AC_DEFINE([SIZEOF_MPI_OFFSET], [8], [netCDF classic library available]) +# If we want the timing library, we must find it. +if test "x$enable_timing" = xyes; then + AC_CHECK_HEADERS([gptl.h]) + AC_CHECK_LIB([gptl], [GPTLinitialize], [], + [AC_MSG_ERROR([Can't find or link to the GPTL library.])]) + if test "x$enable_fortran" = xyes; then + AC_LANG_PUSH([Fortran]) +# AC_CHECK_HEADERS([gptl.inc]) + AC_CHECK_LIB([gptlf], [gptlstart], [], + [AC_MSG_ERROR([Can't find or link to the GPTL Fortran library.])]) + AC_LANG_POP([Fortran]) + fi + + # Check for papi library. + AC_CHECK_LIB([papi], [PAPI_library_init]) + AC_MSG_CHECKING([whether system can support PAPI]) + have_papi=no + if test $enable_papi = yes; then + if test "x$ac_cv_lib_papi_PAPI_library_init" = xyes; then + # If we have PAPI library, check /proc/sys/kernel/perf_event_paranoid + # to see if we have permissions. + if test -f /proc/sys/kernel/perf_event_paranoid; then + if test `cat /proc/sys/kernel/perf_event_paranoid` != 1; then + AC_MSG_ERROR([PAPI library found, but /proc/sys/kernel/perf_event_paranoid != 1 + try sudo sh -c 'echo 1 >/proc/sys/kernel/perf_event_paranoid']) + fi + fi + AC_DEFINE([HAVE_PAPI], [1], [PAPI library is present and usable]) + have_papi=yes + fi + fi + AC_MSG_RESULT($have_papi) +fi +AM_CONDITIONAL([HAVE_PAPI], [test "x$have_papi" = xyes]) + +AC_CONFIG_FILES([tests/general/pio_tutil.F90:tests/general/util/pio_tutil.F90]) + +AC_CONFIG_LINKS([tests/unit/input.nl:tests/unit/input.nl]) + # Create the config.h file. AC_CONFIG_HEADERS([config.h]) @@ -105,5 +262,17 @@ AC_CONFIG_HEADERS([config.h]) AC_OUTPUT(Makefile src/Makefile src/clib/Makefile + src/flib/Makefile + src/gptl/Makefile tests/Makefile - tests/cunit/Makefile) + tests/cunit/Makefile + tests/unit/Makefile + tests/general/Makefile + tests/general/util/Makefile + tests/performance/Makefile + doc/Makefile + doc/source/Makefile + doc/images/Makefile + examples/Makefile + examples/c/Makefile + scripts/Makefile) diff --git a/ctest/CTestEnvironment-nwsc.cmake b/ctest/CTestEnvironment-nwsc.cmake deleted file mode 100644 index 4a0d6fd3acd5..000000000000 --- a/ctest/CTestEnvironment-nwsc.cmake +++ /dev/null @@ -1,18 +0,0 @@ -#============================================================================== -# -# This file sets the environment variables needed to configure and build -# on the NCAR Wyoming Supercomputing Center systems -# (yellowstone/caldera/geyser). -# -#============================================================================== - -# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already -# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. - -# Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF") - -# If MPISERIAL environment variable is set, then enable MPISERIAL -if (DEFINED ENV{MPISERIAL}) - set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON -DPIO_ENABLE_EXAMPLES=OFF ") -endif () diff --git a/ctest/CTestEnvironment-nwscla.cmake b/ctest/CTestEnvironment-nwscla.cmake index b7f1d1c9aef7..efee6bf659d9 100644 --- a/ctest/CTestEnvironment-nwscla.cmake +++ b/ctest/CTestEnvironment-nwscla.cmake @@ -10,7 +10,7 @@ # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. # Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE") +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE ") # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) diff --git a/ctest/runcdash-nwsc-gnu.sh b/ctest/runcdash-nwscla-gnu.sh similarity index 56% rename from ctest/runcdash-nwsc-gnu.sh rename to ctest/runcdash-nwscla-gnu.sh index d406138bcfdb..cdc9eb9916d5 100755 --- a/ctest/runcdash-nwsc-gnu.sh +++ b/ctest/runcdash-nwscla-gnu.sh @@ -9,16 +9,17 @@ fi module reset module unload netcdf -module swap intel gnu/6.1.0 -module load git/2.3.0 -module load cmake/3.0.2 -module load netcdf-mpi/4.4.1 -module load pnetcdf/1.7.0 +module swap intel gnu/8.1.0 +module swap mpt openmpi/3.1.0 +module load git/2.10.2 +module load cmake/3.12.1 +module load netcdf/4.6.1 +module load pnetcdf/1.10.0 export CC=mpicc export FC=mpif90 -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard export PIO_COMPILER_ID=GNU-`$CC --version | head -n 1 | tail -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then @@ -27,10 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone --branch develop https://github.com/PARALLELIO/ParallelIO src + git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout develop -git pull origin develop +git checkout master +git pull origin master ctest -S CTestScript.cmake,${model} -VV diff --git a/ctest/runcdash-nwscla-intel.sh b/ctest/runcdash-nwscla-intel.sh index d7a7f8c91204..9393086ed8d4 100755 --- a/ctest/runcdash-nwscla-intel.sh +++ b/ctest/runcdash-nwscla-intel.sh @@ -11,11 +11,11 @@ source /etc/profile.d/modules.sh module reset module unload netcdf -module swap intel intel/17.0.1 +module swap intel intel/19.0.2 +module switch mpt mpt/2.19 module load cmake/3.7.2 -module load netcdf-mpi/4.4.1.1 -module load pnetcdf/1.8.1 -module switch mpt mpt/2.16 +module load netcdf-mpi/4.6.1 +module load pnetcdf/1.11.0 echo "MODULE LIST..." module list @@ -31,10 +31,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone --branch develop https://github.com/PARALLELIO/ParallelIO src + git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout develop -git pull origin develop +git checkout master +git pull origin master ctest -S CTestScript.cmake,${model} -VV diff --git a/ctest/runcdash-nwsc-pgi.sh b/ctest/runcdash-nwscla-pgi.sh similarity index 55% rename from ctest/runcdash-nwsc-pgi.sh rename to ctest/runcdash-nwscla-pgi.sh index 20c09d619e34..08538185f447 100755 --- a/ctest/runcdash-nwsc-pgi.sh +++ b/ctest/runcdash-nwscla-pgi.sh @@ -9,16 +9,17 @@ fi module reset module unload netcdf -module swap intel pgi/16.5 -module load git/2.3.0 -module load cmake/3.0.2 -module load netcdf-mpi/4.4.1 -module load pnetcdf/1.7.0 +module swap intel pgi/17.9 +module swap mpt mpt/2.19 +module load git/2.10.2 +module load cmake/3.12.1 +module load netcdf-mpi/4.6.1 +module load pnetcdf/1.11.0 export CC=mpicc export FC=mpif90 - -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export MPI_TYPE_DEPTH=24 +export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard export PIO_COMPILER_ID=PGI-`$CC --version | head -n 2 | tail -n 1 | cut -d' ' -f2` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then @@ -27,10 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone --branch develop https://github.com/PARALLELIO/ParallelIO src + git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout develop -git pull origin develop +git checkout master +git pull origin master ctest -S CTestScript.cmake,${model} -VV diff --git a/ctest/runctest-nwsc.sh b/ctest/runctest-nwsc.sh deleted file mode 100755 index 64b8e9a81811..000000000000 --- a/ctest/runctest-nwsc.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -#============================================================================== -# -# This script defines how to run CTest on the NCAR Wyoming Supercomputing -# Center systems (yellowstone/caldera/geyser). -# -# This assumes the CTest model name (e.g., "Nightly") is passed to it when -# run. -# -#============================================================================== - -# Get the CTest script directory -scrdir=$1 - -# Get the CTest model name -model=$2 - -# Run the "ctest" command through an interactive parallel session -DAV_CORES=4 execca ctest -S ${scrdir}/CTestScript-Test.cmake,${model} -V diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index d9fe2ca3260f..0f43531ce462 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -58,7 +58,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = +OUTPUT_DIRECTORY = .. # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -291,7 +291,7 @@ OPTIMIZE_OUTPUT_VHDL = NO # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. -EXTENSION_MAPPING = F90=FortranFree +EXTENSION_MAPPING = f90=Fortran # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable @@ -756,7 +756,7 @@ WARN_FORMAT = "$file:$line: $text" # messages should be written. If left blank the output is written to standard # error (stderr). -WARN_LOGFILE = +WARN_LOGFILE = doxywarn.log #--------------------------------------------------------------------------- # Configuration options related to the input files @@ -768,15 +768,12 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = @CMAKE_CURRENT_SOURCE_DIR@/source \ - @CMAKE_CURRENT_SOURCE_DIR@/../src/flib \ +INPUT = @CMAKE_CURRENT_SOURCE_DIR@/../doc/source \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \ - @CMAKE_BINARY_DIR@/src/flib \ - @C_SRC_FILES@ + @FORTRAN_SRC_FILES@ \ + @C_SRC_FILES@ -# Uncomment this after the async code is fully merged into PIO. -# @CMAKE_CURRENT_SOURCE_DIR@/../src/clib # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -820,6 +817,7 @@ RECURSIVE = YES EXCLUDE = gptl \ @CMAKE_BINARY_DIR@/src/flib/*.dir \ @CMAKE_BINARY_DIR@/src/flib/genf90 \ + ../src/clib/uthash.h \ _UNUSED_ # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or @@ -1051,7 +1049,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = html +HTML_OUTPUT = docs # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). @@ -1113,7 +1111,8 @@ HTML_STYLESHEET = # list). For an example see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_EXTRA_STYLESHEET = @CMAKE_CURRENT_SOURCE_DIR@/customdoxygen.css +#HTML_EXTRA_STYLESHEET = ../../docs/customdoxygen.css +HTML_EXTRA_STYLESHEET = customdoxygen.css # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note diff --git a/doc/DoxygenLayout.xml b/doc/DoxygenLayout.xml index 1907176b46ae..563e9b1df9a2 100644 --- a/doc/DoxygenLayout.xml +++ b/doc/DoxygenLayout.xml @@ -1,7 +1,7 @@ - + diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 000000000000..83ea41382d9e --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,20 @@ +# This is part of PIO. It creates the doc Makefile. + +# Ed Hartnett 4/1/19 + +# Run doxygen, then confirm warning log file is empty. +all: + doxygen Doxyfile + cat doxywarn.log + [ ! -s doxywarn.log ] + +check: all + +# Include these subdirs to include the documention files in the +# distribution. +SUBDIRS = source images + +CLEANFILES = *.log + +EXTRA_DIST = CMakeLists.txt customdoxygen.css Doxyfile.in \ +DoxygenLayout.xml doxygen.sty diff --git a/doc/images/I_O_on_Few.png b/doc/images/I_O_on_Few.png new file mode 100644 index 000000000000..8909c4e1b6a5 Binary files /dev/null and b/doc/images/I_O_on_Few.png differ diff --git a/doc/images/I_O_on_Many_Async.png b/doc/images/I_O_on_Many_Async.png new file mode 100644 index 000000000000..aa32ad7863ea Binary files /dev/null and b/doc/images/I_O_on_Many_Async.png differ diff --git a/doc/images/I_O_on_Many_Intracomm.png b/doc/images/I_O_on_Many_Intracomm.png new file mode 100644 index 000000000000..e48efad2c250 Binary files /dev/null and b/doc/images/I_O_on_Many_Intracomm.png differ diff --git a/doc/images/Makefile.am b/doc/images/Makefile.am new file mode 100644 index 000000000000..1ab0bd06f213 --- /dev/null +++ b/doc/images/Makefile.am @@ -0,0 +1,9 @@ +# This is part of PIO. It creates the doc/images Makefile. + +# Ed Hartnett 5/25/19 + +# These are the images used in the documentation. +EXTRA_DIST = block-cyclic.png block-cyclic-rearr.png dof.png \ +dof-rearr.png PIO_Intracomm1.png PIO_Library_Architecture1.jpg \ +PIO_Decomposition.png I_O_on_Few.png I_O_on_Many_Intracomm.png \ +I_O_on_Many_Async.png PIO_Async.png diff --git a/doc/images/PIO_Async.png b/doc/images/PIO_Async.png new file mode 100644 index 000000000000..60711639a1c6 Binary files /dev/null and b/doc/images/PIO_Async.png differ diff --git a/doc/images/PIO_Decomposition.png b/doc/images/PIO_Decomposition.png new file mode 100644 index 000000000000..cfa0de11e569 Binary files /dev/null and b/doc/images/PIO_Decomposition.png differ diff --git a/doc/images/PIO_Intracomm1.png b/doc/images/PIO_Intracomm1.png new file mode 100644 index 000000000000..127b2ffe8f21 Binary files /dev/null and b/doc/images/PIO_Intracomm1.png differ diff --git a/doc/images/PIO_Library_Architecture1.jpg b/doc/images/PIO_Library_Architecture1.jpg new file mode 100644 index 000000000000..d8058dfb1844 Binary files /dev/null and b/doc/images/PIO_Library_Architecture1.jpg differ diff --git a/doc/source/Decomp.txt b/doc/source/Decomp.txt index 9ccce62a58f2..f47c8bf18e23 100644 --- a/doc/source/Decomp.txt +++ b/doc/source/Decomp.txt @@ -1,48 +1,42 @@ -/****************************************************************************** - * - * - * - * Copyright (C) 2009 - * - * Permission to use, copy, modify, and distribute this software and its - * documentation under the terms of the GNU General Public License is hereby - * granted. No representations are made about the suitability of this software - * for any purpose. It is provided "as is" without express or implied warranty. - * See the GNU General Public License for more details. - * - * Documents produced by Doxygen are derivative works derived from the - * input used in their production; they are not affected by this license. - * - */ /*! \page decomp Describing decompositions - - -One of the biggest challenges to working with PIO is setting up -the call to \ref PIO_initdecomp. The user must properly describe -how the data within each MPI tasks memory should be placed or retrieved from -disk. PIO provides two methods to rearrange data from compute tasks to IO tasks. -The first method, called box rearrangement is the only one provided in PIO1. -The second called subset rearrangement is introduced in PIO2. - -\section BOXREARR Box rearrangement +/** @page decomp Describing decompositions -In this method data is rearranged from compute to IO tasks such that -the arrangement of data on the IO tasks optimizes the call from the IO -tasks to the underlying (NetCDF) IO library. In this case each -compute task will transfer data to one or more IO tasks. +One of the biggest challenges to working with PIO is setting up the +decomposition of the data (Fortran users see @ref PIO_initdecomp, C +users @ref PIO_initdecomp_c). The user must properly describe how the +data within each MPI tasks memory should be placed or retrieved from +disk. + +@section The Compmap + +When initializing a new decomposition, each task calling +PIOc_init_decomp() or PIO_initdecomp(). +@image html PIO_Decomposition.png +@section Rearrangers +PIO provides two methods to rearrange data from compute tasks to +IO tasks. +@subsection BOXREARR Box rearrangement -\section SUBSETREARR Subset rearrangement +In this method data is rearranged from compute to IO tasks such that +the arrangement of data on the IO tasks optimizes the call from the IO +tasks to the underlying (NetCDF) IO library. In this case each compute +task will transfer data to one or more IO tasks. + +@subsection SUBSETREARR Subset rearrangement In this method each IO task is associated with a unique subset of compute tasks so that each compute task will transfer data to one and -only one IO task. Since this technique does not guarantee that data -on the IO node represents a contiguous block of data on the file it -may require multiple calls to the underlying (NetCDF) IO library. +only one IO task. Since this technique does not guarantee that data on +the IO node represents a contiguous block of data on the file it may +require multiple calls to the underlying (NetCDF) IO library. + +As an example suppose we have a global two dimensional grid of size +4x5 decomposed over 5 tasks. We represent the two dimensional grid in +terms of offset from the initial element ie -As an example suppose we have a global two dimensional grid of size 4x5 decomposed over 5 tasks. We represent the two dimensional grid in terms of offset from the initial element ie 
      0  1  2  3 
      4  5  6  7 
@@ -50,7 +44,9 @@ As an example suppose we have a global two dimensional grid of size 4x5 decompos
     12 13 14 15
     16 17 18 19
+ Now suppose this data is distributed over the compute tasks as follows: + 
 0: {   0  4 8 12  } 
 1: {  16 1 5 9  } 
@@ -60,11 +56,14 @@ Now suppose this data is distributed over the compute tasks as follows:
If we have 2 io tasks the Box rearranger would give: + 
 0: { 0  1  2  3  4  5  6  7  8  9  }
 1: { 10 11 12 13 14 15 16 17 18 19 }
+ While the subset rearranger would give: + 
 0: { 0  1  4  5  8  9  12 16 }
 1: { 2  3  6  7  10 11 13 14 15 17 18 19 }
@@ -72,17 +71,16 @@ While the subset rearranger would give:
 
 Note that while the box rearranger gives a data layout which is well
 balanced and well suited for the underlying io library, it had to
-communicate with every compute task to do so.  On the other hand the
-subset rearranger communicated with only a portion of the compute tasks
-but requires more work on the part of the underlying io library to complete
-the operation.
-
-Also note if every task is an IO task then the box rearranger will need
-to do an alltoall communication, while the subset rearranger does none.
-In fact using the subset rearranger with every compute task an IO task
-provides a measure of what you might expect the performance of the underlying
-IO library to be if it were used without PIO.
-
+communicate with every compute task to do so. On the other hand the
+subset rearranger communicated with only a portion of the compute
+tasks but requires more work on the part of the underlying io library
+to complete the operation.
+
+Also note if every task is an IO task then the box rearranger will
+need to do an alltoall communication, while the subset rearranger does
+none. In fact using the subset rearranger with every compute task an
+IO task provides a measure of what you might expect the performance of
+the underlying IO library to be if it were used without PIO.
 
 */
 
diff --git a/doc/source/Error.txt b/doc/source/Error.txt
index 72c0da23e205..1234c9f7c134 100644
--- a/doc/source/Error.txt
+++ b/doc/source/Error.txt
@@ -1,27 +1,16 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! 
+/*! 
 \page error Error Handling
 
 By default, PIO handles errors internally by printing a string
-describing the error and then calling mpi_abort.  Application
+describing the error and then calling mpi_abort. Application
 developers can change this behaivior with a call to
-\ref PIO_seterrorhandling
+\ref PIO_seterrorhandling or PIOc_set_iosystem_error_handling().
 
-\verbinclude errorhandle
+The three types of error handling are:
 
-\copydoc PIO_error_method
+1 - ::PIO_INTERNAL_ERROR abort on error from any task.
+
+2 - ::PIO_BCAST_ERROR broadcast error to all tasks on IO communicator
+
+3 - ::PIO_RETURN_ERROR return error and do nothing else
 */
diff --git a/doc/source/Examples.txt b/doc/source/Examples.txt
index e18d2926c7c0..932d27895788 100644
--- a/doc/source/Examples.txt
+++ b/doc/source/Examples.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page examp Examples
+/*! \page examp Examples
 
 ## Examples Included with PIO Distribution
 
diff --git a/doc/source/GettingStarted.txt b/doc/source/GettingStarted.txt
deleted file mode 100644
index 40fe348d5349..000000000000
--- a/doc/source/GettingStarted.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ 
-
-
-\tableofcontents
-
-/*! \page intro Introduction 
-
-PIO is a software interface layer designed to encapsolate the complexities of parallel IO and make it easier to replace the lower level software backend.  It currently supports  netcdf   and  pnetcdf .
-
-Basic description of how to optimize IO in a parallel environment...
-
-PIO calls are collective.  A MPI communicator is set in a call to \ref PIO_init and all tasks associated with that communicator must participate in all subsequent calls to PIO.  An application can make multiple calls to \ref PIO_init in order to support multiple MPI communicators.
-
-Begin by checking out a copy from [gitHub](https://github.com/PARALLELIO/ParallelIO) and installing on your system as per the instructions in the [Installation](@ref install) document. Take a look at examples of PIO usage in both complex and simple test programs in the [Examples](@ref examp) document. Finally, read through the [FAQ](@ref faq) to see if any remaining questions can be answered. 
-
-### Using PIO has three basic steps. ###
-
-1. Your program should call the \ref PIO_init function, and provide the MPI communicator (and the rank within that communicator) of the calling task. This call initializes an IO system type structure that will be used in subsequent file and decomposition functions.
-
-2. You can open a file for reading or writing with a call to \ref PIO_createfile or \ref PIO_openfile. In this call you will specify the file type: pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4c or pio_iotype_netcdf4p; along with the file name and optionally the netcdf mode.
-
-3. Finally, you can read or write decomposed data to the output file. You must describe the mapping between the organization of data in the file and that same data in the application space.  This is done in a call to \ref PIO_initdecomp. In the simplest call to this function, a one dimensional integer array is passed from each task, the values in the array represent the offset from the beginning of the array on file. (what happens next?)
-
-
-*/
diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt
index 6f2811522488..d0423abc1fc1 100644
--- a/doc/source/Installing.txt
+++ b/doc/source/Installing.txt
@@ -1,40 +1,92 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2013
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page install Installing PIO
-
-
-The PIO code is currently stored on github at . For questions about downloading or developing this code, consult the [CIME Git Wiki](https://github.com/CESM-Development/cime/wiki/CIME-Git-Workflow) or email .
-
-### Dependencies ###
-
-PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.1+) for I/O.
+/*! \page install Installing PIO
+
+
+## Getting the Release ##
+
+The PIO code is currently stored on github at
+. For questions about
+downloading or developing this code, consult the [CIME Git
+Wiki](https://github.com/CESM-Development/cime/wiki/CIME-Git-Workflow)
+or email .
+
+Download the latest release from the GitHub releases page. Download
+the release tarball, which will be named something like
+pio-2.4.3.tar.gz.
+
+## Dependencies ##
+
+PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) for I/O.
 Ideally, the NetCDF version should be built with MPI, which requires that it
 be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be 
 built with DAP support, which introduces a dependency on CURL.  Additionally,
 HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP.
 
-### Configuring with CMake ###
+@image html PIO_Library_Architecture1.jpg "PIO Library Architecture"
+
+## Building PIO C and Fortran Libraries ##
+
+Unpack the tarball and build with:
+
+
+./configure --enable-fortran
+make
+make check
+make install
+

+
+Environment flags CC and FC should be set to MPI C and Fortran
+compilers. CPPFLAGS may be set to a list of directories which have the
+include files for netCDF and pnetcdf. LDFLAGS may be set to a list of
+directories where libraries may be found.
+
+A complete example:
+
+
+export CPPFLAGS='-I/usr/local/pnetcdf-1.11.0_shared/include -I/usr/local/netcdf-c-4.7.0_hdf5-1.10.5_mpich-3.2/include -I/usr/local/netcdf-fortran-4.4.5_c_4.6.3_mpich-3.2/include'
+export LDFLAGS='-L/usr/local/pnetcdf-1.11.0_shared/lib  -L/usr/local/netcdf-c-4.7.0_hdf5-1.10.5_mpich-3.2/lib'
+export CC=mpicc
+export FC=mpifort
+export CFLAGS='-g -Wall'
+./configure --enable-fortran
+make check
+make install
+

+
+### Testing with MPI ###
+
+The tests are run as a bash script with called mpiexec to launch
+programs. If this will not work for the install system, use the
+--disable-test-runs option to configure. This will cause the tests to
+be built, but not run. The tests may be run them manually.
+
+### Optional GPTL Use ###
+
+PIO may optionally be built with the General Purpose Timing Library
+(GPTL). This is necessary for the performance testing program pioperf,
+but optional for the rest of the library and tests. To build with
+GPTL, include a path to its include and lib directories in the
+CPPFLAGS/LDFLAGS flags before running configure.
+
+### PIO Library Logging ###
+
+If built with --enable-logging, the PIO libraries will output logging
+statements to files (one per task) and stdout. Use the
+PIOc_set_log_level() function to turn on logging. This will have a
+negative impact on performance, when used, but helps with debugging.
+
+## Building with CMake ##
+
+A CMake build system is also avaible for the PIO C and Fortran
+libraries. User may prefer to use a CMake build instead of the
+autotools build.
+
+ - @ref mach_walkthrough
 
 To configure the build, PIO requires CMake version 2.8.12+.  The typical
 configuration with CMake can be done as follows:
 
-
     > CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source
 
-
 where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers
 for your system.
 
@@ -87,7 +139,7 @@ If you wish to install PIO in a safe location for use later with other
 software, you may set the `CMAKE_INSTALL_PREFIX` variable to point to the
 desired install location.
 
-### Building ###
+### Building with CMake ###
 
 Once you have successfully configured PIO with CMake in a build directory.
 From within the build directory, build PIO with:
@@ -98,7 +150,7 @@ From within the build directory, build PIO with:
 
 This will build the `pioc` and `piof` libraries.
 
-### Testing ###
+### Testing with CMake ###
 
 If you desire to do testing, and `PIO_ENABLE_TESTS=ON` (which is the default
 setting), you may build the test executables with:
@@ -125,11 +177,21 @@ immediately with:
 
 (similar to the typical `make check` Autotools target).
 
-*ANOTHER NOTE:* These tests are designed to run in parallel.
-If you are on one of the supported supercomputing platforms (i.e., NERSC, NWSC, ALCF, 
-etc.), then the `ctest` command will assume that the tests will be run in an appropriately configured and scheduled parallel job.  This can be done by requesting an interactive session from the login nodes and then running `ctest` from within the interactive terminal.  Alternatively, this can be done by running the `ctest` command from a job submission script.  It is important to understand, however, that `ctest` itself will preface all of the test executable commands with the appropriate `mpirun`/`mpiexec`/`runjob`/etc. Hence, you should not further preface the `ctest` command with these MPI launchers.
+*ANOTHER NOTE:* These tests are designed to run in parallel.  If you
+are on one of the supported supercomputing platforms (i.e., NERSC,
+NWSC, ALCF, etc.), then the `ctest` command will assume that the tests
+will be run in an appropriately configured and scheduled parallel job.
+This can be done by requesting an interactive session from the login
+nodes and then running `ctest` from within the interactive terminal.
+Alternatively, this can be done by running the `ctest` command from a
+job submission script.  It is important to understand, however, that
+`ctest` itself will preface all of the test executable commands with
+the appropriate `mpirun`/`mpiexec`/`runjob`/etc. Hence, you should not
+further preface the `ctest` command with these MPI launchers.
+
+ - @ref test
 
-### Installing ###
+### Installing with CMake ###
 
 Once you have built the PIO libraries, you may install them in the location
 specified by the `CMAKE_INSTALL_PREFIX`.  To do this, simply type:
@@ -142,7 +204,7 @@ If the internal GPTL libraries were built (because GPTL could not be found
 and the `PIO_ENABLE_TIMING` variable is set to `ON`), then these libraries
 will be installed with PIO.
 
-### Examples ###
+### CMake Build Examples ###
 
 From within the build directory, build the PIO examples with:
 
diff --git a/doc/source/Introduction.txt b/doc/source/Introduction.txt
new file mode 100644
index 000000000000..888c44a84f84
--- /dev/null
+++ b/doc/source/Introduction.txt
@@ -0,0 +1,89 @@
+@tableofcontents
+
+/** @page intro Introduction
+
+Performing I/O is straightforward when a small number of processors
+are being used.
+
+@image html I_O_on_Few.png "I/O on One or a Few Processors"
+
+Parallel I/O does not scale to thousands of processors, because the
+parallel disk systems do not support the bandwitdh to allow thousands
+of processors to access the disk at the same time. As a result, most
+of the processors will have to wait to do I/O.
+
+An obvious solution is to designate a small number of processors to do
+I/O, and use the rest for computation.
+
+@image html I_O_on_Many_Intracomm.png "I/O on Many Processors Intracomm"
+
+PIO provides a netCDF-like API which provides this service. User code
+is written as if parallel I/O is being used from every processor, but,
+under the hood, PIO uses the I/O processors to do all data access.
+
+With Intracomm Mode, the I/O processors are a subset of the
+computational processors, and only one computational unit is
+supported. In Async Mode, the I/O processors are dedicated to I/O, and
+do not perform computation. Also, more than one computational unit may
+be designated.
+
+@image html I_O_on_Many_Async.png "I/O on Many Processors Async"
+
+The user initializes the PIO IO System, designating some processors
+for I/O, others for computation.
+
+PIO decompositions and distributed arrays allow the code to be written
+in terms of the local, distributed sub-array (see @ref decomp). PIO
+handles the stitching of all data into the correct global space in a
+netCDF variable.
+
+PIO also allows for the creation of multiple computational units. Each
+computational unit consists of many processors. I/O for all
+computational units is accomplished through one set of dedicated I/O
+processors (see @ref iosystem).
+
+PIO uses 
+netcdf  and  pnetcdf to
+read and write the netCDF files (see @ref install).
+
+## Basic description of how to optimize IO in a parallel environment:
+
+PIO calls are collective.  A MPI communicator is set in a call to @ref
+PIO_init and all tasks associated with that communicator must
+participate in all subsequent calls to PIO.  An application can make
+multiple calls to @ref PIO_init in order to support multiple MPI
+communicators.
+
+Begin by getting and unpacking the most recent release of PIO from
+[gitHub](https://github.com/PARALLELIO/ParallelIO/releases) and
+installing on your system as per the instructions in the
+[Installation](@ref install) document. Take a look at examples of PIO
+usage in both complex and simple test programs in the [Examples](@ref
+examp) document. Finally, read through the [FAQ](@ref faq) to see if
+any remaining questions can be answered.
+
+### Using PIO has three basic steps. ###
+
+1. Your program should call the @ref PIO_init function, and provide
+the MPI communicator (and the rank within that communicator) of the
+calling task. This call initializes an IO system type structure that
+will be used in subsequent file and decomposition functions.
+
+2. You can open a file for reading or writing with a call to @ref
+PIO_createfile or @ref PIO_openfile. In this call you will specify the
+file type: pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4c
+or pio_iotype_netcdf4p; along with the file name and optionally the
+netcdf mode.
+
+3. Finally, you can read or write decomposed data to the output
+file. You must describe the mapping between the organization of data
+in the file and that same data in the application space.  This is done
+in a call to @ref PIO_initdecomp. In the simplest call to this
+function, a one dimensional integer array is passed from each task,
+the values in the array represent the offset from the beginning of the
+array on file.
+
+
+*/
diff --git a/doc/source/Makefile.am b/doc/source/Makefile.am
new file mode 100644
index 000000000000..97f8e16dc3fe
--- /dev/null
+++ b/doc/source/Makefile.am
@@ -0,0 +1,8 @@
+# This is part of PIO. It creates the doc/source Makefile.
+
+# Ed Hartnett 5/20/19
+
+EXTRA_DIST = api.txt CAMexample.txt Decomp.txt faq.txt Installing.txt	\
+Testing.txt base.txt c_api.txt contributing_code.txt Error.txt		\
+Examples.txt Introduction.txt mach_walkthrough.txt			\
+testpio_example.txt users_guide.txt iosystem.txt
diff --git a/doc/source/Testing.txt b/doc/source/Testing.txt
index 1d15f73b7abb..0ef7c48a31b5 100644
--- a/doc/source/Testing.txt
+++ b/doc/source/Testing.txt
@@ -1,29 +1,24 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page test Testing
+/*! \page test Cmake Testing Information
 
-## Building PIO2 Tests
+## Building PIO Tests
 
-To build both the Unit and Performance tests for PIO2, follow the general instructions for building PIO2 in either the [Installation](@ref install) page or the [Machine Walk-Through](@ref mach_walkthrough) page. During the Build step after (or instead of) the **make** command, type **make tests**. 
+To build both the Unit and Performance tests for PIO, follow the
+general instructions for building PIO in either the
+[Installation](@ref install) page or the [Machine Walk-Through](@ref
+mach_walkthrough) page. During the Build step after (or instead of)
+the **make** command, type **make tests**.
 
-## PIO2 Unit Tests
+## PIO Unit Tests
 
-The Parallel IO library comes with more than 20 built-in unit tests to verify that the library is installed and working correctly. These tests utilize the _CMake_ and _CTest_ automation framework. Because the Parallel IO library is built for parallel applications, the unit tests should be run in a parallel environment. The simplest way to do this is to submit a PBS job to run the **ctest** command. 
+The Parallel IO library comes with more than 20 built-in unit tests to
+verify that the library is installed and working correctly. These
+tests utilize the _CMake_ and _CTest_ automation framework. Because
+the Parallel IO library is built for parallel applications, the unit
+tests should be run in a parallel environment. The simplest way to do
+this is to submit a PBS job to run the **ctest** command.
 
-For a library built into the example directory `/scratch/user/PIO_build/`, an example PBS script would be:
+For a library built into the example directory
+`/scratch/user/PIO_build/`, an example PBS script would be:
 
       #!/bin/bash
 
@@ -101,12 +96,18 @@ On Yellowstone, the unit tests can run using the **execca** or **execgy** comman
       > setenv DAV_CORES 4
       > execca ctest
 
-## PIO2 Performance Test
+## PIO Performance Test
 
-To run the performance tests, you will need to add two files to the **tests/performance** subdirectory of the PIO build directory. First, you will need a decomp file. You can download one from our google code page here: 
-https://svn-ccsm-piodecomps.cgd.ucar.edu/trunk/ . 
-You can use any of these files, and save them to your home or base work directory. Secondly, you will need to add a namelist file, named "pioperf.nl". Save this file in the directory with your **pioperf** executable (this is found in the **tests/performance** subdirectory of the PIO build directory).
+To run the performance tests, you will need to add two files to the
+**tests/performance** subdirectory of the PIO build directory. First,
+you will need a decomp file. You can download one from our google code
+page here: https://svn-ccsm-piodecomps.cgd.ucar.edu/trunk/ .
 
+You can use any of these files, and save them to your home or base
+work directory. Secondly, you will need to add a namelist file, named
+"pioperf.nl". Save this file in the directory with your **pioperf**
+executable (this is found in the **tests/performance** subdirectory of
+the PIO build directory).
 
 The contents of the namelist file should look like:
 
@@ -124,7 +125,11 @@ The contents of the namelist file should look like:
      
      /
 
-Here, the second line ("decompfile") points to the path for your decomp file (wherever you saved it). For the rest of the lines, each item added to the list adds another test to be run. For instance, to test all of the types of supported IO, your pio_typenames would look like:
+Here, the second line ("decompfile") points to the path for your
+decomp file (wherever you saved it). For the rest of the lines, each
+item added to the list adds another test to be run. For instance, to
+test all of the types of supported IO, your pio_typenames would look
+like:
 
      pio_typenames = 'pnetcdf','netcdf','netcdf4p','netcdf4c'
 
@@ -140,7 +145,10 @@ To test with both of the rearranger algorithms:
 
      rearrangers = 1,2
 
-(Each rearranger is a different algorithm for converting from data in memory to data in a file on disk. The first one, BOX, is the older method from PIO1, the second, SUBSET, is a newer method that seems to be more efficient in large numbers of tasks)
+(Each rearranger is a different algorithm for converting from data in
+memory to data in a file on disk. The first one, BOX, is the older
+method from PIO1, the second, SUBSET, is a newer method that seems to
+be more efficient in large numbers of tasks)
 
 To test with different numbers of variables:
 
@@ -148,7 +156,9 @@ To test with different numbers of variables:
 
 (The more variables you use, the higher data throughput goes, usually)
 
-To run, submit a job with 'pioperf' as the executable, and at least as many tasks as you have specified in the decomposition file. On yellowstone, a submit script could look like:
+To run, submit a job with 'pioperf' as the executable, and at least as
+many tasks as you have specified in the decomposition file. On
+yellowstone, a submit script could look like:
 
      #!/bin/tcsh
 
@@ -171,11 +181,23 @@ RESULT: write       BOX         4        30         2       16.9905924688
 
 You can decode this as:
 1. Read/write describes the io operation performed
+
 2. BOX/SUBSET is the algorithm for the rearranger (as described above)
-3. 4 [1-4] is the io library used for the operation. The options here are [1] Parallel-netcdf [2] NetCDF3 [3] NetCDF4-Compressed [4] NetCDF4-Parallel
-4. 30 [any number] is the number of io-specific tasks used in the operation. Must be less than the number of MPI tasks used in the test.
-5. 2 [any number] is the number of variables read or written during the operation
-6. 16.9905924688 [any number] is the Data Rate of the operation in MB/s. This is the important value for determining performance of the system. The higher this numbre is, the better the PIO2 library is performing for the given operation.
+
+3. 4 [1-4] is the io library used for the operation. The options here
+are [1] Parallel-netcdf [2] NetCDF3 [3] NetCDF4-Compressed [4]
+NetCDF4-Parallel
+
+4. 30 [any number] is the number of io-specific tasks used in the
+operation. Must be less than the number of MPI tasks used in the test.
+
+5. 2 [any number] is the number of variables read or written during
+the operation
+
+6. 16.9905924688 [any number] is the Data Rate of the operation in
+MB/s. This is the important value for determining performance of the
+system. The higher this numbre is, the better the PIO2 library is
+performing for the given operation.
 
 _Last updated: 05-17-2016_
 */
diff --git a/doc/source/api.txt b/doc/source/api.txt
index 275224e160f9..0e36baa82afb 100644
--- a/doc/source/api.txt
+++ b/doc/source/api.txt
@@ -1,61 +1,42 @@
- /*!  \page api PIO user interface
+ /*!  \page api PIO Fortran Interface
   This is a list of all user interface routines:
 
-  \section api_fileops PIO file Operations
-     - \ref PIO_openfile
-     - \ref PIO_createfile
-     - \ref PIO_syncfile
-     - \ref PIO_closefile
-  \section api_system PIO startup and shutdown routines
-     - \ref PIO_init
-     - \ref PIO_finalize
-  \section api_decomp PIO decomposition routines
-    - \ref PIO_initdecomp
-    - \ref PIO_freedecomp
-  \section readwrite Reading and Writing distributed variables
-    - \ref PIO_read_darray
-    - \ref PIO_write_darray
-  \section utility Utility routines
-    - \ref PIO_set_hint
-    - \ref PIO_setframe
-    - \ref PIO_advanceframe
-    - \ref PIO_setdebuglevel
-    - \ref PIO_seterrorhandling
-    - \ref PIO_get_local_array_size
+  \section api_system PIO Startup and Shutdown
+    - \ref PIO_init
+    - \ref PIO_finalize
     - \ref PIO_getnumiotasks
     - \ref PIO_set_blocksize
-  \section netcdf NetCDF format specific routines 
-     Also see: http://www.unidata.ucar.edu/software/netcdf/docs/
-     \subsection putget Reading/Writing netcdf metadata  
-         - \ref PIO_get_att
-         - \ref PIO_put_att
-         - \ref PIO_get_var
-         - \ref PIO_put_var
-     \subsection utilnc Netcdf utility routines
+    - \ref PIO_set_hint
+  \section api_fileops File Operations
+    - \ref PIO_openfile
+    - \ref PIO_createfile
+    - \ref PIO_syncfile
     - \ref PIO_enddef
-    - \ref PIO_redef
-    - \ref PIO_def_dim 
-    - \ref PIO_def_var
-     \subsection inqnc NetCDF file inquiry routines
+    - \ref PIO_closefile
+  \section inqnc Inquiry
     - \ref PIO_inquire
-    - \ref PIO_inq_attname
-    - \ref PIO_inq_att
-    - \ref PIO_inq_attlen
-    - \ref PIO_inq_var
-    - \ref PIO_inq_varid
-    - \ref PIO_inq_varname
-    - \ref PIO_inq_vartype
-    - \ref PIO_inq_varndims
-    - \ref PIO_inq_vardimid
-    - \ref PIO_inq_varnatts
-    - \ref PIO_inq_dimid
-    - \ref PIO_inq_dimname
-    - \ref PIO_inq_dimlen
-    - \ref PIO_inq_ndims
-    - \ref PIO_inq_nvars
-    - \ref PIO_inq_natts
-    - \ref PIO_inquire_variable
-    - \ref PIO_inquire_dimension
     - \ref PIO_inquire_dimension
+    - \ref PIO_inquire_variable
+    - \ref PIO_inq_att
+  \section metadata Defining Metadata
+    - \ref PIO_def_dim 
+    - \ref PIO_def_var
+    - \ref PIO_get_att
+    - \ref PIO_put_att
+  \section api_decomp Distributed Arrays
+    - \ref PIO_initdecomp
+    - \ref PIO_freedecomp
+    - \ref PIO_setframe
+    - \ref PIO_read_darray
+    - \ref PIO_write_darray
+    - \ref PIO_get_local_array_size
+  \section standard_arrays Standard Arrays
+    - \ref PIO_get_var
+    - \ref PIO_put_var
+  \section utility Errors
+    - \ref PIO_seterrorhandling
+    - \ref PIO_setdebuglevel
+
+Also see: http://www.unidata.ucar.edu/software/netcdf/docs/
 
 */
diff --git a/doc/source/base.txt b/doc/source/base.txt
index 210c9bbfbf34..126bba5be2a6 100644
--- a/doc/source/base.txt
+++ b/doc/source/base.txt
@@ -1,52 +1,32 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ 
 /*! 
 
-\mainpage Parallel I/O library (PIO)
+@mainpage Parallel I/O Libraries (PIO)
 
-\author Jim Edwards
-\author John M. Dennis
-\author Mariana Vertenstein
-\author Edward Hartnett
+@author Jim Edwards
+@author John M. Dennis
+@author Mariana Vertenstein
+@author Edward Hartnett
 
-The Parallel I/O (PIO) library has been developed over several years
-to improve the ability of component models of the Community Earth
-System Model (CESM) to perform I/O.  We believe that the interface is
-sufficiently general to be useful to a broader spectrum of
-applications.  The focus of development has been on backend tools that
-use the NetCDF file format.  PIO currently supports NetCDF and PnetCDF
-as backend libraries, both can be linked and used with runtime options
-controlling which is used for a given file.
+The Parallel IO libraries (PIO) are high-level parallel I/O C and
+Fortran libraries for structured grid applications. PIO provides a
+netCDF-like API, and allows users to designate some subset of
+processors to perform IO. Computational code calls netCDF-like
+functions to read and write data, and PIO uses the IO processors to
+perform all necessary IO.
 
-PIO2 represents a significant rewrite of the PIO library and includes
-a C API as well as the original F90 API.  A new decomposition strategy
-has been introduced which gives the user more ability to tune io communications.
+PIO also supports the creation of multiple computation components,
+each containing many processors, and one shared set of IO
+processors. The computational components can perform write operation
+asynchronously, and the IO processors will take care of all storage
+interaction.
 
-This user's guide provides information about the PIO library and examples on how it can be used. 
-Please review the ChangeLog that is included with the distribution for up-to-date release information.
+This user's guide provides information about the PIO library and
+examples on how it can be used. Please watch the PIO GitHub site
+[https://github.com/NCAR/ParallelIO] for announcements and new
+releases.
 
- - \ref intro
- - \ref install
- - \ref mach_walkthrough
- - \ref decomp
- - \ref error
- - \ref test
- - \ref examp
- - \ref faq
- - \ref api
- - \ref contributing_code
+ - @ref install
+ - @ref users_guide
+ - @ref api
+ - @ref c_api
 */
diff --git a/doc/source/c_api.txt b/doc/source/c_api.txt
new file mode 100644
index 000000000000..e9d3ed3f62ee
--- /dev/null
+++ b/doc/source/c_api.txt
@@ -0,0 +1,50 @@
+ /*!  \page c_api PIO C Interface
+  This is a list of all user interface routines:
+
+  \section api_system_c PIO Startup and Shutdown
+     - \ref PIO_init_c
+     - \ref PIO_finalize_c
+  \section api_fileops_c PIO File Operations
+     - \ref PIO_open_file_c
+     - \ref PIO_create_file_c
+     - \ref PIO_sync_file_c
+     - \ref PIO_close_file_c
+  \section api_decomp_c PIO Decompositions
+    - \ref PIO_initdecomp_c
+    - \ref PIO_freedecomp_c
+  \section readwrite_c Reading and Writing Distributed Arrays
+    - \ref PIO_read_darray_c
+    - \ref PIO_write_darray_c
+    - \ref PIO_setframe_c
+  \section utility_c Utility 
+    - \ref PIO_set_hint_c
+    - \ref PIO_error_method_c
+    - \ref PIO_get_local_array_size_c
+    - \ref PIO_getnumiotasks_c
+    - \ref PIO_set_blocksize_c
+  \section netcdf_c NetCDF-Like Functions
+     Also see: http://www.unidata.ucar.edu/software/netcdf/docs/
+     \subsection utilnc_c File Operations
+       - \ref PIO_enddef_c
+       - \ref PIO_redef_c
+     \subsection write_metadata_c Writing Metadata   
+       - \ref PIO_def_dim_c
+       - \ref PIO_def_var_c
+       - \ref PIO_put_att_c
+     \subsection putget_c Reading/Writing Data
+       - \ref PIO_get_vara_c
+       - \ref PIO_get_var_c
+       - \ref PIO_get_var1_c
+       - \ref PIO_get_vars_c
+       - \ref PIO_put_vara_c
+       - \ref PIO_put_var_c
+       - \ref PIO_put_var1_c
+       - \ref PIO_put_vars_c
+     \subsection inqnc_c Learn about Files and Metadata
+       - \ref PIO_inq_c
+       - \ref PIO_get_att_c
+       - \ref PIO_inq_att_c
+       - \ref PIO_inq_var_c
+       - \ref PIO_inq_dim_c
+
+*/
diff --git a/doc/source/contributing_code.txt b/doc/source/contributing_code.txt
index 264484f29510..cf7df469c199 100644
--- a/doc/source/contributing_code.txt
+++ b/doc/source/contributing_code.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2016
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page contributing_code Guide for Contributors
+/*! @page contributing_code Guide for Contributors
 
 # Introduction #
 
@@ -50,7 +35,7 @@ add to it, carefully.
 
  • Use spaces not tabs.
 
  • 4 spaces is the unit of intendation.
 
  • Indentation as defined by the "linux" style in emacs (see below).
-
  • Use spaces around most operators (=+-*/) not pointer or prefix/postfile (*++--)
+
  • Use spaces around most operators (= + - * /) not pointer or prefix/postfile (* ++ --)
 
  • Spaces after most keywords (if, for, while, etc.)
 
  • No spaces after function name.
 
@@ -133,140 +118,20 @@ all issues have been resolved.
 
      
 
 
    • Programmers begin work on a feature or fix by branching from
-develop.
+master.
 
-
    • When a branch is ready, it is submitted to code review.
+
    • When a branch is ready, it is submitted to code review via pull
+request. Travis CI testing is automatically run on the PR.
 
 
    • When code review is complete, and the changes are approved, the PR
-is merged into the develop branch.
+is merged into the master branch.
 
-
    • Mutliple merges into the develop branch may take place between
-test cycles. (???)
-
-
    • The develop branch is tested automatically by Jenkins.
-
-
    • The develop branch is tested periodically by CDash (every ~6
-hours).
-
-
    • After all jenkins and Cdash builds complete successfully, with all
-tests passing, and no warnings, the PR is merged into master by the
-integrator.
-
-
    • Multiple PRs may be merged to master between test cycles. (???)
+
    • The master branch is tested periodically by CDash (every ~6
+hours). Any test failures and the merge to master may be rolled back.
 
 
    • The branch is then deleted by whomever merged it to master.
 
-
    • The master branch is then tested on Jenkins.
-
-
    • The master branch is tested on CDash. Any test failures and the
-merge to master will be rolled back.
-
 
    
 
-## Formatting Example ##
-
-
    -/** 
- * \@ingroup PIOc_inq_attname
- * The PIO-C interface for the NetCDF function nc_inq_attname.
- *
- * This routine is called collectively by all tasks in the communicator 
- * ios.union_comm. For more information on the underlying NetCDF commmand
- * please read about this function in the NetCDF documentation at: 
- * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
- *
- * \@param ncid the ncid of the open file, obtained from
- * PIOc_openfile() or PIOc_createfile().
- * \@param varid the variable ID.
- * \@param attnum the attribute ID.
- * \@return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
- */
-int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) 
-{
-    iosystem_desc_t *ios;  /* Pointer to io system information. */
-    file_desc_t *file;     /* Pointer to file information. */
-    int ierr = PIO_NOERR;  /* Return code from function calls. */
-    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
-
-    LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid,
-         attnum));
-
-    /* Find the info about this file. */
-    if (!(file = pio_get_file_from_id(ncid)))
-        return PIO_EBADID;
-    ios = file->iosystem;
-
-    /* If async is in use, and this is not an IO task, bcast the parameters. */
-    if (ios->async_interface)
-    {
-        if (!ios->ioproc)
-        {
-            int msg = PIO_MSG_INQ_ATTNAME;
-            char name_present = name ? true : false;
-
-            if(ios->compmaster) 
-                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-            
-            if (!mpierr)
-                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
-            if (!mpierr)
-                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
-            if (!mpierr)
-                mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm);
-            if (!mpierr)
-                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
-        }
-
-        /* Handle MPI errors. */
-        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
-        if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
-    }
-
-    /* If this is an IO task, then call the netCDF function. */
-    if (ios->ioproc)
-    {
-#ifdef _PNETCDF
-        if (file->iotype == PIO_IOTYPE_PNETCDF)
-            ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);
-#endif /* _PNETCDF */
-#ifdef _NETCDF
-        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
-            ierr = nc_inq_attname(file->fh, varid, attnum, name);
-#endif /* _NETCDF */
-        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
-    }
-
-    /* Broadcast and check the return code. */
-    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-    {
-        check_mpi(file, mpierr, __FILE__, __LINE__);            
-        return PIO_EIO;
-    }
-    check_netcdf(file, ierr, __FILE__, __LINE__);
-    
-    /* Broadcast results to all tasks. Ignore NULL parameters. */
-    if (!ierr)
-        if (name)
-        {
-            int namelen = strlen(name);
-            if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-                check_mpi(file, mpierr, __FILE__, __LINE__);
-            if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot,
-                                    ios->my_comm)))
-                check_mpi(file, mpierr, __FILE__, __LINE__);
-        }
-
-    return ierr;
-}
-
    
-
-## Further Information ##
-
-
    For style issues not already covered in this document, see this style
-guide.
 
-_Last updated: 05-16-2016_
 */
diff --git a/doc/source/faq.txt b/doc/source/faq.txt
index 99d7c46b7a47..c3c4bca33684 100644
--- a/doc/source/faq.txt
+++ b/doc/source/faq.txt
@@ -1,31 +1,42 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2009 
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page faq Frequently Asked Questions 
-
-   Here is a list of frequently asked questions and their answers.
+/*! \page faq Frequently Asked Questions 
+
+Here is a list of frequently asked questions and their answers.
+   
 
    
-
    How do I specify which tasks perform IO?  
    
-    
    This is done in the call to \ref PIO_init which has two interfaces: init_intracom and init_intercom.  
-  
    •  In the init_intracom interface, use the num_iotasks and stride variables to specify the total number of io tasks and the stride between them with respect to the mpi communicator, comp_comm, which is provided.  You can also use the optional base argument to shift the first IO task away from the first computational task, this is often desirable because the applications first computational task often has higher memory requirements than other tasks.   IO tasks are a subset of the tasks defined in comp_comm.
-
    •  In the init_intercom interface, IO tasks are a disjoint set of tasks from those in the computational communicator.  This interface is still experimental and not recommended for production use at this time. 
+
+
      How do I specify which tasks perform IO?
      
+
+
      This is done in the call to \ref PIO_init which has two
+interfaces: init_intracom and init_intercom.
+    
+  
        
+
+    
      •  In the init_intracom interface, use the num_iotasks and
+    stride variables to specify the total number of io tasks and the
+    stride between them with respect to the mpi communicator,
+    comp_comm, which is provided.  You can also use the optional base
+    argument to shift the first IO task away from the first
+    computational task, this is often desirable because the
+    applications first computational task often has higher memory
+    requirements than other tasks.  IO tasks are a subset of the tasks
+    defined in comp_comm.
+
+    
      •  In the init_intercom interface, IO tasks are a disjoint set
+    of tasks from those in the computational communicator.
+
 
      
-Note that num_iotasks is the maximum number of IO tasks to use for an IO operation.   The size of the field being read or written along with the tunable blocksize parameter, \ref PIO_set_blocksize, determines the actual number of tasks used for a given IO operation.
+
+Note that num_iotasks is the maximum number of IO tasks to use for an
+IO operation.  The size of the field being read or written along with
+the tunable blocksize parameter, \ref PIO_set_blocksize, determines
+the actual number of tasks used for a given IO operation.
 
      
-
      How do I test if PIO is installed and working correctly? 
       
-    
      The PIO Library distribution contains a testpio subdirectory with a number of programs to test the PIO library. Please see the \ref examp page for details. 
      
+
+
      How do I test if PIO is installed and working correctly?
       
+
+
      The PIO Library distribution contains tests for PIO. They are run
+my 'make check'. The tests use mpiexec to run tests on 4, 8, or 16
+processors. 
      
 
 
    
 */
diff --git a/doc/source/iosystem.txt b/doc/source/iosystem.txt
new file mode 100644
index 000000000000..5acce7fc9289
--- /dev/null
+++ b/doc/source/iosystem.txt
@@ -0,0 +1,24 @@
+/** @page iosystem Initializing the IO System
+
+Using PIO begins with initializing the IO System. This sets up the MPI
+communicators with the computational and I/O processors.
+
+When the IO System is created, an IOSystem ID is returned and must be
+used in future PIO calls. The IOSystem ID is returned by C functions
+PIOc_Init_Intracomm() and PIOc_init_async(). Fortran users see @ref
+PIO_init.
+
+When the user program is complete, the IOSystem should be released by
+calling C function PIOc_finalize() or Fortran function piolib_mod::finalize()
+for each open IOSystem.
+
+@section intracomm_mode Intracomm Mode
+
+@image html PIO_Intracomm1.png "PIO Intracomm Mode"
+
+@section async_mode Async Mode
+
+@image html PIO_Async.png "PIO Async Mode"
+
+*/
+
diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt
index 05cd8776d659..357b88821367 100644
--- a/doc/source/mach_walkthrough.txt
+++ b/doc/source/mach_walkthrough.txt
@@ -1,21 +1,8 @@
-/******************************************************************************
- *
- * 
- *
- * Copyright (C) 2013
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby 
- * granted. No representations are made about the suitability of this software 
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page mach_walkthrough Install Walk-through
-
-This document provides specific instructions for installing PIO using a variety of compilers on a few commonly used super computers. Click on the link below to go directly to the machine of interest.
+/*! \page mach_walkthrough CMake Install Walk-through
+
+This document provides specific instructions for installing PIO using
+a variety of compilers on a few commonly used super computers. Click
+on the link below to go directly to the machine of interest.
 
 -    Yellowstone (NCAR's 1.5-petaflop IBM Supercomputer)
 -    Edison (A NERSC Cray XC30 Supercomputer)
@@ -30,11 +17,16 @@ This document provides specific instructions for installing PIO using a variety
 
      
 
    1. Directory setup
 
-Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. 
+Download a copy of the PIO source into a sub-directory of your working
+directory (refered to here as the PIO_source directory). Create
+another sub-directory for the build (refered to here as the PIO_build
+directory) and 'cd' into it.
 
 
    2. Modules
 
-Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Yellowstone. 
+Modules required for installation depend on your prefered
+compiler. Issue the commands below to set the module environment for
+building PIO on Yellowstone.
 
 + Intel
 
@@ -170,7 +162,7 @@ Download a copy of the PIO source into a sub-directory of your working directory
 
 It is not necessary to edit your .soft file on Mira inorder to build PIO. Execute the following commands to temporarily load packages into your softenv. These packages use the IBM/XL compiler.
      
   %> soft add +mpiwrapper-xl (or switch from the default in your softenv)
      
-  %> soft add @ibm-compilers-2015-02
      
+  %> soft add \@ibm-compilers-2015-02
      
   %> soft add +cmake
      
   %> soft add +git
      
 
@@ -374,7 +366,7 @@ found.
 
      cd hdf5-1.10.1
 CC=mpicc ./configure --with-zlib=/usr/local/zlib-1.2.11_mpich-3.2 --with-szlib=/usr/local/szip-2.1_mpich-3.2 --prefix=/usr/local/hdf5-1.10.1_mpich-3.2 --enable-parallel
 make all check
-sudo PATH=$PATH:/usr/local/bin make install
+sudo PATH=$PATH:/usr/local/bin make install
      
 
 
    3. Installing NetCDF-4 C Library
 
diff --git a/doc/source/testpio_example.txt b/doc/source/testpio_example.txt
index 07fb3b04b478..c5d4ef7e5ca5 100644
--- a/doc/source/testpio_example.txt
+++ b/doc/source/testpio_example.txt
@@ -1,19 +1,3 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */
 /*! \page testpio_example testpio: a regression and benchmarking code
 
 The testpio directory, included with the release package, tests both the accuracy 
diff --git a/doc/source/users_guide.txt b/doc/source/users_guide.txt
new file mode 100644
index 000000000000..f97123ac50af
--- /dev/null
+++ b/doc/source/users_guide.txt
@@ -0,0 +1,20 @@
+/*! 
+
+@page users_guide PIO User's Guide
+
+This user's guide provides information about the PIO library and
+examples on how it can be used. Please watch the PIO GitHub site
+[https://github.com/NCAR/ParallelIO] for announcements and new
+releases.
+
+ - @ref intro
+ - @ref iosystem
+ - @ref decomp
+ - @ref error
+ - @ref examp
+ - @ref faq
+ - @ref api
+ - @ref c_api
+ - @ref contributing_code
+
+*/
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index cd4fc6c23dfe..b8fa8191de9b 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -9,7 +9,7 @@ if (PIO_ENABLE_FORTRAN)
     ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
   else()
-    ADD_SUBDIRECTORY(f03)
+#    ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
     #    ADD_SUBDIRECTORY(cxx)
   endif()
diff --git a/examples/Makefile.am b/examples/Makefile.am
new file mode 100644
index 000000000000..38c170d4579d
--- /dev/null
+++ b/examples/Makefile.am
@@ -0,0 +1,6 @@
+# This is part of PIO. It creates the examples Makefile.
+
+# Ed Hartnett
+
+SUBDIRS = c
+
diff --git a/examples/c/Makefile.am b/examples/c/Makefile.am
new file mode 100644
index 000000000000..50750dd91322
--- /dev/null
+++ b/examples/c/Makefile.am
@@ -0,0 +1,22 @@
+## This is the automake file for building the C examples for the PIO
+## library.
+
+# Ed Hartnett 5/7/18
+
+# Link to our assembled library.
+AM_LDFLAGS = ${top_builddir}/src/clib/libpio.la
+AM_CPPFLAGS = -I$(top_srcdir)/src/clib
+
+# Build the tests for make check.
+check_PROGRAMS = example1 examplePio darray_no_async
+
+if RUN_TESTS
+# Tests will run from a bash script.
+TESTS = run_tests.sh
+endif # RUN_TESTS
+
+# Distribute the test script.
+EXTRA_DIST = run_tests.sh
+
+# Clean up files produced during testing.
+CLEANFILES = *.nc *.log
diff --git a/examples/c/darray_async.c b/examples/c/darray_async.c
index 1b15607f0f62..1dce8144200b 100644
--- a/examples/c/darray_async.c
+++ b/examples/c/darray_async.c
@@ -9,6 +9,7 @@
  * This example can be run in parallel for 4 processors.
  */
 
+#include "config.h"
 #include 
 #include 
 #include 
@@ -116,7 +117,7 @@ int resultlen;
 /*     nc_type xtype;    /\* NetCDF data type of this variable. *\/ */
 /*     int ret;          /\* Return code for function calls. *\/ */
 /*     int dimids[NDIM3]; /\* Dimension ids for this variable. *\/ */
-/*     char var_name[NC_MAX_NAME];   /\* Name of the variable. *\/ */
+/*     char var_name[PIO_MAX_NAME];   /\* Name of the variable. *\/ */
 /*     /\* size_t start[NDIM3];           /\\* Zero-based index to start read. *\\/ *\/ */
 /*     /\* size_t count[NDIM3];           /\\* Number of elements to read. *\\/ *\/ */
 /*     /\* int buffer[DIM_LEN_X];          /\\* Buffer to read in data. *\\/ *\/ */
@@ -136,7 +137,7 @@ int resultlen;
 /*         return ERR_BAD; */
 /*     for (int d = 0; d < NDIM3; d++) */
 /*     { */
-/*         char my_dim_name[NC_MAX_NAME]; */
+/*         char my_dim_name[PIO_MAX_NAME]; */
 /*         PIO_Offset dimlen;  */
         
 /*         if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */
@@ -187,114 +188,118 @@ int resultlen;
 
 /* Write, then read, a simple example with darrays.
 
-    The sample file created by this program is a small netCDF file. It
-    has the following contents (as shown by ncdump):
-
-    
      -netcdf darray_no_async_iotype_1 {
-dimensions:
-	unlimted = UNLIMITED ; // (2 currently)
-	x = 4 ;
-	y = 4 ;
-variables:
-	int foo(unlimted, x, y) ;
-data:
-
- foo =
-  42, 42, 42, 42,
-  43, 43, 43, 43,
-  44, 44, 44, 44,
-  45, 45, 45, 45,
-  142, 142, 142, 142,
-  143, 143, 143, 143,
-  144, 144, 144, 144,
-  145, 145, 145, 145 ;
-}
-    
      
+   The sample file created by this program is a small netCDF file. It
+   has the following contents (as shown by ncdump):
+
+   
      +   netcdf darray_no_async_iotype_1 {
+   dimensions:
+   unlimted = UNLIMITED ; // (2 currently)
+   x = 4 ;
+   y = 4 ;
+   variables:
+   int foo(unlimted, x, y) ;
+   data:
+
+   foo =
+   42, 42, 42, 42,
+   43, 43, 43, 43,
+   44, 44, 44, 44,
+   45, 45, 45, 45,
+   142, 142, 142, 142,
+   143, 143, 143, 143,
+   144, 144, 144, 144,
+   145, 145, 145, 145 ;
+   }
+   
      
 
 */
-    int main(int argc, char* argv[])
-    {
-	int my_rank;  /* Zero-based rank of processor. */
-	int ntasks;   /* Number of processors involved in current execution. */
-        int iosysid; /* The ID for the parallel I/O system. */
-	/* int ncid;     /\* The ncid of the netCDF file. *\/ */
-	/* int dimid[NDIM3];    /\* The dimension ID. *\/ */
-	/* int varid;    /\* The ID of the netCDF varable. *\/ */
-        /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */
-        /* int num_flavors = 0;            /\* Number of iotypes available in this build. *\/ */
-	/* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */
-	int ret;                        /* Return value. */
+int main(int argc, char* argv[])
+{
+    int my_rank;  /* Zero-based rank of processor. */
+    int ntasks;   /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    /* int ncid;     /\* The ncid of the netCDF file. *\/ */
+    /* int dimid[NDIM3];    /\* The dimension ID. *\/ */
+    /* int varid;    /\* The ID of the netCDF varable. *\/ */
+    /* char filename[PIO_MAX_NAME + 1]; /\* Test filename. *\/ */
+    /* int num_flavors = 0;            /\* Number of iotypes available in this build. *\/ */
+    /* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */
+    int ret;                        /* Return value. */
 
 #ifdef TIMING
-	/* Initialize the GPTL timing library. */
-	if ((ret = GPTLinitialize ()))
-	    return ret;
+    /* Initialize the GPTL timing library. */
+    if ((ret = GPTLinitialize ()))
+        return ret;
 #endif
         
-	/* Initialize MPI. */
-	if ((ret = MPI_Init(&argc, &argv)))
-	    MPIERR(ret);
-	if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
-	    MPIERR(ret);
-
-	/* Learn my rank and the total number of processors. */
-	if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	    MPIERR(ret);
-	if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
-	    MPIERR(ret);
-
-	/* Check that a valid number of processors was specified. */
-        printf("%d: ParallelIO Library darray_async example running on %d processors.\n",
-               my_rank, ntasks);
-	if (ntasks != TARGET_NTASKS)
-        {
-	    fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS);
-            return ERR_BAD;
-        }
-
-        /* Turn on logging. */
-        if ((ret = PIOc_set_log_level(LOG_LEVEL)))
-            return ret;
-
-        /* Num procs for computation. */
-        int num_procs2[COMPONENT_COUNT] = {4};
+    /* Initialize MPI. */
+    if ((ret = MPI_Init(&argc, &argv)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+        MPIERR(ret);
+
+    /* Learn my rank and the total number of processors. */
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+        MPIERR(ret);
+
+    /* Check that a valid number of processors was specified. */
+    printf("%d: ParallelIO Library darray_async example running on %d processors.\n",
+           my_rank, ntasks);
+    if (ntasks != TARGET_NTASKS)
+    {
+        fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS);
+        return ERR_BAD;
+    }
+
+    /* Turn on logging. */
+    if ((ret = PIOc_set_log_level(LOG_LEVEL)))
+        return ret;
+
+    /* Change error handling so we can test inval parameters. */
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Num procs for computation. */
+    int num_procs2[COMPONENT_COUNT] = {4};
         
-        /* Is the current process a computation task? */
-        int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1;
+    /* Is the current process a computation task? */
+    int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1;
 
-        /* Initialize the IO system. */
-        if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT,
-                                   num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid)))
-            ERR(ret);
+    /* Initialize the IO system. */
+    if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT,
+                               num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid)))
+        ERR(ret);
 
 
-        /* The rest of the code executes on computation tasks only. As
-         * PIO functions are called on the computation tasks, the
-         * async system will call them on the IO task. When the
-         * computation tasks call PIO_finalize(), the IO task will get
-         * a message to shut itself down. */
-        if (comp_task)
-        {
-            /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */
-            /* int ioid;     /\* The I/O description ID. *\/ */
+    /* The rest of the code executes on computation tasks only. As
+     * PIO functions are called on the computation tasks, the
+     * async system will call them on the IO task. When the
+     * computation tasks call PIO_finalize(), the IO task will get
+     * a message to shut itself down. */
+    if (comp_task)
+    {
+        /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */
+        /* int ioid;     /\* The I/O description ID. *\/ */
             
-            /* /\* How many elements on each computation task? *\/ */
-            /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */
-
-            /* /\* Allocate and initialize array of decomposition mapping. *\/ */
-            /* PIO_Offset compdof[elements_per_pe]; */
-            /* for (int i = 0; i < elements_per_pe; i++) */
-            /*     compdof[i] = my_rank * elements_per_pe + i; */
-
-            /* /\* Create the PIO decomposition for this example. Since */
-            /*    this is a variable with an unlimited dimension, we want */
-            /*    to create a 2-D composition which represents one */
-            /*    record. *\/ */
-            /* printf("rank: %d Creating decomposition...\n", my_rank); */
-            /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */
-            /*                             compdof, &ioid, 0, NULL, NULL))) */
-            /*     ERR(ret); */
+        /* /\* How many elements on each computation task? *\/ */
+        /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */
+
+        /* /\* Allocate and initialize array of decomposition mapping. *\/ */
+        /* PIO_Offset compdof[elements_per_pe]; */
+        /* for (int i = 0; i < elements_per_pe; i++) */
+        /*     compdof[i] = my_rank * elements_per_pe + i; */
+
+        /* /\* Create the PIO decomposition for this example. Since */
+        /*    this is a variable with an unlimited dimension, we want */
+        /*    to create a 2-D composition which represents one */
+        /*    record. *\/ */
+        /* printf("rank: %d Creating decomposition...\n", my_rank); */
+        /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */
+        /*                             compdof, &ioid, 0, NULL, NULL))) */
+        /*     ERR(ret); */
 
 /*         /\* The number of favors may change with the build parameters. *\/ */
 /* #ifdef _PNETCDF */
@@ -362,26 +367,26 @@ netcdf darray_no_async_iotype_1 {
 /*             /\*     ERR(ret); *\/ */
 /* 	} */
 
-            /* Free the PIO decomposition. */
-            /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */
-            /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */
-            /*     ERR(ret); */
+        /* Free the PIO decomposition. */
+        /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */
+        /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */
+        /*     ERR(ret); */
 
-            /* Finalize the IO system. Only call this from the computation tasks. */
-            printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
-            if ((ret = PIOc_finalize(iosysid)))
-                ERR(ret);
-        } /* endif comp_task */
+        /* Finalize the IO system. Only call this from the computation tasks. */
+        printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+        if ((ret = PIOc_free_iosystem(iosysid)))
+            ERR(ret);
+    } /* endif comp_task */
 
-	/* Finalize the MPI library. */
-	MPI_Finalize();
+    /* Finalize the MPI library. */
+    MPI_Finalize();
 
 #ifdef TIMING
-	/* Finalize the GPTL timing library. */
-	if ((ret = GPTLfinalize ()))
-	    return ret;
+    /* Finalize the GPTL timing library. */
+    if ((ret = GPTLfinalize ()))
+        return ret;
 #endif
 
-        printf("rank: %d SUCCESS!\n", my_rank);
-	return 0;
-    }
+    printf("rank: %d SUCCESS!\n", my_rank);
+    return 0;
+}
diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c
index 14228ab422c6..ce50003a29a0 100644
--- a/examples/c/darray_no_async.c
+++ b/examples/c/darray_no_async.c
@@ -6,9 +6,10 @@
  * (one unlimited) and one variable. It first writes, then reads the
  * sample file using distributed arrays.
  *
- * This example can be run in parallel for 4 processors.
+ * This example can be run in parallel for 16 processors.
  */
 
+#include "config.h"
 #include 
 #include 
 #include 
@@ -31,10 +32,10 @@
 #define NUM_TIMESTEPS 2
 
 /* The length of our sample data in X dimension.*/
-#define DIM_LEN_X 4
+#define DIM_LEN_X 8
 
 /* The length of our sample data in Y dimension.*/
-#define DIM_LEN_Y 4
+#define DIM_LEN_Y 8
 
 /* The name of the variable in the netCDF output file. */
 #define VAR_NAME "foo"
@@ -47,10 +48,10 @@
 #define START_DATA_VAL 42
 
 /* Number of tasks this example runs on. */
-#define TARGET_NTASKS 4
+#define TARGET_NTASKS 16
 
 /* Logging level. */
-#define LOG_LEVEL 3
+#define LOG_LEVEL -1
 
 /* Lengths of dimensions. */
 int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
@@ -58,6 +59,11 @@ int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
 /* Names of dimensions. */
 char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
 
+/* These are used when writing the decomposition file. */
+#define DECOMP_FILENAME "darray_no_async_decomp.nc"
+#define DECOMP_TITLE "Example Decomposition from darray_no_async.c"
+#define DECOMP_HISTORY "This file is created by the program darray_no_async in the PIO C library"
+
 /* Handle MPI errors. This should only be used with MPI library
  * function calls. */
 #define MPIERR(e) do {                                                  \
@@ -104,7 +110,7 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
     nc_type xtype;    /* NetCDF data type of this variable. */
     int ret;          /* Return code for function calls. */
     int dimids[NDIM3]; /* Dimension ids for this variable. */
-    char var_name[NC_MAX_NAME];   /* Name of the variable. */
+    char var_name[PIO_MAX_NAME];   /* Name of the variable. */
     /* size_t start[NDIM3];           /\* Zero-based index to start read. *\/ */
     /* size_t count[NDIM3];           /\* Number of elements to read. *\/ */
     /* int buffer[DIM_LEN_X];          /\* Buffer to read in data. *\/ */
@@ -124,9 +130,9 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
         return ERR_BAD;
     for (int d = 0; d < NDIM3; d++)
     {
-        char my_dim_name[NC_MAX_NAME];
-        PIO_Offset dimlen; 
-        
+        char my_dim_name[PIO_MAX_NAME];
+        PIO_Offset dimlen;
+
         if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen)))
             return ret;
         if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d]))
@@ -137,7 +143,7 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
     if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts)))
         return ret;
     if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 ||
-            dimids[2] != 2 || natts != 0)
+        dimids[2] != 2 || natts != 0)
         return ERR_BAD;
 
     /* Allocate storage for sample data. */
@@ -148,7 +154,7 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
     for (int t = 0; t < NUM_TIMESTEPS; t++)
     {
         int varid = 0; /* There's only one var in sample file. */
-        
+
         /* This is the data we expect for this timestep. */
         for (int i = 0; i < elements_per_pe; i++)
             buffer[i] = 100 * t + START_DATA_VAL + my_rank;
@@ -175,184 +181,202 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
 
 /* Write, then read, a simple example with darrays.
 
-    The sample file created by this program is a small netCDF file. It
-    has the following contents (as shown by ncdump):
+   The sample file created by this program is a small netCDF file. It
+   has the following contents (as shown by ncdump):
 
-    
      +   
       netcdf darray_no_async_iotype_1 {
 dimensions:
 	unlimted = UNLIMITED ; // (2 currently)
-	x = 4 ;
-	y = 4 ;
+	x = 8 ;
+	y = 8 ;
 variables:
 	int foo(unlimted, x, y) ;
 data:
 
  foo =
-  42, 42, 42, 42,
-  43, 43, 43, 43,
-  44, 44, 44, 44,
-  45, 45, 45, 45,
-  142, 142, 142, 142,
-  143, 143, 143, 143,
-  144, 144, 144, 144,
-  145, 145, 145, 145 ;
+  42, 42, 42, 42, 43, 43, 43, 43,
+  44, 44, 44, 44, 45, 45, 45, 45,
+  46, 46, 46, 46, 47, 47, 47, 47,
+  48, 48, 48, 48, 49, 49, 49, 49,
+  50, 50, 50, 50, 51, 51, 51, 51,
+  52, 52, 52, 52, 53, 53, 53, 53,
+  54, 54, 54, 54, 55, 55, 55, 55,
+  56, 56, 56, 56, 57, 57, 57, 57,
+  142, 142, 142, 142, 143, 143, 143, 143,
+  144, 144, 144, 144, 145, 145, 145, 145,
+  146, 146, 146, 146, 147, 147, 147, 147,
+  148, 148, 148, 148, 149, 149, 149, 149,
+  150, 150, 150, 150, 151, 151, 151, 151,
+  152, 152, 152, 152, 153, 153, 153, 153,
+  154, 154, 154, 154, 155, 155, 155, 155,
+  156, 156, 156, 156, 157, 157, 157, 157 ;
 }
-    
      
+   
      
 
 */
-    int main(int argc, char* argv[])
-    {
-	int my_rank;  /* Zero-based rank of processor. */
-	int ntasks;   /* Number of processors involved in current execution. */
-	int ioproc_stride = 1;	    /* Stride in the mpi rank between io tasks. */
-	int ioproc_start = 0; 	    /* Rank of first task to be used for I/O. */
-        PIO_Offset elements_per_pe; /* Array elements per processing unit. */
-	int iosysid;  /* The ID for the parallel I/O system. */	
-	int ncid;     /* The ncid of the netCDF file. */
-	int dimid[NDIM3];    /* The dimension ID. */
-	int varid;    /* The ID of the netCDF varable. */
-	int ioid;     /* The I/O description ID. */
-        char filename[NC_MAX_NAME + 1]; /* Test filename. */
-        int num_flavors = 0;            /* Number of iotypes available in this build. */
-	int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */
-	int ret;                        /* Return value. */
+int main(int argc, char* argv[])
+{
+    int my_rank;  /* Zero-based rank of processor. */
+    int ntasks;   /* Number of processors involved in current execution. */
+    int ioproc_stride = 1;	    /* Stride in the mpi rank between io tasks. */
+    int ioproc_start = 0; 	    /* Rank of first task to be used for I/O. */
+    PIO_Offset elements_per_pe; /* Array elements per processing unit. */
+    int iosysid;  /* The ID for the parallel I/O system. */
+    int ncid;     /* The ncid of the netCDF file. */
+    int dimid[NDIM3];    /* The dimension ID. */
+    int varid;    /* The ID of the netCDF varable. */
+    int ioid;     /* The I/O description ID. */
+    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+    int num_flavors = 0;            /* Number of iotypes available in this build. */
+    int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */
+    int ret;                        /* Return value. */
 
 #ifdef TIMING
-	/* Initialize the GPTL timing library. */
-	if ((ret = GPTLinitialize ()))
-	    return ret;
+    /* Initialize the GPTL timing library. */
+    if ((ret = GPTLinitialize ()))
+        return ret;
 #endif
 
-	/* Initialize MPI. */
-	if ((ret = MPI_Init(&argc, &argv)))
-	    MPIERR(ret);
-	if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
-	    MPIERR(ret);
-
-	/* Learn my rank and the total number of processors. */
-	if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	    MPIERR(ret);
-	if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
-	    MPIERR(ret);
-
-	/* Check that a valid number of processors was specified. */
-	if (ntasks != TARGET_NTASKS)
-	    fprintf(stderr, "Number of processors must be 4!\n");
-        printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
-               my_rank, ntasks);
-
-        /* Turn on logging. */
-        if ((ret = PIOc_set_log_level(LOG_LEVEL)))
-            return ret;
-        
-	/* Initialize the PIO IO system. This specifies how many and
-	 * which processors are involved in I/O. */
-	if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride,
-				       ioproc_start, PIO_REARR_BOX, &iosysid)))
-	    ERR(ret);
-
-	/* Describe the decomposition. */
-	elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS;
-
-        /* Allocate and initialize array of decomposition mapping. */
-	PIO_Offset compdof[elements_per_pe];
-	for (int i = 0; i < elements_per_pe; i++)
-	    compdof[i] = my_rank * elements_per_pe + i;
-
-	/* Create the PIO decomposition for this example. Since this
-         * is a variable with an unlimited dimension, we want to
-         * create a 2-D composition which represents one record. */
-        printf("rank: %d Creating decomposition...\n", my_rank);
-	if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
-				   compdof, &ioid, 0, NULL, NULL)))
-	    ERR(ret);
-
-        /* The number of favors may change with the build parameters. */
+    /* Initialize MPI. */
+    if ((ret = MPI_Init(&argc, &argv)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+        MPIERR(ret);
+
+    /* Learn my rank and the total number of processors. */
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+        MPIERR(ret);
+
+    /* Check that a valid number of processors was specified. */
+    if (ntasks != TARGET_NTASKS)
+        fprintf(stderr, "Number of processors must be 16!\n");
+    printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
+           my_rank, ntasks);
+
+    /* Turn on logging. */
+    if ((ret = PIOc_set_log_level(LOG_LEVEL)))
+        return ret;
+
+    /* /\* Change error handling so we can test inval parameters. *\/ */
+    /* if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) */
+    /*     return ret; */
+
+    /* Initialize the PIO IO system. This specifies how many and
+     * which processors are involved in I/O. */
+    if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 4, ioproc_stride,
+                                   ioproc_start, PIO_REARR_BOX, &iosysid)))
+        ERR(ret);
+
+    /* Describe the decomposition. */
+    elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS;
+
+    /* Allocate and initialize array of decomposition mapping. */
+    PIO_Offset compdof[elements_per_pe];
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this example. Since this
+     * is a variable with an unlimited dimension, we want to
+     * create a 2-D composition which represents one record. */
+    printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank,
+           elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
+                                compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL)))
+        ERR(ret);
+
+    /* Write the decomposition file. */
+    if ((ret = PIOc_write_nc_decomp(iosysid, DECOMP_FILENAME, NC_CLOBBER,
+                                    ioid, DECOMP_TITLE, DECOMP_HISTORY, 0)))
+        ERR(ret);
+
+    /* The number of favors may change with the build parameters. */
 #ifdef _PNETCDF
-        format[num_flavors++] = PIO_IOTYPE_PNETCDF;
+    format[num_flavors++] = PIO_IOTYPE_PNETCDF;
 #endif
-        format[num_flavors++] = PIO_IOTYPE_NETCDF;
+    format[num_flavors++] = PIO_IOTYPE_NETCDF;
 #ifdef _NETCDF4
-        format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
-        format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
+    format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
+    format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
 #endif
 
-	/* Use PIO to create the example file in each of the four
-	 * available ways. */
-	for (int fmt = 0; fmt < num_flavors; fmt++)
-	{
-            /* Create a filename. */
-            sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
-
-	    /* Create the netCDF output file. */
-            printf("rank: %d Creating sample file %s with format %d...\n",
-                   my_rank, filename, format[fmt]);
-	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
-		ERR(ret);
-
-	    /* Define netCDF dimension and variable. */
-            printf("rank: %d Defining netCDF metadata...\n", my_rank);
-            for (int d = 0; d < NDIM3; d++)
-                if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
-                    ERR(ret);
-	    if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid)))
-	        ERR(ret);
-	    if ((ret = PIOc_enddef(ncid)))
-	        ERR(ret);
-
-	    /* Allocate storage for sample data. */
-            int buffer[elements_per_pe];
-
-            /* Write each timestep. */
-            for (int t = 0; t < NUM_TIMESTEPS; t++)
-            {
-                /* Create some data for this timestep. */
-                for (int i = 0; i < elements_per_pe; i++)
-                    buffer[i] = 100 * t + START_DATA_VAL + my_rank;
-                
-                /* Write data to the file. */
-                printf("rank: %d Writing sample data...\n", my_rank);
-                if ((ret = PIOc_setframe(ncid, varid, t)))
-                    ERR(ret);
-                if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
-                    ERR(ret);
-            }
-
-            /* THis will cause all data to be written to disk. */
-            if ((ret = PIOc_sync(ncid)))
-	        ERR(ret);
-
-	    /* Close the netCDF file. */
-            printf("rank: %d Closing the sample data file...\n", my_rank);
-	    if ((ret = PIOc_closefile(ncid)))
-		ERR(ret);
-
-            /* Check the output file. */
-            /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */
-            /*                       my_rank, ioid))) */
-            /*     ERR(ret); */
-	}
-
-	/* Free the PIO decomposition. */
-        printf("rank: %d Freeing PIO decomposition...\n", my_rank);
-	if ((ret = PIOc_freedecomp(iosysid, ioid)))
-	    ERR(ret);
-
-	/* Finalize the IO system. */
-        printf("rank: %d Freeing PIO resources...\n", my_rank);
-	if ((ret = PIOc_finalize(iosysid)))
-	    ERR(ret);
-
-	/* Finalize the MPI library. */
-	MPI_Finalize();
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create a filename. */
+        sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n",
+               my_rank, filename, format[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimension and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM3; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
+                ERR(ret);
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid)))
+            ERR(ret);
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Allocate storage for sample data. */
+        int buffer[elements_per_pe];
+
+        /* Write each timestep. */
+        for (int t = 0; t < NUM_TIMESTEPS; t++)
+        {
+            /* Create some data for this timestep. */
+            for (int i = 0; i < elements_per_pe; i++)
+                buffer[i] = 100 * t + START_DATA_VAL + my_rank;
+
+            /* Write data to the file. */
+            printf("rank: %d Writing sample data...\n", my_rank);
+            if ((ret = PIOc_setframe(ncid, varid, t)))
+                ERR(ret);
+            if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
+                ERR(ret);
+        }
+
+        /* THis will cause all data to be written to disk. */
+        if ((ret = PIOc_sync(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check the output file. */
+        /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */
+        /*                       my_rank, ioid))) */
+        /*     ERR(ret); */
+    }
+
+    /* Free the PIO decomposition. */
+    printf("rank: %d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    /* Finalize the IO system. */
+    printf("rank: %d Freeing PIO resources...\n", my_rank);
+    if ((ret = PIOc_free_iosystem(iosysid)))
+        ERR(ret);
+
+    /* Finalize the MPI library. */
+    MPI_Finalize();
 
 #ifdef TIMING
-	/* Finalize the GPTL timing library. */
-	if ((ret = GPTLfinalize ()))
-	    return ret;
+    /* Finalize the GPTL timing library. */
+    if ((ret = GPTLfinalize ()))
+        return ret;
 #endif
 
-        printf("rank: %d SUCCESS!\n", my_rank);
-	return 0;
-    }
+    printf("rank: %d SUCCESS!\n", my_rank);
+    return 0;
+}
diff --git a/examples/c/example1.c b/examples/c/example1.c
index ad650f0b1faf..3821a9ab6f5c 100644
--- a/examples/c/example1.c
+++ b/examples/c/example1.c
@@ -10,6 +10,7 @@
  * processors.
  */
 
+#include "config.h"
 #include 
 #include 
 #include 
@@ -303,8 +304,15 @@ int check_file(int ntasks, char *filename) {
 		   my_rank, ntasks);
 
 	/* keep things simple - 1 iotask per MPI process */    
-	niotasks = ntasks; 
+	niotasks = ntasks;
 
+        /* Turn on logging if available. */
+        /* PIOc_set_log_level(4); */
+
+        /* Change error handling to return errors. */
+        if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+            return ret;
+        
 	/* Initialize the PIO IO system. This specifies how
 	 * many and which processors are involved in I/O. */
 	if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride,
diff --git a/examples/c/example2.c b/examples/c/example2.c
index becbcaccd292..fc05e61aa8b7 100644
--- a/examples/c/example2.c
+++ b/examples/c/example2.c
@@ -92,7 +92,7 @@ char err_buffer[MPI_MAX_ERROR_STRING];
 int resultlen;
 
 /** The dimension names. */
-char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
 
 /** Length of the dimensions in the sample data. */
 int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
@@ -228,8 +228,8 @@ int check_file(int ntasks, char *filename) {
     nc_type xtype;    /**< NetCDF data type of this variable. */
     int ret;          /**< Return code for function calls. */
     int dimids[NDIM]; /**< Dimension ids for this variable. */
-    char my_dim_name[NC_MAX_NAME + 1]; /**< Name of the dimension. */
-    char var_name[NC_MAX_NAME + 1];    /**< Name of the variable. */
+    char my_dim_name[PIO_MAX_NAME + 1]; /**< Name of the dimension. */
+    char var_name[PIO_MAX_NAME + 1];    /**< Name of the variable. */
     size_t start[NDIM];                /**< Zero-based index to start read. */
     size_t count[NDIM];                /**< Number of elements to read. */
     int buffer[X_DIM_LEN];             /**< Buffer to read in data. */
@@ -359,7 +359,7 @@ int main(int argc, char* argv[])
      * (serial4 and parallel4) will be in netCDF-4/HDF5
      * format. All four can be read by the netCDF library, and all
      * will contain the same contents. */
-    char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc",
+    char filename[NUM_NETCDF_FLAVORS][PIO_MAX_NAME + 1] = {"example2_pnetcdf.nc",
 							  "example2_classic.nc",
 							  "example2_serial4.nc",
 							  "example2_parallel4.nc"};
diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c
index 297f3923ce5c..30a7945fe966 100644
--- a/examples/c/examplePio.c
+++ b/examples/c/examplePio.c
@@ -10,6 +10,7 @@
  * processors.
  */
 
+#include "config.h"
 #include 
 #include 
 #include 
@@ -441,7 +442,7 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this )
 
     This function frees the memory used in this example. It calls the
     ParallelIO library function PIOc_freedecomp() to free
-    decomposition resources. Then calles PIOc_finalize() and
+    decomposition resources. Then calles PIOc_free_iosystem() and
     MPI_finalize() to free library resources.
     
     @param [in] this  Pointer to self.
@@ -456,7 +457,7 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
     free(this->compdof);
     
     PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
-    PIOc_finalize(this->pioIoSystem);
+    PIOc_free_iosystem(this->pioIoSystem);
     MPI_Finalize();
     
     return this;
@@ -563,6 +564,7 @@ int main(int argc, char* argv[])
 {
     /* Parse command line. */
     int c, verbose = 0;
+    int ret;
     while ((c = getopt(argc, argv, "v")) != -1)
 	switch (c)
 	{
@@ -573,11 +575,15 @@ int main(int argc, char* argv[])
             break;
 	}
 
+
+    /* Change error handling so we can test inval parameters. */
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
     struct examplePioClass* pioExInst = epc_new(verbose);
     
 #ifdef TIMING    
     /* Initialize the GPTL timing library. */
-    int ret;
     if ((ret = GPTLinitialize ()))
       return ret;
 #endif    
diff --git a/examples/c/run_tests.sh b/examples/c/run_tests.sh
new file mode 100755
index 000000000000..0d369bbb53dd
--- /dev/null
+++ b/examples/c/run_tests.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+# This is a test script for PIO examples.
+# Ed Hartnett 5/7/18
+
+# Stop execution of script if error is returned.
+set -e
+
+# Stop loop if ctrl-c is pressed.
+trap exit INT TERM
+
+printf 'running PIO examples...\n'
+
+#PIO_EXAMPLES='examplePio'
+PIO_EXAMPLES='example1 examplePio'
+PIO_EXAMPLES_16='darray_no_async'
+
+success1=true
+for EXAMPLE in $PIO_EXAMPLES
+do
+    success1=false
+    echo "running ${EXAMPLE}"
+    mpiexec -n 4 ./${EXAMPLE} && success1=true
+    if test $success1 = false; then
+        break
+    fi
+done
+success2=true
+for EXAMPLE in $PIO_EXAMPLES_16
+do
+    success2=false
+    echo "running ${EXAMPLE}"
+    mpiexec -n 16 ./${EXAMPLE} && success2=true
+    if test $success2 = false; then
+        break
+    fi
+done
+
+# Did we succeed?
+if test x$success1 = xtrue -a  x$success2 = xtrue; then
+    exit 0
+fi
+exit 1
diff --git a/scripts/Makefile.am b/scripts/Makefile.am
new file mode 100644
index 000000000000..9e65f3829025
--- /dev/null
+++ b/scripts/Makefile.am
@@ -0,0 +1,6 @@
+# This is part of PIO. It handles the scripts directory which has some
+# useful scripts.
+
+# Ed Hartnett 5/17/19
+
+EXTRA_DIST = prune_decomps.pl genf90.pl
diff --git a/scripts/genf90.pl b/scripts/genf90.pl
new file mode 100755
index 000000000000..ec7d0dba28b9
--- /dev/null
+++ b/scripts/genf90.pl
@@ -0,0 +1,387 @@
+#!/usr/bin/env perl
+use strict;
+my $outfile;
+# Beginning with F90, Fortran has strict typing of variables based on "TKR"
+# (type, kind, and rank). In many cases we want to write subroutines that
+# provide the same functionality for different variable types and ranks. In
+# order to do this without cut-and-paste duplication of code, we create a
+# template file with the extension ".F90.in", which can be parsed by this script
+# to generate F90 code for all of the desired specific types.
+#
+# Keywords are delimited by curly brackets: {}
+# 
+# {TYPE} and {DIMS} are used to generate the specific subroutine names from the 
+#            generic template
+# {TYPE} : Variable type name; implemented types are character, 4 or 8 byte real,
+#          and 4 or 8 byte integer.
+#                allowed values: text, real, double, int, long, logical
+#                default values:  text, real, double, int
+# {VTYPE} : Used to generate variable declarations to match the specific type.
+#                if {TYPE}=double then {VTYPE} is "real(r8)"
+# {ITYPE}, {ITYPENAME} : Used to generate CPP statements for the specific type.
+# {MPITYPE} : Used to generate MPI types corresponding to the specific type.
+# 
+# {DIMS} : Rank of arrays, "0" for scalar.
+#                allowed values: 0-7
+#                default values : 0-5
+# {DIMSTR} : Generates the parenthesis and colons used for a variable
+#            declaration of {DIMS} dimensions.
+#                if {DIMS}=3 then {DIMSTR} is (:,:,:)
+# {REPEAT} : Repeats an expression for each number from 1 to {DIMS}, with each
+#            iteration separated by commas.
+#                {REPEAT: foo(#, bar)}
+#                expands to this:
+#                foo(1, bar), foo(2, bar), foo(3, bar), ...
+
+# defaults 
+my @types = qw(text real double int);
+my $vtype = {'text' => 'character(len=*)', 
+	     'real' => 'real(r4)', 
+	     'double' => 'real(r8)',
+	     'int'    => 'integer(i4)',
+	     'long'   => 'integer(i8)',
+             'logical' => 'logical' };
+my $itype = {'text' => 100, 
+	     'real' => 101, 
+	     'double' => 102,
+	     'int'    => 103,
+	     'long'   => 104,
+             'logical' => 105};
+my $itypename = {'text' => 'TYPETEXT', 
+	     'real' =>  'TYPEREAL', 
+	     'double' => 'TYPEDOUBLE',
+	     'int'    => 'TYPEINT',
+	     'long'   =>  'TYPELONG',
+             'logical' => 'TYPELOGICAL'};
+my $mpitype = {'text' => 'MPI_CHARACTER',
+	       'real' => 'MPI_REAL4',
+	       'double' => 'MPI_REAL8',
+	       'int' => 'MPI_INTEGER'};
+# Netcdf C datatypes
+my $nctype = {'text' => 'text',
+	      'real' => 'float',
+	      'double' => 'double',
+	      'int' => 'int'};
+# C interoperability types
+my $ctype = {'text' => 'character(C_CHAR)',
+	     'real' => 'real(C_FLOAT)',
+	     'double' => 'real(C_DOUBLE)',
+	     'int' => 'integer(C_INT)'};
+
+
+
+my @dims =(0..5);
+
+my $write_dtypes = "no";
+# begin
+
+foreach(@ARGV){
+    my $infile = $_;
+    usage() unless($infile =~ /(.*.F90).in/);
+    $outfile = $1;
+    open(F,"$infile") || die "$0 Could not open $infile to read";
+    my @parsetext;
+    my $cnt=0;
+    foreach(){
+	$cnt++;
+	if(/^\s*contains/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^\s*interface/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^[^!]*subroutine/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^[^!]*function/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+
+	push(@parsetext,$_);
+    }
+
+    close(F);
+
+    my $end;
+    my $contains=0;
+    my $in_type_block=0;
+    my @unit;
+    my $unitcnt=0;
+    my $date = localtime();
+    my $preamble = 
+"!===================================================
+! DO NOT EDIT THIS FILE, it was generated using $0 
+! Any changes you make to this file may be lost
+!===================================================\n";
+    my @output ;
+    push(@output,$preamble);
+
+    my $line;
+    my $dimmodifier;
+    my $typemodifier;
+    my $itypeflag;
+    my $block;
+    my $block_type;
+    my $cppunit;
+    foreach $line (@parsetext){
+# skip parser comments
+	next if($line =~ /\s*!pl/);
+
+	$itypeflag=1 if($line =~ /{ITYPE}/);
+	$itypeflag=1 if($line =~ /TYPETEXT/);
+	$itypeflag=1 if($line =~ /TYPEREAL/);
+	$itypeflag=1 if($line =~ /TYPEDOUBLE/);
+	$itypeflag=1 if($line =~ /TYPEINT/);
+	$itypeflag=1 if($line =~ /TYPELONG/);
+
+	
+        if($contains==0){
+	    if($line=~/\s*!\s*DIMS\s+[\d,]+!*/){
+		$dimmodifier=$line;
+		next;
+	    }
+	    if($line=~/\s*!\s*TYPE\s+[^!]+!*$/){
+		$typemodifier=$line;
+		next;
+	    }
+            if ((defined $typemodifier or defined $dimmodifier)
+                and not defined $block and $line=~/^\s*#[^{]*$/) {
+                push(@output, $line);
+                next;
+            }
+            # Figure out the bounds of a type statement.
+            # Type blocks start with "type," "type foo" or "type::" but not
+            # "type(".
+            $in_type_block=1 if($line=~/^\s*type\s*[,:[:alpha:]]/i);
+            $in_type_block=0 if($line=~/^\s*end\s*type/i);
+	    if(not defined $block) {
+                if ($line=~/^\s*type[^[:alnum:]_].*(\{TYPE\}|\{DIMS\})/i or
+                    $line=~/^[^!]*(function|subroutine).*(\{TYPE\}|\{DIMS\})/i) {
+                    $block=$line;
+                    next;
+                }
+                if ($line=~/^\s*interface.*(\{TYPE\}|\{DIMS\})/i) {
+                    $block_type="interface";
+                    $block=$line;
+                    next;
+                }
+	    }
+	    if(not defined $block_type and
+               ($line=~/^\s*end\s+type\s+.*(\{TYPE\}|\{DIMS\})/i or
+                $line=~/^\s*end\s+(function|subroutine)\s+.*(\{TYPE\}|\{DIMS\})/i)){
+
+		$line = $block.$line;
+		undef $block;
+	    }
+            if ($line=~/^\s*end\s*interface/i and
+                defined $block) {
+                $line = $block.$line;
+		undef $block;
+		undef $block_type;
+            }
+	    if(defined $block){
+		$block = $block.$line;
+		next;
+	    }
+	    if(defined $dimmodifier){
+		$line = $dimmodifier.$line;
+		undef $dimmodifier;
+	    } 
+	    if(defined $typemodifier){
+		$line = $typemodifier.$line;
+		undef $typemodifier;
+	    } 
+	    
+	    push(@output, buildout($line));
+            if(($line =~ /^\s*contains\s*!*/i && ! $in_type_block) or
+               ($line =~ /^\s*!\s*Not a module/i)){
+                $contains=1;
+                next;
+            }
+	}
+	if($line=~/^\s*end module\s*/){
+	    $end = $line;
+	    last;
+	}
+
+	if($contains==1){
+	    # first parse into functions or subroutines
+            if($cppunit || !(defined($unit[$unitcnt]))){
+		# Make cpp lines and blanks between routines units.
+		if($line =~ /^\s*\#(?!\s[[:digit:]]+)/ || $line =~/^\s*$/ || $line=~/^\s*!(?!\s*(TYPE|DIMS))/){
+		    push(@{$unit[$unitcnt]},$line);
+		    $cppunit=1;
+		    next;
+		} else {
+                    $cppunit=0;
+                    $unitcnt++;
+                }
+	    }
+
+	       
+	    push(@{$unit[$unitcnt]},$line);
+	    if ($line=~/^\s*interface/i) {
+		$block_type="interface";
+		$block=$line;
+	    }
+	    if ($line=~/^\s*end\s*interface/i) {
+		undef $block_type;
+		undef $block;
+	    }
+	    unless(defined $block){
+		if($line =~ /\s*end function/i or $line =~ /\s*end subroutine/i){
+		    $unitcnt++;
+		}
+	    }
+	}
+    }
+    my $i;
+
+
+    for($i=0;$i<$unitcnt;$i++){
+	if(defined($unit[$i])){
+		my $func = join('',@{$unit[$i]});
+		push(@output, buildout($func));
+	}	
+    }
+    push(@output,@{$unit[$#unit]}) if($unitcnt==$#unit);
+    push(@output, $end);	
+    if($itypeflag==1){
+	my $str;
+	$str.="#include \"dtypes.h\"\n";
+	$write_dtypes = "yes";
+	print $str;
+    }
+    print @output;
+    writedtypes() if(!(-e "dtypes.h") && $write_dtypes == "yes");
+
+
+}
+
+
+sub usage{
+    die("$0 Expected input filename of the form .*.F90.in");
+}
+
+sub build_repeatstr{
+    my($dims) = @_;
+   # Create regex to repeat expression DIMS times.
+    my $repeatstr;
+    for(my $i=1;$i<=$dims;$i++){
+	$repeatstr .="\$\{1\}$i\$\{2\},&\n";
+    }
+    if(defined $repeatstr){
+	$repeatstr="\"$repeatstr";
+	chop $repeatstr;
+	chop $repeatstr;
+	chop $repeatstr;
+	$repeatstr.="\"";
+    }else{
+	$repeatstr='';
+    }
+}
+
+sub writedtypes{
+    open(F,">dtypes.h");
+    print F 
+"#define TYPETEXT 100
+#define TYPEREAL 101	
+#define TYPEDOUBLE 102
+#define TYPEINT 103
+#define TYPELONG 104
+#define TYPELOGICAL 105
+";
+    close(F);
+}
+
+sub buildout{
+    my ($func) = @_;
+    
+    my $outstr;
+    my(@ldims, @ltypes);
+
+    if($func=~/\s*!\s*DIMS\s+([\d,]+)\s*/){
+	@ldims = split(/,/,$1);
+    }else{
+	@ldims = @dims;
+    }
+    if($func=~/\s*!\s*TYPE\s+([^!\s]+)\s*/){
+	@ltypes = split(/,/,$1);	
+#	print ">$func<>@ltypes<\n";
+    }else{
+	@ltypes = @types;
+    }
+    
+
+    if(($func =~ /{TYPE}/ && $func =~ /{DIMS}/) ){
+	my ($type, $dims);
+	foreach $type (@ltypes){
+	    foreach $dims (@ldims){
+		my $dimstr;
+		for(my $i=1;$i<=$dims;$i++){
+		    $dimstr .=':,';
+		}
+		if(defined $dimstr){
+		    $dimstr="($dimstr";
+		    chop $dimstr;
+		    $dimstr.=')';
+		}else{
+		    $dimstr='';
+		}
+
+		my $repeatstr = build_repeatstr($dims);
+		
+		my $str = $func;
+		$str =~ s/{TYPE}/$type/g;
+		$str =~ s/{VTYPE}/$vtype->{$type}/g;
+		$str =~ s/{ITYPE}/$itype->{$type}/g;
+		$str =~ s/{MPITYPE}/$mpitype->{$type}/g;
+		$str =~ s/{NCTYPE}/$nctype->{$type}/g;
+		$str =~ s/{CTYPE}/$ctype->{$type}/g;
+		$str =~ s/{DIMS}/$dims/g;
+		$str =~ s/{DIMSTR}/$dimstr/g;
+                $str =~ s/{REPEAT:([^#}]*)#([^#}]*)}/$repeatstr/eeg;
+		$outstr .= $str;
+	    }
+	}
+    }elsif($func =~ /{DIMS}/){
+        my $dims;
+	foreach $dims (@ldims){
+	    my $dimstr;
+	    for(my $i=1;$i<=$dims;$i++){
+		$dimstr .=':,';
+	    }
+	    if(defined $dimstr){
+		$dimstr="($dimstr";
+		chop $dimstr;
+		$dimstr.=')';
+	    }else{
+		$dimstr='';
+	    }
+		
+	    my $repeatstr = build_repeatstr($dims);
+		
+	    my $str = $func;
+	    $str =~ s/{DIMS}/$dims/g;
+	    $str =~ s/{DIMSTR}/$dimstr/g;
+            $str =~ s/{REPEAT:([^#}]*)#([^#}]*)}/$repeatstr/eeg;
+	    $outstr .= $str;
+	}
+    }elsif($func =~ /{TYPE}/){
+	my ($type);
+	foreach $type (@ltypes){
+	    my $str = $func;
+	    $str =~ s/{TYPE}/$type/g;
+	    $str =~ s/{VTYPE}/$vtype->{$type}/g;
+	    $str =~ s/{ITYPE}/$itype->{$type}/g;
+	    $str =~ s/{MPITYPE}/$mpitype->{$type}/g;
+	    $str =~ s/{NCTYPE}/$nctype->{$type}/g;
+	    $str =~ s/{CTYPE}/$ctype->{$type}/g;
+	    $outstr.=$str;
+	}
+    }else{
+	$outstr=$func;
+    }
+
+    return $outstr;
+}
diff --git a/set_flags.am b/set_flags.am
new file mode 100644
index 000000000000..438deee610d8
--- /dev/null
+++ b/set_flags.am
@@ -0,0 +1,18 @@
+# This is part of the PIO package.
+#
+# Assemble the CPPFLAGS and for PIO Fortran tests.
+#
+# Ed Hartnett 3/26/19
+
+# Set the CPPFLAGS.
+AM_CPPFLAGS = -I$(top_srcdir)/src/flib -D_NETCDF
+
+# Is the user building with pnetcdf?
+if BUILD_PNETCDF
+AM_CPPFLAGS += -D_PETCDF
+endif
+
+# Is the user building with netCDF-4 parallel I/O?
+if BUILD_NETCDF4
+AM_CPPFLAGS += -D_NETCDF4
+endif
diff --git a/src/Makefile.am b/src/Makefile.am
index f4f0e71e6fb6..7cd8d61f77b9 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1 +1,16 @@
-SUBDIRS = clib
+# This is part of PIO. It creates the Makefile for the src directory.
+
+# Ed Hartnett
+
+# Does the user want to build fortran?
+if BUILD_FORTRAN
+FLIB = flib
+endif
+
+if USE_GPTL
+GPTL = gptl
+endif
+
+SUBDIRS = clib ${GPTL} $(FLIB)
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt
index 1a0d7e17773d..83635d9f6855 100644
--- a/src/clib/CMakeLists.txt
+++ b/src/clib/CMakeLists.txt
@@ -8,12 +8,12 @@ project (PIOC C)
 
 add_library (pioc topology.c pio_file.c pioc_support.c pio_lists.c
   pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c
-  pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c pio_varm.c
-  pio_darray.c pio_darray_int.c)
+  pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c
+  pio_darray.c pio_darray_int.c pio_get_vard.c pio_put_vard.c)
 
 # set up include-directories
 include_directories(
-  "${CMAKE_BINARY_DIR}"   
+  "${CMAKE_BINARY_DIR}"
   "${PROJECT_SOURCE_DIR}"   # to find foo/foo.h
   "${PROJECT_BINARY_DIR}")  # to find foo/config.h
 
@@ -36,7 +36,7 @@ elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
     PRIVATE -c99)
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
   target_compile_options (pioc
-    PRIVATE -std=c99)
+    PRIVATE -std=c99 -debug minimal)
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
   target_compile_options (pioc
     PRIVATE -std=c99)
diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am
index 756a004c35df..eba8240a83e4 100644
--- a/src/clib/Makefile.am
+++ b/src/clib/Makefile.am
@@ -4,11 +4,19 @@
 # The library we are building.
 lib_LTLIBRARIES = libpio.la
 
+# These linker flags specify libtool version info.
+# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
+# for information regarding incrementing `-version-info`.
+libpio_la_LDFLAGS = -version-info 2:0:1
+
 # The header file.
 include_HEADERS = pio.h
 
-# THe soure files.
-libpio_la_SOURCES = bget.c pioc_sc.c pio_darray.c pio_file.c \
-pio_getput_int.c pio_msg.c pio_nc.c pio_rearrange.c pio_varm.c \
-pioc.c pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c \
-pio_nc4.c pio_put_nc.c pio_spmd.c pio_internal.h bget.h
+# The library soure files.
+libpio_la_SOURCES = bget.c pioc_sc.c pio_darray.c pio_file.c		\
+pio_getput_int.c pio_msg.c pio_nc.c pio_rearrange.c pioc.c		\
+pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c	\
+pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_internal.h	\
+bget.h uthash.h
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/src/clib/pio.h b/src/clib/pio.h
index 02a51a56de5c..69e2a7fde81b 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -4,7 +4,7 @@
  * @author Jim Edwards
  * @date  2014
  *
- * @see http://code.google.com/p/parallelio/
+ * @see https://github.com/NCAR/ParallelIO
  */
 
 #ifndef _PIO_H_
@@ -14,6 +14,7 @@
 #include 
 #include  /* memcpy */
 #include 
+#include 
 
 #ifdef _NETCDF
 #include 
@@ -33,8 +34,14 @@
 #endif
 
 /** PIO_OFFSET is an integer type of size sufficient to represent the
- * size (in bytes) of the largest file supported by MPI. */
+ * size (in bytes) of the largest file supported by MPI. This is not
+ * actually used by the code. */
 #define PIO_OFFSET MPI_OFFSET
+
+/** PIO_OFFSET is defined as MPI_Offset, which is defined in
+ * pio_internal.h as long long. This is what is used throughout the C
+ * code. */
+
 #define PIO_Offset MPI_Offset
 
 /** The maximum number of variables allowed in a netCDF file. */
@@ -49,66 +56,68 @@
 
 /** Used in the decomposition netCDF file. */
 
-/* Holds the version of the decomposition file. */
+/** Holds the version of the decomposition file. */
 #define DECOMP_VERSION_ATT_NAME "PIO_library_version"
 
-/* Holds the maximum length of any task map. */
+/** Holds the maximum length of any task map. */
 #define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
 
-/* Name of title attribute. */
+/** Name of title attribute in decomposition file. */
 #define DECOMP_TITLE_ATT_NAME "title"
 
-/* Name of history attribute. */
+/** Name of history attribute in decomposition file. */
 #define DECOMP_HISTORY_ATT_NAME "history"
 
-/* Name of source attribute. */
+/** Name of source attribute in decomposition file. */
 #define DECOMP_SOURCE_ATT_NAME "source"
 
-/* Name of array order (C or Fortran) attribute. */
+/** Name of array order (C or Fortran) attribute in decomposition
+ * file. */
 #define DECOMP_ORDER_ATT_NAME "array_order"
 
-/* Name of backtrace attribute. */
+/** Name of backtrace attribute in decomposition file. */
+
 #define DECOMP_BACKTRACE_ATT_NAME "backtrace"
 
-/* Name for the dim dim in decomp file. */
+/** Name for the dim dim in decomp file. */
 #define DECOMP_DIM_DIM "dims"
 
-/* Name for the npes dim in decomp file. */
+/** Name for the npes dim in decomp file. */
 #define DECOMP_TASK_DIM_NAME "task"
 
-/* Name for the npes dim in decomp file. */
+/** Name for the npes dim in decomp file. */
 #define DECOMP_MAPELEM_DIM_NAME "map_element"
 
+/** Name for the number of dimensions dim in decomp file. */
 #define DECOMP_NDIMS "ndims"
 
-/* Name of var in decomp file that holds global array sizes. */
+/** Name of var in decomp file that holds global array sizes. */
 #define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
 
-/* Name of var in decomp file that holds the length of the map for
+/** Name of var in decomp file that holds the length of the map for
  * each task. */
 #define DECOMP_MAPLEN_VAR_NAME "maplen"
 
-/* Name of var in decomp file that holds map. */
+/** Name of var in decomp file that holds map. */
 #define DECOMP_MAP_VAR_NAME "map"
 
-/* String used to indicate a decomposition file is in C
+/** String used to indicate a decomposition file is in C
  * array-order. */
 #define DECOMP_C_ORDER_STR "C"
 
-/* String used to indicate a decomposition file is in Fortran
+/** String used to indicate a decomposition file is in Fortran
  * array-order. */
 #define DECOMP_FORTRAN_ORDER_STR "Fortran"
 
-
 /**
  * Variable description structure.
  */
 typedef struct var_desc_t
 {
-    /* Variable ID. */
+    /** Variable ID. */
     int varid;
 
-    /* Non-zero if this is a record var (i.e. uses unlimited
+    /** Non-zero if this is a record var (i.e. uses unlimited
      * dimension). */
     int rec_var;
 
@@ -119,10 +128,10 @@ typedef struct var_desc_t
     /** ID of each outstanding pnetcdf request for this variable. */
     int *request;
 
-    /** Number of requests bending with pnetcdf. */
+    /** Number of requests pending with pnetcdf. */
     int nreqs;
 
-    /* Holds the fill value of this var. */
+    /** Holds the fill value of this var. */
     void *fillvalue;
 
     /** Non-zero if fill mode is turned on for this var. */
@@ -144,8 +153,9 @@ typedef struct var_desc_t
     /** The size in bytes of a datum of MPI type mpitype. */
     int mpi_type_size;
 
-    /** Pointer to next var in list. */
-    struct var_desc_t *next;
+    /** Hash table entry. */
+    UT_hash_handle hh;
+
 } var_desc_t;
 
 /**
@@ -208,7 +218,7 @@ enum PIO_REARR_COMM_FC_DIR
     PIO_REARR_COMM_FC_2D_DISABLE
 };
 
-/* Constant to indicate unlimited requests. */
+/** Constant to indicate unlimited requests for the rearranger. */
 #define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
 
 /**
@@ -267,6 +277,13 @@ typedef struct io_desc_t
      * 1-based mappings to the global array for that task. */
     PIO_Offset *map;
 
+    /** If the map passed in is not monotonically increasing
+     *  then map is sorted and remap is an array of original
+     * indices of map. */
+
+    /** Remap. */
+    int *remap;
+
     /** Number of tasks involved in the communication between comp and
      * io tasks. */
     int nrecvs;
@@ -278,7 +295,7 @@ typedef struct io_desc_t
      * dimensions. */
     int ndims;
 
-    /** An array of size ndims with the length of each dimension. */
+    /** An array of size ndims with the global length of each dimension. */
     int *dimlen;
 
     /** The actual number of IO tasks participating. */
@@ -294,6 +311,9 @@ typedef struct io_desc_t
      * everywhere (false) */
     bool needsfill;
 
+    /** If the map is not monotonically increasing we will need to sort it. */
+    bool needssort;
+
     /** The maximum number of bytes of this iodesc before flushing. */
     int maxbytes;
 
@@ -371,8 +391,9 @@ typedef struct io_desc_t
      * group. */
     MPI_Comm subset_comm;
 
-    /** Pointer to the next io_desc_t in the list. */
-    struct io_desc_t *next;
+    /** Hash table entry. */
+    UT_hash_handle hh;
+
 } io_desc_t;
 
 /**
@@ -400,7 +421,7 @@ typedef struct iosystem_desc_t
     MPI_Comm comp_comm;
 
     /** This is an MPI inter communicator between IO communicator and
-     * computation communicator. */
+     * computation communicator, only used for async mode. */
     MPI_Comm intercomm;
 
     /** This is a copy (but not an MPI copy) of either the comp (for
@@ -519,8 +540,11 @@ typedef struct wmulti_buffer
     /** Pointer to the data. */
     void *data;
 
-    /** Pointer to the next multi-buffer in the list. */
-    struct wmulti_buffer *next;
+    /** uthash handle for hash of buffers */
+    int htid;
+
+    /** Hash table entry. */
+    UT_hash_handle hh;
 } wmulti_buffer;
 
 /**
@@ -554,7 +578,7 @@ typedef struct file_desc_t
 
     /** The wmulti_buffer is used to aggregate multiple variables with
      * the same communication pattern prior to a write. */
-    struct wmulti_buffer buffer;
+    struct wmulti_buffer *buffer;
 
     /** Data buffer for this file. */
     void *iobuf;
@@ -562,8 +586,8 @@ typedef struct file_desc_t
     /** PIO data type. */
     int pio_type;
 
-    /** Pointer to the next file_desc_t in the list of open files. */
-    struct file_desc_t *next;
+    /** Hash table entry. */
+    UT_hash_handle hh;
 
     /** True if this task should participate in IO (only true for one
      * task with netcdf serial files. */
@@ -748,11 +772,16 @@ enum PIO_ERROR_HANDLERS
 #define PIO_EINDEP  (-203)
 #endif /* _PNETCDF */
 
-/** Define error codes for PIO. */
+/** The first error code for PIO. */
 #define PIO_FIRST_ERROR_CODE (-500)
+
+/** Bad IOTYPE error. */
 #define PIO_EBADIOTYPE  (-500)
 
-/** ??? */
+/** Variable dimensions do not match in a multivar call. */
+#define PIO_EVARDIMMISMATCH (-501)
+
+/** Request null. */
 #define PIO_REQ_NULL (NC_REQ_NULL-1)
 
 #if defined(__cplusplus)
@@ -804,36 +833,63 @@ extern "C" {
                         int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm,
                         int rearranger, int *iosysidp);
 
-    int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms,
-                            MPI_Comm io_comm, int *iosysidp);
+    /* How many IO tasks in this iosysid? */
     int PIOc_get_numiotasks(int iosysid, int *numiotasks);
+
+    /* Initialize PIO for intracomm mode. */
     int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
                             int *iosysidp);
+
+    /** Shut down an iosystem and free all associated resources. Use
+     * PIOc_free_iosystem() instead. */
     int PIOc_finalize(int iosysid);
 
+    /* Shut down an iosystem and free all associated resources. */
+    int PIOc_free_iosystem(int iosysid);
+
     /* Set error handling for entire io system. */
     int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method);
 
     /* Set error handling for entire io system. */
     int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method);
 
+    /* Determine whether this is IO task. */
     int PIOc_iam_iotask(int iosysid, bool *ioproc);
+
+    /* What is the iorank? */
     int PIOc_iotask_rank(int iosysid, int *iorank);
+
+    /* Is this iosystem active? */
     int PIOc_iosystem_is_active(int iosysid, bool *active);
+
+    /* Is this IOTYPE available? */
     int PIOc_iotype_available(int iotype);
+
+    /* Set the options for the rearranger. */
     int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd,
                             bool enable_hs_c2i, bool enable_isend_c2i,
                             int max_pend_req_c2i,
                             bool enable_hs_i2c, bool enable_isend_i2c,
                             int max_pend_req_i2c);
-    /* Distributed data. */
+
+    /* Increment record number. */
     int PIOc_advanceframe(int ncid, int varid);
+
+    /* Set the record number. */
     int PIOc_setframe(int ncid, int varid, int frame);
+
+    /* Write a distributed array. */
     int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
                           void *fillvalue);
+
+    /* Write multiple darrays. */
     int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen,
                                 void *array, const int *frame, void **fillvalue, bool flushtodisk);
+
+    /* Read distributed array. */
     int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array);
+
+    /* Get size of local distributed array. */
     int PIOc_get_local_array_size(int ioid);
 
     /* Handling files. */
@@ -899,7 +955,6 @@ extern "C" {
                              int deflate_level);
     int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
                              int *deflate_levelp);
-    int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
     int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
     int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
     int PIOc_def_var_endian(int ncid, int varid, int endian);
@@ -1140,68 +1195,56 @@ extern "C" {
                                 const PIO_Offset *count, const PIO_Offset *stride,
                                 const unsigned long long *op);
 
-    /* Varm functions are deprecated and should be used with extreme
-     * caution or not at all. Varm functions are not supported in
-     * async mode. */
-    int PIOc_put_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                      const PIO_Offset *stride, const PIO_Offset *imap, const void *buf,
-                      PIO_Offset bufcount, MPI_Datatype buftype);
-    int PIOc_get_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap, signed char *buf);
-    int PIOc_put_varm_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap,
+    /* Data reads - vard. */
+    int PIOc_get_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, void *buf);
+    int PIOc_get_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                           char *buf);
+    int PIOc_get_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            signed char *buf);
+    int PIOc_get_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            short *buf);
+    int PIOc_get_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                          int *buf);
+    int PIOc_get_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            float *buf);
+    int PIOc_get_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                             double *buf);
+    int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            unsigned char *buf);
+    int PIOc_get_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                             unsigned short *buf);
+    int PIOc_get_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                           unsigned int *buf);
+    int PIOc_get_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                               long long *buf);
+    int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                                unsigned long long *buf);
+
+    /* Data writes - vard. */
+    int PIOc_put_vard(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                      const void *buf);
+    int PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                           const char *op);
+    int PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            const signed char *op);
+    int PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            const short *op);
+    int PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                          const int *op);
+    int PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                            const float *op);
+    int PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                             const double *op);
+    int PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum,
                             const unsigned char *op);
-    int PIOc_put_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap, const short *op);
-    int PIOc_get_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap, short *buf);
-    int PIOc_get_varm_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                                const PIO_Offset *stride, const PIO_Offset *imap, unsigned long long *buf);
-    int PIOc_get_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                             const PIO_Offset *stride, const PIO_Offset *imap, unsigned short *buf);
-    int PIOc_get_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                               const PIO_Offset *stride, const PIO_Offset *imap, long long *buf);
-    int PIOc_put_varm_text(int ncid, int varid, const PIO_Offset *start,
-                           const PIO_Offset *count, const PIO_Offset *stride,
-                           const PIO_Offset *imap, const char *op);
-    int PIOc_put_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                             const PIO_Offset *stride, const PIO_Offset *imap, const unsigned short *op);
-    int PIOc_put_varm_ulonglong(int ncid, int varid, const PIO_Offset *start,
-                                const PIO_Offset *count, const PIO_Offset *stride,
-                                const PIO_Offset *imap, const unsigned long long *op);
-    int PIOc_put_varm_int(int ncid, int varid, const PIO_Offset *start,
-                          const PIO_Offset *count, const PIO_Offset *stride,
-                          const PIO_Offset *imap, const int *op);
-    int PIOc_put_varm_float(int ncid, int varid, const PIO_Offset *start,
-                            const PIO_Offset *count, const PIO_Offset *stride,
-                            const PIO_Offset *imap, const float *op);
-    int PIOc_put_varm_long(int ncid, int varid, const PIO_Offset *start,
-                           const PIO_Offset *count, const PIO_Offset *stride,
-                           const PIO_Offset *imap, const long *op);
-    int PIOc_put_varm_uint(int ncid, int varid, const PIO_Offset *start,
-                           const PIO_Offset *count, const PIO_Offset *stride,
-                           const PIO_Offset *imap, const unsigned int *op);
-    int PIOc_put_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                             const PIO_Offset *stride, const PIO_Offset *imap, const double *op);
-    int PIOc_put_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap, const signed char *op);
-    int PIOc_put_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                               const PIO_Offset *stride, const PIO_Offset *imap, const long long *op);
-    int PIOc_get_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                             const PIO_Offset *stride, const PIO_Offset *imap, double *buf);
-    int PIOc_get_varm_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                           const PIO_Offset *stride, const PIO_Offset *imap, char *buf);
-    int PIOc_get_varm_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                          const PIO_Offset *stride, const PIO_Offset *imap, int *buf);
-    int PIOc_get_varm_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                           const PIO_Offset *stride, const PIO_Offset *imap, unsigned int *buf);
-    int PIOc_get_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                      const PIO_Offset *stride, const PIO_Offset *imap, void *buf,
-                      PIO_Offset bufcount, MPI_Datatype buftype);
-    int PIOc_get_varm_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const PIO_Offset *imap, float *buf);
-    int PIOc_get_varm_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                           const PIO_Offset *stride, const PIO_Offset *imap, long *buf);
+    int PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                             const unsigned short *op);
+    int PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                           const unsigned int *op);
+    int PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                               const long long *op);
+    int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                                const unsigned long long *op);
 #if defined(__cplusplus)
 }
 #endif
diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c
index 4f205cede7a9..c8e0f91ec648 100644
--- a/src/clib/pio_darray.c
+++ b/src/clib/pio_darray.c
@@ -11,19 +11,31 @@
 #include 
 #include 
 #include 
+#include 
 
-/* 10MB default limit. */
+/**
+ * @defgroup PIO_read_darray_c Reading Distributes Arrays
+ * Read data from a netCDF file to a distributed array in C.
+ *
+ * @defgroup PIO_write_darray_c Writing Distributes Arrays
+ * Write data from a distributed array to a netCDF file in C.
+ */
+
+/** 10MB default limit. */
 PIO_Offset pio_buffer_size_limit = PIO_BUFFER_SIZE;
 
-/* Global buffer pool pointer. */
+/** Global buffer pool pointer. */
 void *CN_bpool = NULL;
 
-/* Maximum buffer usage. */
+/** Maximum buffer usage. */
 PIO_Offset maxusage = 0;
 
-/* For write_darray_multi_serial() and write_darray_multi_par() to
- * indicate whether fill or data are being written. */
+/** For write_darray_multi_serial() and write_darray_multi_par() to
+ * indicate that fill is being written. */
 #define DARRAY_FILL 1
+
+/** For write_darray_multi_serial() and write_darray_multi_par() to
+ * indicate that data are being written. */
 #define DARRAY_DATA 0
 
 /**
@@ -36,7 +48,8 @@ PIO_Offset maxusage = 0;
  * @return The previous limit setting.
  * @author Jim Edwards
  */
-PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
+PIO_Offset
+PIOc_set_buffer_size_limit(PIO_Offset limit)
 {
     PIO_Offset oldsize = pio_buffer_size_limit;
 
@@ -98,21 +111,23 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
  * the fill value to be used for missing data.
  * @param flushtodisk non-zero to cause buffers to be flushed to disk.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
-                            PIO_Offset arraylen, void *array, const int *frame,
-                            void **fillvalue, bool flushtodisk)
+int
+PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
+                        PIO_Offset arraylen, void *array, const int *frame,
+                        void **fillvalue, bool flushtodisk)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
     io_desc_t *iodesc;     /* Pointer to IO description information. */
     int rlen;              /* Total data buffer size. */
     var_desc_t *vdesc0;    /* First entry in array of var_desc structure for each var. */
-    int fndims;            /* Number of dims in the var in the file. */
+    int fndims, fndims2;            /* Number of dims in the var in the file. */
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
     int ierr;              /* Return code. */
+    void *tmparray;
 
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -144,8 +159,8 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         var_desc_t *vdesc;
         if ((ierr = get_var_desc(varids[v], &file->varlist, &vdesc)))
             return pio_err(ios, file, ierr, __FILE__, __LINE__);
-        if (vdesc->pio_type != iodesc->piotype)
-            return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+        /* if (vdesc->pio_type != iodesc->piotype)
+           return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);*/
     }
 
     /* Get a pointer to the variable info for the first variable. */
@@ -161,9 +176,17 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[0], &fndims)))
             return check_netcdf(file, ierr, __FILE__, __LINE__);
         LOG((3, "called PIOc_inq_varndims varids[0] = %d fndims = %d", varids[0], fndims));
+        for (int v=1; v < nvars; v++){
+            if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+            if(fndims != fndims2)
+                return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
+        }
+
     }
 
-    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters. */
     if (ios->async)
     {
         if (!ios->ioproc)
@@ -209,13 +232,13 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Share results known only on computation tasks with IO tasks. */
         if ((mpierr = MPI_Bcast(&fndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((3, "shared fndims = %d", fndims));
     }
 
@@ -246,7 +269,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
 
         /* If fill values are desired, and we're using the BOX
          * rearranger, insert fill values. */
-        if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX)
+        if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX && fillvalue)
         {
             LOG((3, "inerting fill values iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
             for (int nv = 0; nv < nvars; nv++)
@@ -257,16 +280,26 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     }
     else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc)
     {
-	/* this assures that iobuf is allocated on all iotasks thus
+        /* this assures that iobuf is allocated on all iotasks thus
            assuring that the flush_output_buffer call above is called
            collectively (from all iotasks) */
         if (!(file->iobuf = bget(1)))
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         LOG((3, "allocated token for variable buffer"));
     }
+    if (iodesc->needssort)
+    {
+        if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
+    }
+    else
+    {
+        tmparray = array;
+    }
 
     /* Move data from compute to IO tasks. */
-    if ((ierr = rearrange_comp2io(ios, iodesc, array, file->iobuf, nvars)))
+    if ((ierr = rearrange_comp2io(ios, iodesc, tmparray, file->iobuf, nvars)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
     /* Write the darray based on the iotype. */
@@ -296,7 +329,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         /* Release resources. */
         if (file->iobuf)
         {
-	    LOG((3,"freeing variable buffer in pio_darray"));
+            LOG((3,"freeing variable buffer in pio_darray"));
             brel(file->iobuf);
             file->iobuf = NULL;
         }
@@ -317,21 +350,22 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
              iodesc->holegridsize, iodesc->needsfill));
 
-	pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
+        pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
 
         /* Get a buffer. */
-	if (ios->io_rank == 0)
-	    vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
-	else if (iodesc->holegridsize > 0)
-	    vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
+        if (ios->io_rank == 0)
+            vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
+        else if (iodesc->holegridsize > 0)
+            vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
 
         /* copying the fill value into the data buffer for the box
          * rearranger. This will be overwritten with data where
          * provided. */
-        for (int nv = 0; nv < nvars; nv++)
-            for (int i = 0; i < iodesc->holegridsize; i++)
-                memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
-                       &((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
+        if(fillvalue)
+            for (int nv = 0; nv < nvars; nv++)
+                for (int i = 0; i < iodesc->holegridsize; i++)
+                    memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
+                           &((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
 
         /* Write the darray based on the iotype. */
         switch (file->iotype)
@@ -364,6 +398,9 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         }
     }
 
+    if(iodesc->needssort && tmparray != NULL)
+        free(tmparray);
+
     /* Flush data to disk for pnetcdf. */
     if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
         if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
@@ -372,6 +409,112 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     return PIO_NOERR;
 }
 
+/**
+ * Find the fill value that would be used for a variable, if fill mode
+ * was turned on.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param pio_type Type of the variable.
+ * @param type_size Size of one element of this type.
+ * @param fillvalue Pointer that will get the fill value.
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray_c
+ * @author Ed Hartnett
+ */
+static int
+pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_size,
+                          void *fillvalue)
+{
+    signed char byte_fill_value = NC_FILL_BYTE;
+    char char_fill_value = NC_FILL_CHAR;
+    short short_fill_value = NC_FILL_SHORT;
+    int int_fill_value = NC_FILL_INT;
+    float float_fill_value = NC_FILL_FLOAT;
+    double double_fill_value = NC_FILL_DOUBLE;
+    unsigned char ubyte_fill_value = NC_FILL_UBYTE;
+    unsigned short ushort_fill_value = NC_FILL_USHORT;
+    unsigned int uint_fill_value = NC_FILL_UINT;
+    long long int64_fill_value = NC_FILL_INT64;
+    unsigned long long uint64_fill_value = NC_FILL_UINT64;
+    char *string_fill_value = "";
+    int ret;
+
+    /* Check inputs. */
+    assert(fillvalue);
+
+    LOG((2, "pio_inq_var_fill_expected ncid %d varid %d pio_type %d type_size %d",
+         ncid, varid, pio_type, type_size));
+
+    /* Is there a _FillValue attribute? */
+    ret = PIOc_inq_att_eh(ncid, varid, "_FillValue", 0, NULL, NULL);
+
+    LOG((3, "pio_inq_var_fill_expected ret %d", ret));
+
+    /* If there is a fill value, get it. */
+    if (!ret)
+    {
+        if ((ret = PIOc_get_att(ncid, varid, "_FillValue", fillvalue)))
+            return ret;
+    }
+    else /* If no _FillValue at was found we still have work to do. */
+    {
+        /* Did we get some other error? */
+        if (ret != PIO_ENOTATT)
+            return ret;
+
+        /* What is the default fill value for this type? */
+        switch (pio_type)
+        {
+        case PIO_BYTE:
+            memcpy(fillvalue, &byte_fill_value, type_size);
+            break;
+        case PIO_CHAR:
+            memcpy(fillvalue, &char_fill_value, type_size);
+            break;
+        case PIO_SHORT:
+            memcpy(fillvalue, &short_fill_value, type_size);
+            break;
+        case PIO_INT:
+            memcpy(fillvalue, &int_fill_value, type_size);
+            break;
+        case PIO_FLOAT:
+            memcpy(fillvalue, &float_fill_value, type_size);
+            break;
+        case PIO_DOUBLE:
+            memcpy(fillvalue, &double_fill_value, type_size);
+            break;
+#if defined(_NETCDF4) || defined(_PNETCDF)
+        case PIO_UBYTE:
+            memcpy(fillvalue, &ubyte_fill_value, type_size);
+            break;
+        case PIO_USHORT:
+            memcpy(fillvalue, &ushort_fill_value, type_size);
+            break;
+        case PIO_UINT:
+            memcpy(fillvalue, &uint_fill_value, type_size);
+            break;
+        case PIO_INT64:
+            memcpy(fillvalue, &int64_fill_value, type_size);
+            break;
+        case PIO_UINT64:
+            memcpy(fillvalue, &uint64_fill_value, type_size);
+            break;
+#ifdef _NETCDF4
+        case PIO_STRING:
+            memcpy(fillvalue, string_fill_value, type_size);
+            break;
+#endif /* _NETCDF4 */
+#endif/* _NETCDF4 || _PNETCDF */
+        default:
+            return PIO_EBADTYPE;
+        }
+    }
+
+    return PIO_NOERR;
+}
+
 /**
  * Find the fillvalue that should be used for a variable.
  *
@@ -379,10 +522,11 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
  * @param varid the variable ID.
  * @param vdesc pointer to var_desc_t info for this var.
  * @returns 0 for success, non-zero error code for failure.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Ed Hartnett
  */
-int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
+int
+find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     int pio_type;
@@ -410,12 +554,20 @@ int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
     if (!(vdesc->fillvalue = malloc(type_size)))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
-    /* Get the fill value. */
+    /* Get the fill mode and value, if fill mode is on (which is will
+     * not be, because it is turned off at open/create). */
     if ((ierr = PIOc_inq_var_fill(file->pio_ncid, varid, &no_fill, vdesc->fillvalue)))
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
     vdesc->use_fill = no_fill ? 0 : 1;
     LOG((3, "vdesc->use_fill = %d", vdesc->use_fill));
 
+    /* Get the fill value one would expect, if NOFILL were not turned
+     * on. */
+    if (!vdesc->use_fill)
+        if ((ierr = pio_inq_var_fill_expected(file->pio_ncid, varid, pio_type, type_size,
+                                              vdesc->fillvalue)))
+            return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
     return PIO_NOERR;
 }
 
@@ -464,11 +616,12 @@ int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
  * @param fillvalue pointer to the fill value to be used for missing
  * data.
  * @returns 0 for success, non-zero error code for failure.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
-                      void *fillvalue)
+int
+PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                  void *fillvalue)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Info about file we are writing to. */
@@ -483,6 +636,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
     bufsize totfree;       /* Amount of free space in the buffer. */
     bufsize maxfree;       /* Max amount of free space in buffer. */
 #endif
+    int hashid;
     int mpierr = MPI_SUCCESS;  /* Return code from MPI functions. */
     int ierr = PIO_NOERR;      /* Return code. */
     size_t io_data_size;          /* potential size of data on io task */
@@ -519,38 +673,38 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
 
     /* If the type of the var doesn't match the type of the
      * decomposition, return an error. */
-    if (iodesc->piotype != vdesc->pio_type)
-        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
-    pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info",
-              __FILE__, __LINE__);
+    /* if (iodesc->piotype != vdesc->pio_type) */
+    /*     return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); */
+    /* pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info", */
+    /*           __FILE__, __LINE__); */
 
     /* If we don't know the fill value for this var, get it. */
     if (!vdesc->fillvalue)
         if ((ierr = find_var_fillvalue(file, varid, vdesc)))
             return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
 
-    /* Check that if the user passed a fill value, it is correct. */
-    if (fillvalue)
+    /* Check that if the user passed a fill value, it is correct. If
+     * use_fill is false, then find_var_fillvalue will not end up
+     * getting a fill value. */
+    if (fillvalue && vdesc->use_fill)
         if (memcmp(fillvalue, vdesc->fillvalue, vdesc->pio_type_size))
             return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
 
     /* Move to end of list or the entry that matches this ioid. */
-    for (wmb = &file->buffer; wmb->next; wmb = wmb->next)
-        if (wmb->ioid == ioid && wmb->recordvar == vdesc->rec_var)
-            break;
-    LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
+    hashid = ioid*10 + vdesc->rec_var;
+    HASH_FIND_INT( file->buffer, &hashid, wmb);
+    if (wmb)
+        LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
 
     /* If we did not find an existing wmb entry, create a new wmb. */
-    if (wmb->ioid != ioid || wmb->recordvar != vdesc->rec_var)
+    if (!wmb)
     {
         /* Allocate a buffer. */
-        if (!(wmb->next = bget((bufsize)sizeof(wmulti_buffer))))
+        if (!(wmb = bget((bufsize)sizeof(wmulti_buffer))))
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
 
         /* Set pointer to newly allocated buffer and initialize.*/
-        wmb = wmb->next;
         wmb->recordvar = vdesc->rec_var;
-        wmb->next = NULL;
         wmb->ioid = ioid;
         wmb->num_arrays = 0;
         wmb->arraylen = arraylen;
@@ -558,14 +712,16 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
         wmb->data = NULL;
         wmb->frame = NULL;
         wmb->fillvalue = NULL;
+        wmb->htid = hashid;
+        HASH_ADD_INT( file->buffer, htid, wmb );
     }
-    LOG((2, "wmb->num_arrays = %d arraylen = %d vdesc->mpi_type_size = %d\n",
-         wmb->num_arrays, arraylen, vdesc->mpi_type_size));
+    LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->mpitype_size = %d\n",
+         wmb->num_arrays, arraylen, iodesc->mpitype_size));
 #if PIO_USE_MALLOC
     /* Try realloc first and call flush if realloc fails. */
     if (arraylen > 0)
     {
-        size_t data_size = (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size;
+        size_t data_size = (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size;
 
         if ((realloc_data = realloc(wmb->data, data_size)))
         {
@@ -586,20 +742,20 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
     /* maxfree is the available memory. If that is < 10% greater than
      * the size of the current request needsflush is true. */
     if (needsflush == 0)
-        needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size);
+        needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size);
 #endif
     /* the limit of data_size < INT_MAX is due to a bug in ROMIO which limits
        the size of contiguous data to INT_MAX, a fix has been proposed in
        https://github.com/pmodels/mpich/pull/2888 */
-    io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * vdesc->mpi_type_size;
+    io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * iodesc->mpitype_size;
     if(io_data_size > INT_MAX)
-	needsflush = 2;
+        needsflush = 2;
 
     /* Tell all tasks on the computation communicator whether we need
      * to flush data. */
     if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1,  MPI_INT,  MPI_MAX,
                                 ios->comp_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     LOG((2, "needsflush = %d", needsflush));
 
     /* Flush data if needed. */
@@ -610,8 +766,8 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
         /* Collect a debug report about buffer. */
         cn_buffer_report(ios, true);
         LOG((2, "maxfree = %ld wmb->num_arrays = %d (1 + wmb->num_arrays) *"
-             " arraylen * vdesc->mpi_type_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays,
-             (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size, totfree));
+             " arraylen * iodesc->mpitype_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays,
+             (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size, totfree));
 #endif /* PIO_ENABLE_LOGGING */
 #endif /* !PIO_USE_MALLOC */
 
@@ -626,17 +782,17 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
     /* Try realloc again if there is a flush. */
     if (arraylen > 0 && needsflush > 0)
     {
-        if (!(wmb->data = realloc(wmb->data, (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size)))
+        if (!(wmb->data = realloc(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size)))
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
-        LOG((2, "after a flush, realloc got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size));
+        LOG((2, "after a flush, realloc got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size));
     }
 #else
     /* Get memory for data. */
     if (arraylen > 0)
     {
-        if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size)))
+        if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size)))
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
-        LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * vdesc->mpi_type_size));
+        LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size));
     }
 #endif
 
@@ -658,11 +814,11 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
     if (iodesc->needsfill)
     {
         /* Get memory to hold fill value. */
-        if (!(wmb->fillvalue = bgetr(wmb->fillvalue, vdesc->mpi_type_size * (1 + wmb->num_arrays))))
+        if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->mpitype_size * (1 + wmb->num_arrays))))
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
 
-        memcpy((char *)wmb->fillvalue + vdesc->mpi_type_size * wmb->num_arrays,
-               vdesc->fillvalue, vdesc->mpi_type_size);
+        memcpy((char *)wmb->fillvalue + iodesc->mpitype_size * wmb->num_arrays,
+               vdesc->fillvalue, iodesc->mpitype_size);
     }
 
     /* Tell the buffer about the data it is getting. */
@@ -672,11 +828,11 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
          wmb->vid[wmb->num_arrays]));
 
     /* Copy the user-provided data to the buffer. */
-    bufptr = (void *)((char *)wmb->data + arraylen * vdesc->mpi_type_size * wmb->num_arrays);
+    bufptr = (void *)((char *)wmb->data + arraylen * iodesc->mpitype_size * wmb->num_arrays);
     if (arraylen > 0)
     {
-        memcpy(bufptr, array, arraylen * vdesc->mpi_type_size);
-        LOG((3, "copied %ld bytes of user data", arraylen * vdesc->mpi_type_size));
+        memcpy(bufptr, array, arraylen * iodesc->mpitype_size);
+        LOG((3, "copied %ld bytes of user data", arraylen * iodesc->mpitype_size));
     }
 
     /* Add the unlimited dimension value of this variable to the frame
@@ -685,9 +841,9 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
         wmb->frame[wmb->num_arrays] = vdesc->record;
     wmb->num_arrays++;
 
-    LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / vdesc->mpi_type_size = %d "
+    LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->mpitype_size = %d "
          "iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays,
-         iodesc->maxbytes / vdesc->mpi_type_size, iodesc->ndof, iodesc->llen));
+         iodesc->maxbytes / iodesc->mpitype_size, iodesc->ndof, iodesc->llen));
 
     return PIO_NOERR;
 }
@@ -697,27 +853,31 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID to be read
- * @param ioid: the I/O description ID as passed back by
+ * @param ioid the I/O description ID as passed back by
  * PIOc_InitDecomp().
- * @param arraylen: the length of the array to be read. This
- * is the length of the distrubited array. That is, the length of
- * the portion of the data that is on the processor.
- * @param array: pointer to the data to be read. This is a
+ * @param arraylen this parameter is ignored. Nominally it is the
+ * length of the array to be read. This is the length of the
+ * distrubited array. That is, the length of the portion of the data
+ * that is on the processor. This is already known because it is in
+ * the decomposition.
+ * @param array pointer to the data to be read. This is a
  * pointer to the distributed portion of the array that is on this
  * processor.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_read_darray
+ * @ingroup PIO_read_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
-                     void *array)
+int
+PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
+                 void *array)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
     io_desc_t *iodesc;     /* Pointer to IO description information. */
     void *iobuf = NULL;    /* holds the data as read on the io node. */
     size_t rlen = 0;       /* the length of data in iobuf. */
-    int ierr;           /* Return code. */
+    int ierr;              /* Return code. */
+    void *tmparray;        /* unsorted copy of array buf if required */
 
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -758,10 +918,26 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
         return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
     }
 
+    if (iodesc->needssort)
+    {
+        if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for(int m=0; mmaplen;m++)
+            ((int *) array)[m] = -1;
+    }
+    else
+        tmparray = array;
+
     /* Rearrange the data. */
-    if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array)))
+    if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+    if (iodesc->needssort)
+    {
+        pio_sorted_copy(tmparray, array, iodesc, 1, 1);
+        free(tmparray);
+    }
+
     /* Free the buffer. */
     if (rlen > 0)
         brel(iobuf);
diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c
index ea4c23b2de97..58859bba4322 100644
--- a/src/clib/pio_darray_int.c
+++ b/src/clib/pio_darray_int.c
@@ -1,4 +1,5 @@
-/** @file
+/**
+ * @file
  *
  * Private functions to help read and write distributed arrays in PIO.
  *
@@ -13,20 +14,31 @@
 #include 
 #include 
 
-/* 10MB default limit. */
+#if USE_VARD
+#define USE_VARD_READ 1
+#define USE_VARD_WRITE 1
+#endif
+
+/** 10MB default limit. */
 extern PIO_Offset pio_buffer_size_limit;
 
-/* Initial size of compute buffer. */
+/** Initial size of compute buffer. */
 bufsize pio_cnbuffer_limit = 33554432;
 
-/* Global buffer pool pointer. */
+/** Global buffer pool pointer. */
 extern void *CN_bpool;
 
-/* Maximum buffer usage. */
+/** Maximum buffer usage. */
 extern PIO_Offset maxusage;
 
-/* handler for freeing the memory buffer pool */
-void bpool_free(void *p)
+/**
+ * Handler for freeing the memory buffer pool.
+ *
+ * @param p pointer to the memory buffer pool.
+ * @author Jim Edwards
+ */
+void
+bpool_free(void *p)
 {
     free(p);
     if(p == CN_bpool){
@@ -46,7 +58,8 @@ void bpool_free(void *p)
  * @returns 0 for success, error code otherwise.
  * @author Jim Edwards
  */
-int compute_buffer_init(iosystem_desc_t *ios)
+int
+compute_buffer_init(iosystem_desc_t *ios)
 {
 #if !PIO_USE_MALLOC
 
@@ -67,9 +80,234 @@ int compute_buffer_init(iosystem_desc_t *ios)
     return PIO_NOERR;
 }
 
+#if USE_VARD
+/**
+ * Get the length of dimension 0.
+ *
+ * @param file pointer to the file descriptor.
+ * @param iosdesc pointer to the iosystem descriptor.
+ * @param varid variable ID.
+ * @param fndims number of dimensions in the file.
+ * @param gdim0 pointer that gets gdim0.
+ * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
+ */
+int
+get_gdim0(file_desc_t *file,io_desc_t *iodesc, int varid, int fndims,
+          MPI_Offset *gdim0)
+{
+    int ierr = PIO_NOERR;
+
+    *gdim0 = 0;
+    if (file->iotype == PIO_IOTYPE_PNETCDF && iodesc->ndims < fndims)
+    {
+        int numunlimdims;
+
+        /* We need to confirm the file has an unlimited dimension and
+           if it doesn't we need to find the extent of the first
+           variable dimension. */
+        LOG((3,"look for numunlimdims"));
+        if ((ierr = PIOc_inq_unlimdims(file->pio_ncid, &numunlimdims, NULL)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((3,"numunlimdims = %d", numunlimdims));
+        if (numunlimdims <= 0)
+        {
+            int dimids[fndims];
+            if ((ierr = PIOc_inq_vardimid(file->pio_ncid, varid, dimids)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+            if ((ierr = PIOc_inq_dimlen(file->pio_ncid, dimids[0], gdim0)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+    }
+    LOG((3,"gdim0 = %d",*gdim0));
+    return ierr;
+}
+
+
+/**
+ * Get the MPI data type of vard.
+ *
+ * @param iosdesc pointer to the iosystem descriptor.
+ * @param gdim0
+ * @param unlimdimoffset
+ * @param rrcnt
+ * @param ndims the number of dimensions in the decomposition.
+ * @param fndims the number of dimensions in the file.
+ * @param varid variable ID.
+ * @param fndims number of dimensions in the file.
+ * @param frame the record number.
+ * @param startlist
+ * @param countlist
+ * @param filetype a pointer that gets the MPI data type.
+ * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
+ */
+static
+int get_vard_mpidatatype(io_desc_t *iodesc, MPI_Offset gdim0, PIO_Offset unlimdimoffset,
+                         int rrcnt, int ndims, int fndims,
+                         int frame, PIO_Offset **startlist, PIO_Offset **countlist,
+                         MPI_Datatype *filetype)
+{
+
+    int sa_ndims;
+    int gdims[fndims];
+    int dim_offset;
+    int mpierr;
+    MPI_Aint displacements[rrcnt];
+    int blocklengths[rrcnt];
+    MPI_Datatype subarray[rrcnt];
+
+    /* preserve the value of unlimdimoffset, as it may be changed */
+    PIO_Offset _unlimdimoffset = unlimdimoffset;
+
+    *filetype = MPI_DATATYPE_NULL;
+
+    if(rrcnt == 0)
+        return PIO_NOERR;
+
+    for ( int rc=0; rc ndims)
+    {
+        if ( gdim0 > 0)
+        {
+            gdims[0] = gdim0;
+            sa_ndims = fndims;
+            dim_offset = 0;
+            for (int i=1; i < fndims; i++)
+                gdims[i] = iodesc->dimlen[i-1];
+        }
+        else
+        {
+            sa_ndims = ndims;
+            dim_offset = 1;
+            for (int i=0; i < ndims; i++)
+                gdims[i] = iodesc->dimlen[i];
+        }
+    }
+    else
+    {
+        sa_ndims = fndims;
+        dim_offset = 0;
+        for (int i=0; i < fndims; i++)
+            gdims[i] = iodesc->dimlen[i];
+    }
+
+    int true_rrcnt=-1; /* true number of contiguous requests */
+    MPI_Aint prev_end=-1; /* end offset of rc-1 request */
+    for( int rc=0; rc 0)
+        {
+            unlimdimoffset = gdim0;
+            sastart[0] = max(0, frame);
+            displacements[rc]=0;
+        }
+        else
+            displacements[rc] = unlimdimoffset * max(0, frame);
+
+        /* Check whether this request is actually contiguous. If contiguous,
+         * we do not need to create an MPI derived datatype.
+         */
+        int blocklen=1, isContig=1, warnContig=0;
+        MPI_Aint disp=0, shape=iodesc->mpitype_size;
+        for (int i=sa_ndims-1; i>=0; i--)
+        {
+            if (isContig) {
+                /* blocklen is the amount of this request, rc */
+                blocklen *= sacount[i];
+                /* disp is the flattened starting array index */
+                disp += sastart[i] * shape;
+                /* shape is the dimension product from sa_ndims-1 to i */
+                shape *= gdims[i];
+
+                if (warnContig == 0) {
+                    if (sacount[i] < gdims[i])
+                        /* first i detected to access partial dimension. If this
+                         * one is contiguous, the remaining sacount[i-1 ... 0]
+                         * must all == 1 */
+                        warnContig = 1; /* possible non-contiguos */
+                }
+                else if (sacount[i] != 1) {
+                    isContig = 0;
+                    break; /* loop i */
+                }
+            }
+        }
+        /* if this is a record variable, add the gap of record size */
+        disp += _unlimdimoffset * max(0, frame);
+
+#if PIO_ENABLE_LOGGING
+        for (int i=0; i< sa_ndims; i++)
+            LOG((3, "vard: sastart[%d]=%d sacount[%d]=%d gdims[%d]=%d %ld %ld displacement = %ld un %d",
+                 i,sastart[i], i,sacount[i], i, gdims[i], startlist[rc][i], countlist[rc][i], displacements[rc], unlimdimoffset));
+#endif
+        if (isContig) { /* this request rc is contiguous, no need to create a new MPI datatype */
+            if (prev_end == disp) {
+                /* this request rc can be coalesced into the previous
+                 * displacements and blocklengths.
+                 */
+                blocklengths[true_rrcnt] += blocklen;
+                prev_end += blocklen;
+            }
+            else {
+                /* this request cannot be coalesced with the previous one */
+                true_rrcnt++;
+                subarray[true_rrcnt] = iodesc->mpitype;
+                displacements[true_rrcnt] = disp;
+                blocklengths[true_rrcnt] = blocklen;
+                prev_end = disp + blocklen;
+            }
+        }
+        else { /* request rc is not contiguous, must create a new MPI datatype */
+            true_rrcnt++;
+            if((mpierr = MPI_Type_create_subarray(sa_ndims, gdims,
+                                                  sacount, sastart,MPI_ORDER_C
+                                                  ,iodesc->mpitype, subarray + true_rrcnt)))
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+            if((mpierr = MPI_Type_commit(subarray + true_rrcnt)))
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+        }
+
+#if PIO_ENABLE_LOGGING
+        LOG((3,"vard: blocklengths[%d]=%d displacement[%d]=%ld unlimdimoffset=%ld",rc,blocklengths[rc], rc, displacements[rc], unlimdimoffset));
+#endif
+
+    }
+    true_rrcnt++;
+
+    /* concatenate all MPI datatypes into filetype */
+    if((mpierr = MPI_Type_create_struct(true_rrcnt, blocklengths, displacements, subarray, filetype)))
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+    if((mpierr = MPI_Type_commit(filetype)))
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+    for( int rc=0; rcmpitype &&
+            (mpierr = MPI_Type_free(subarray + rc)))
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+#endif
+
 /**
  * Fill start/count arrays for write_darray_multi_par(). This is an
- * internal funciton.
+ * internal function.
  *
  * @param ndims the number of dims in the decomposition.
  * @param fndims the number of dims in the file.
@@ -81,12 +319,13 @@ int compute_buffer_init(iosystem_desc_t *ios)
  * @param count an already-allocated array which gets the count
  * values.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Ed Hartnett
  */
-int find_start_count(int ndims, int fndims, var_desc_t *vdesc,
-                     io_region *region, const int *frame, size_t *start,
-                     size_t *count)
+int
+find_start_count(int ndims, int fndims, var_desc_t *vdesc,
+                 io_region *region, const int *frame, size_t *start,
+                 size_t *count)
 {
     /* Init start/count arrays to zero. */
     for (int i = 0; i < fndims; i++)
@@ -116,7 +355,8 @@ int find_start_count(int ndims, int fndims, var_desc_t *vdesc,
             }
             else if (fndims == ndims)
             {
-                /* ??? */
+                /* In some cases the unlimited dim is not treated as
+                   the pio record dim */
                 start[0] += vdesc->record;
             }
         }
@@ -149,23 +389,29 @@ int find_start_count(int ndims, int fndims, var_desc_t *vdesc,
  * that will be written to
  * @param nvars the number of variables to be written with this
  * decomposition.
- * @param vid: an array of the variable ids to be written.
+ * @param fndims number of dimensions of this var in the file.
+ * @param varids an array of the variable ids to be written.
  * @param iodesc pointer to the io_desc_t info.
  * @param fill Non-zero if this write is fill data.
  * @param frame the record dimension for each of the nvars variables
  * in iobuf. NULL if this iodesc contains non-record vars.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *varids,
-                           io_desc_t *iodesc, int fill, const int *frame)
+int
+write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *varids,
+                       io_desc_t *iodesc, int fill, const int *frame)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     var_desc_t *vdesc;    /* Pointer to var info struct. */
     int dsize;             /* Data size (for one region). */
     int ierr = PIO_NOERR;
-
+#if USE_VARD_WRITE
+    PIO_Offset gdim0;  /* global size of first dimension if no unlimited dimension and ndimsiosystem && varids && varids[0] >= 0 && varids[0] <= PIO_MAX_VARS &&
               iodesc, "invalid input", __FILE__, __LINE__);
@@ -175,9 +421,10 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
          iodesc->mpitype, iodesc->maxregions, iodesc->llen));
 
 #ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:write_darray_multi_par");
-#endif
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:write_darray_multi_par")))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Get pointer to iosystem. */
     ios = file->iosystem;
@@ -192,18 +439,27 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
     PIO_Offset llen = fill ? iodesc->holegridsize : iodesc->llen;
     void *iobuf = fill ? vdesc->fillbuf : file->iobuf;
 
+#if USE_VARD_WRITE
+    if (!ios->async || !ios->ioproc)
+    {
+        if ((ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0)))
+            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+
+    }
+#endif
+
     /* If this is an IO task write the data. */
     if (ios->ioproc)
     {
-        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
         void *bufptr;
         size_t start[fndims];
         size_t count[fndims];
         int ndims = iodesc->ndims;
+#ifdef _PNETCDF
+        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
         PIO_Offset *startlist[num_regions]; /* Array of start arrays for ncmpi_iput_varn(). */
         PIO_Offset *countlist[num_regions]; /* Array of count  arrays for ncmpi_iput_varn(). */
-
-        LOG((3, "num_regions = %d", num_regions));
+#endif /* _PNETCDF */
 
         /* Process each region of data to be written. */
         for (int regioncnt = 0; regioncnt < num_regions; regioncnt++)
@@ -273,22 +529,151 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
                 /* Do this when we reach the last region. */
                 if (regioncnt == num_regions - 1)
                 {
+#ifdef USE_VARD_WRITE
+                    MPI_Aint   var0_offset, var_offsets[nvars];
+                    MPI_Offset vari_offset, vard_llen=0;
+                    MPI_Datatype vartypes[nvars];
+                    MPI_Datatype filetype = MPI_DATATYPE_NULL;
+                    int blocklens[nvars];
+                    int fvartype, var0_id;
+                    int numReqs=0;
+                    void *vard_bufptr;
+                    int doFlush[nvars]; /* whether to flush or not */
+
+                    /* construct doFlush[], so later when looping through nvars,
+                     * it tells whether to flush or not.
+                     */
+                    for (int nv = 0; nv < nvars; nv++) {
+                        /* Get the var info. */
+                        if ((ierr = get_var_desc(varids[nv], &file->varlist, &vdesc)))
+                            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+                        if (nv == 0) { /* first variable */
+                            fvartype = vdesc->pio_type; /* first var's var type */
+                            continue;
+                        }
+                        if (fvartype != vdesc->pio_type) {
+                            /* nv's external datatype is different from nv-1 */
+                            doFlush[nv-1] = 1;
+                            fvartype = vdesc->pio_type;
+                        }
+                        else /* same as nv-1, no flush */
+                            doFlush[nv-1] = 0;
+                    }
+                    doFlush[nvars-1] = 1; /* flush when reach the last variable */
+#endif
                     /* For each variable to be written. */
                     for (int nv = 0; nv < nvars; nv++)
                     {
+#if USE_VARD_WRITE
+                        /* PnetCDF 1.10.0 and later support type conversion in
+                         * vard APIs. However, it requires all variables
+                         * accessed by the filetype are of the same NC data
+                         * type.
+                         */
+
+                        /* obtain file offset of variable nv */
+                        if ((ierr = ncmpi_inq_varoffset(file->fh, varids[nv], &vari_offset)))
+                            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+
+                        if (numReqs == 0) { /* 1st variable of same datatype */
+                            var0_offset = vari_offset;
+                            var0_id = varids[nv];
+                        }
+                        /* calculate the offset relative to the first var */
+                        var_offsets[numReqs] = vari_offset - var0_offset;
+                        blocklens[nv] = 1; /* 1 for each vartypes[nv] */
+
+                        /* If this is the first variable or the frame has changed between variables (this should be rare) */
+                        if(nv==0 || (nv > 0 && frame != NULL && frame[nv] != frame[nv-1])){
+                            int thisframe;
+                            PIO_Offset unlimdimoffset;
+                            if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */
+                            {
+                                if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset)))
+                                    return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+                                LOG((3, "num_regions = %d unlimdimoffset %ld", num_regions, unlimdimoffset));
+                            }else
+                                unlimdimoffset = gdim0;
+                            if (frame)
+                                thisframe = frame[nv];
+                            else
+                                thisframe = 0;
+
+                            ierr = get_vard_mpidatatype(iodesc, gdim0, unlimdimoffset,
+                                                        rrcnt, ndims, fndims,
+                                                        thisframe, startlist, countlist,
+                                                        &vartypes[numReqs]);
+                        }
+                        else /* reuse the previous variable's datatype */
+                            vartypes[numReqs] = vartypes[numReqs-1];
+#else
                         /* Get the var info. */
                         if ((ierr = get_var_desc(varids[nv], &file->varlist, &vdesc)))
                             return pio_err(NULL, file, ierr, __FILE__, __LINE__);
 
-                        /* If this is a record var, set the start for
-                         * the record dimension. */
                         if (vdesc->record >= 0 && ndims < fndims)
                             for (int rc = 0; rc < rrcnt; rc++)
                                 startlist[rc][0] = frame[nv];
-
+#endif
                         /* Get a pointer to the data. */
                         bufptr = (void *)((char *)iobuf + nv * iodesc->mpitype_size * llen);
 
+#if USE_VARD_WRITE
+                        if (numReqs == 0) { /* first var of the same type */
+                            vard_bufptr = bufptr; /* preserve variable ID */
+                            vard_llen = llen;     /* reset I/O request size */
+                        }
+                        numReqs++;
+
+                        if (doFlush[nv]) { /* flush the data now */
+                            int mpierr;
+                            /* concatenate vartypes[0...numReqs-1] */
+                            if (numReqs > 1) {
+                                /* check and remove NULL vartype */
+                                int i, j=0;
+                                for (i=0; i 0) { /* at least one vartypes[] is not NULL */
+                                    /* concatenate non-NULL vartypes */
+                                    if((mpierr = MPI_Type_create_struct(j, blocklens, var_offsets, vartypes, &filetype)))
+                                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+                                    if((mpierr = MPI_Type_commit(&filetype)))
+                                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+                                    /* free vartypes */
+                                    for (i=j-1; i>0; i--) {
+                                        if (vartypes[i] == vartypes[i-1]) continue;
+                                        if((mpierr = MPI_Type_free(&vartypes[i])))
+                                            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+                                    }
+                                    if((mpierr = MPI_Type_free(&vartypes[0])))
+                                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+                                }
+                                else /* all vartypes[] are NULL */
+                                    filetype = MPI_DATATYPE_NULL;
+                            }
+                            else /* there is only one variable to flush */
+                                filetype = vartypes[0];
+
+                            LOG((3, "vard: call ncmpi_put_vard llen = %d %d", llen, iodesc->mpitype_size ));
+                            ierr = ncmpi_put_vard_all(file->fh, var0_id, filetype, vard_bufptr, vard_llen, iodesc->mpitype);
+                            LOG((3, "vard: return ncmpi_put_vard ierr = %d", ierr));
+                            if(filetype != MPI_DATATYPE_NULL)
+                            {
+                                if((mpierr = MPI_Type_free(&filetype)))
+                                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+                            }
+                            vard_llen = 0; /* reset request size to 0 */
+                            numReqs = 0;
+                        }
+                        else /* don't flush yet, accumulate the request size */
+                            vard_llen += llen;
+#else
                         if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0)
                         {
                             if (!(vdesc->request = realloc(vdesc->request, sizeof(int) *
@@ -310,6 +695,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
                             vdesc->request[vdesc->nreqs] = PIO_REQ_NULL;
 
                         vdesc->nreqs++;
+#endif
                     }
 
                     /* Free resources. */
@@ -335,9 +721,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
     ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:write_darray_multi_par");
-#endif
+    if ((ierr = pio_stop_timer("PIO:write_darray_multi_par")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return ierr;
 }
@@ -362,12 +748,13 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
  * fndims * maxregions. This array will get the count values for all
  * regions.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_read_darray
+ * @ingroup PIO_read_darray_c
  * @author Jim Edwards, Ed Hartnett
  **/
-int find_all_start_count(io_region *region, int maxregions, int fndims,
-                         int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start,
-                         size_t *tmp_count)
+int
+find_all_start_count(io_region *region, int maxregions, int fndims,
+                     int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start,
+                     size_t *tmp_count)
 {
     /* Check inputs. */
     pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc &&
@@ -429,12 +816,13 @@ int find_all_start_count(io_region *region, int maxregions, int fndims,
  * than IO task 0. It is called by write_darray_multi_serial().
  *
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset llen,
-                         int maxregions, int nvars, int fndims, size_t *tmp_start,
-                         size_t *tmp_count, void *iobuf)
+int
+send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset llen,
+                     int maxregions, int nvars, int fndims, size_t *tmp_start,
+                     size_t *tmp_count, void *iobuf)
 {
     MPI_Status status;     /* Recv status for MPI. */
     int mpierr;  /* Return code from MPI function codes. */
@@ -446,12 +834,12 @@ int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset lle
 
     /* Do a handshake. */
     if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Send local length of iobuffer for each field (all
      * fields are the same length). */
     if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((3, "sent llen = %d", llen));
 
     /* Send the number of data regions, the start/count for
@@ -460,16 +848,16 @@ int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset lle
     {
         if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks,
                                ios->io_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0,
                                ios->io_rank + 2 * ios->num_iotasks, ios->io_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0,
                                ios->io_rank + 3 * ios->num_iotasks, ios->io_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Send(iobuf, nvars * llen, iodesc->mpitype, 0,
                                ios->io_rank + 4 * ios->num_iotasks, ios->io_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "sent data for maxregions = %d", maxregions));
     }
 
@@ -508,12 +896,13 @@ int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset lle
  * less than blocksize*numiotasks then some iotasks will have a NULL
  * iobuf.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
-                        io_desc_t *iodesc, PIO_Offset llen, int maxregions, int nvars,
-                        int fndims, size_t *tmp_start, size_t *tmp_count, void *iobuf)
+int
+recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
+                    io_desc_t *iodesc, PIO_Offset llen, int maxregions, int nvars,
+                    int fndims, size_t *tmp_start, size_t *tmp_count, void *iobuf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     size_t rlen;    /* Length of IO buffer on this task. */
@@ -546,13 +935,13 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
         {
             /* handshake - tell the sending task I'm ready */
             if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm)))
-                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
             /* Get length of iobuffer for each field on this
              * task (all fields are the same length). */
             if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm,
                                    &status)))
-                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
             LOG((3, "received rlen = %d", rlen));
 
             /* Get the number of regions, the start/count
@@ -561,16 +950,16 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
             {
                 if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks,
                                        ios->io_comm, &status)))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask,
                                        rtask + 2 * ios->num_iotasks, ios->io_comm, &status)))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask,
                                        rtask + 3 * ios->num_iotasks, ios->io_comm, &status)))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Recv(iobuf, nvars * rlen, iodesc->mpitype, rtask,
                                        rtask + 4 * ios->num_iotasks, ios->io_comm, &status)))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
                 LOG((3, "received data rregions = %d fndims = %d", rregions, fndims));
             }
         }
@@ -623,6 +1012,11 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
                         }
                     }
 
+#ifdef LOGGING
+                    for (int i = 1; i < fndims; i++)
+                        LOG((3, "start[%d] %d count[%d] %d", i, start[i], i, count[i]));
+#endif /* LOGGING */
+
                     /* Call the netCDF functions to write the data. */
                     if ((ierr = nc_put_vara(file->fh, varids[nv], start, count, bufptr)))
                         return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
@@ -656,17 +1050,19 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
  * that will be written to.
  * @param nvars the number of variables to be written with this
  * decomposition.
+ * @param fndims number of dims in the vars in the file.
  * @param varids an array of the variable ids to be written
  * @param iodesc pointer to the decomposition info.
  * @param fill Non-zero if this write is fill data.
  * @param frame the record dimension for each of the nvars variables
  * in iobuf. NULL if this iodesc contains non-record vars.
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const int *varids,
-                              io_desc_t *iodesc, int fill, const int *frame)
+int
+write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const int *varids,
+                          io_desc_t *iodesc, int fill, const int *frame)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     var_desc_t *vdesc;     /* Contains info about the variable. */
@@ -677,7 +1073,7 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in
               varids[0] <= PIO_MAX_VARS && iodesc, "invalid input", __FILE__, __LINE__);
 
     LOG((1, "write_darray_multi_serial nvars = %d fndims = %d iodesc->ndims = %d "
-         "iodesc->mpitype = %d", nvars, iodesc->ndims, fndims, iodesc->mpitype));
+         "iodesc->mpitype = %d", nvars, fndims, iodesc->ndims, iodesc->mpitype));
 
     /* Get the iosystem info. */
     ios = file->iosystem;
@@ -694,9 +1090,10 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in
     void *iobuf = fill ? vdesc->fillbuf : file->iobuf;
 
 #ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:write_darray_multi_serial");
-#endif
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:write_darray_multi_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Only IO tasks participate in this code. */
     if (ios->ioproc)
@@ -732,9 +1129,9 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:write_darray_multi_serial");
-#endif
+    if ((ierr = pio_stop_timer("PIO:write_darray_multi_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -754,28 +1151,35 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in
  * less than blocksize*numiotasks then some iotasks will have a NULL
  * iobuf.
  * @return 0 on success, error code otherwise.
- * @ingroup PIO_read_darray
+ * @ingroup PIO_read_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
+int
+pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     var_desc_t *vdesc;     /* Information about the variable. */
     int ndims;             /* Number of dims in decomposition. */
     int fndims;            /* Number of dims for this var in file. */
     int ierr;              /* Return code from netCDF functions. */
+#ifdef USE_VARD_READ
+    MPI_Offset gdim0;
+    gdim0 = 0;
+#endif
 
     /* Check inputs. */
     pioassert(file && file->iosystem && iodesc && vid <= PIO_MAX_VARS, "invalid input",
               __FILE__, __LINE__);
 
-#ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:read_darray_nc");
-#endif
-
     /* Get the IO system info. */
     ios = file->iosystem;
+    LOG((3, "pio_read_darray_nc ios->ioproc %d", ios->ioproc));
+
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:read_darray_nc")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Get the variable info. */
     if ((ierr = get_var_desc(vid, &file->varlist, &vdesc)))
@@ -787,8 +1191,13 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
     /* Get the number of dims for this var in the file. */
     if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
+#if USE_VARD_READ
+    if(!ios->async || !ios->ioproc)
+        ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0);
+#endif
+    /* LOG((4, "fndims %d ndims %d", fndims, ndims)); */
 
-    /* IO procs will actially read the data. */
+    /* IO procs will read the data. */
     if (ios->ioproc)
     {
         io_region *region;
@@ -805,10 +1214,14 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
            the mpitype. */
         region = iodesc->firstregion;
 
-        /* ??? */
+        /* There are different numbers of dims in the decomposition
+         * and the file. */
         if (fndims > ndims)
         {
-            ndims++;
+            /* If the user did not call setframe, use a default frame
+             * of 0. This is required for backward compatibility. */
+            if (vdesc->record < 0)
+                vdesc->record = 0;
         }
 
         /* For each regions, read the data. */
@@ -833,7 +1246,8 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
                 else
                     bufptr=(void *)((char *)iobuf + iodesc->mpitype_size * region->loffset);
 
-                LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset));
+                LOG((2, "iodesc->llen - region->loffset %d, iodesc->llen %d, region->loffset  %d vdesc->record %d",
+                     iodesc->llen - region->loffset, iodesc->llen, region->loffset, vdesc->record));
 
                 /* Get the start/count arrays. */
                 if (vdesc->record >= 0 && fndims > 1)
@@ -841,7 +1255,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
                     /* This is a record var. The unlimited dimension
                      * (0) is handled specially. */
                     start[0] = vdesc->record;
-                    for (int i = 1; i < ndims; i++)
+                    for (int i = 1; i < fndims; i++)
                     {
                         start[i] = region->start[i-1];
                         count[i] = region->count[i-1];
@@ -854,7 +1268,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
                 else
                 {
                     /* Non-time dependent array */
-                    for (int i = 0; i < ndims; i++)
+                    for (int i = 0; i < fndims; i++)
                     {
                         start[i] = region->start[i];
                         count[i] = region->count[i];
@@ -862,6 +1276,10 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
                 }
             }
 
+#ifdef PIO_ENABLE_LOGGING
+            for (int i = 1; i < ndims; i++)
+                LOG((3, "start[%d] %d count[%d] %d", i, start[i], i, count[i]));
+#endif /* LOGGING */
             /* Do the read. */
             switch (file->iotype)
             {
@@ -934,10 +1352,30 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
                 /* Is this is the last region to process? */
                 if (regioncnt == iodesc->maxregions - 1)
                 {
+#if USE_VARD_READ
+                    MPI_Datatype filetype;
+                    PIO_Offset unlimdimoffset;
+                    int mpierr;
+                    if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */
+                    {
+                        if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset)))
+                            return pio_err(NULL, file, ierr, __FILE__, __LINE__);
+                    }
+                    else
+                        unlimdimoffset = gdim0;
+
+                    ierr = get_vard_mpidatatype(iodesc, gdim0, unlimdimoffset,
+                                                rrlen, ndims, fndims,
+                                                vdesc->record, startlist, countlist, &filetype);
+                    ierr = ncmpi_get_vard_all(file->fh, vid, filetype, iobuf, iodesc->llen, iodesc->mpitype);
+                    if(filetype != MPI_DATATYPE_NULL && (mpierr = MPI_Type_free(&filetype)))
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+#else
                     /* Read a list of subarrays. */
                     ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist,
                                               countlist, iobuf, iodesc->llen, iodesc->mpitype);
-
+#endif
                     /* Release the start and count arrays. */
                     for (int i = 0; i < rrlen; i++)
                     {
@@ -963,9 +1401,9 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:read_darray_nc");
-#endif
+    if ((ierr = pio_stop_timer("PIO:read_darray_nc")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -987,11 +1425,12 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
  * less than blocksize * numiotasks then some iotasks will have a NULL
  * iobuf.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_read_darray
+ * @ingroup PIO_read_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
-                              void *iobuf)
+int
+pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
+                          void *iobuf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     var_desc_t *vdesc;    /* Information about the variable. */
@@ -1006,12 +1445,14 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
               "invalid input", __FILE__, __LINE__);
 
     LOG((2, "pio_read_darray_nc_serial vid = %d", vid));
-#ifdef TIMING
-    /* Start timing this function. */
-    GPTLstart("PIO:read_darray_nc_serial");
-#endif
     ios = file->iosystem;
 
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ierr = pio_start_timer("PIO:read_darray_nc_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* Get var info for this var. */
     if ((ierr = get_var_desc(vid, &file->varlist, &vdesc)))
         return pio_err(NULL, file, ierr, __FILE__, __LINE__);
@@ -1023,6 +1464,11 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
     if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+    /* If setframe was not called, use a default value of 0. This is
+     * required for backward compatibility. */
+    if (fndims == ndims + 1 && vdesc->record < 0)
+        vdesc->record = 0;
+
     /* Confirm that we are being called with the correct ndims. */
     pioassert((fndims == ndims && vdesc->record < 0) ||
               (fndims == ndims + 1 && vdesc->record >= 0),
@@ -1043,6 +1489,12 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
            the mpitype. */
         region = iodesc->firstregion;
 
+        /* If setframe was not set before this call, assume a value of
+         * 0. This is required for backward compatibility. */
+        if (fndims > ndims)
+            if (vdesc->record < 0)
+                vdesc->record = 0;
+
         /* Put together start/count arrays for all regions. */
         for (int regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++)
         {
@@ -1103,25 +1555,25 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
         if (ios->io_rank > 0)
         {
             if ((mpierr = MPI_Send(&iodesc->llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
-                return check_mpi(file, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
             LOG((3, "sent iodesc->llen = %d", iodesc->llen));
 
             if (iodesc->llen > 0)
             {
                 if ((mpierr = MPI_Send(&(iodesc->maxregions), 1, MPI_INT, 0,
                                        ios->num_iotasks + ios->io_rank, ios->io_comm)))
-                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Send(tmp_count, iodesc->maxregions * fndims, MPI_OFFSET, 0,
                                        2 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
-                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Send(tmp_start, iodesc->maxregions * fndims, MPI_OFFSET, 0,
                                        3 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
-                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                 LOG((3, "sent iodesc->maxregions = %d tmp_count and tmp_start arrays", iodesc->maxregions));
 
                 if ((mpierr = MPI_Recv(iobuf, iodesc->llen, iodesc->mpitype, 0,
                                        4 * ios->num_iotasks + ios->io_rank, ios->io_comm, &status)))
-                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                 LOG((3, "received %d elements of data", iodesc->llen));
             }
         }
@@ -1139,20 +1591,20 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
                 if (rtask < ios->num_iotasks)
                 {
                     if ((mpierr = MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status)))
-                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                     LOG((3, "received tmp_bufsize = %d", tmp_bufsize));
 
                     if (tmp_bufsize > 0)
                     {
                         if ((mpierr = MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks + rtask,
                                                ios->io_comm, &status)))
-                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                         if ((mpierr = MPI_Recv(this_count, maxregions * fndims, MPI_OFFSET, rtask,
                                                2 * ios->num_iotasks + rtask, ios->io_comm, &status)))
-                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                         if ((mpierr = MPI_Recv(this_start, maxregions * fndims, MPI_OFFSET, rtask,
                                                3 * ios->num_iotasks + rtask, ios->io_comm, &status)))
-                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
                         LOG((3, "received maxregions = %d this_count, this_start arrays ", maxregions));
                     }
                 }
@@ -1250,15 +1702,15 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
                 if (rtask < ios->num_iotasks && tmp_bufsize > 0)
                     if ((mpierr = MPI_Send(iobuf, tmp_bufsize, iodesc->mpitype, rtask,
                                            4 * ios->num_iotasks + rtask, ios->io_comm)))
-                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
             }
         }
     }
 
 #ifdef TIMING
-    /* Stop timing this function. */
-    GPTLstop("PIO:read_darray_nc_serial");
-#endif
+    if ((ierr = pio_stop_timer("PIO:read_darray_nc_serial")))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -1272,10 +1724,11 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
  * @param force true to force the flushing of the buffer
  * @param addsize additional size to add to buffer (in bytes)
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
+int
+flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
 {
     int mpierr;  /* Return code from MPI functions. */
     int ierr = PIO_NOERR;
@@ -1286,30 +1739,31 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
 
     /* Check inputs. */
     pioassert(file, "invalid input", __FILE__, __LINE__);
-
+    LOG((1, "flush_output_buffer"));
     /* Find out the buffer usage. */
     if ((ierr = ncmpi_inq_buffer_usage(file->fh, &usage)))
-	/* allow the buffer to be undefined */
-	if (ierr != NC_ENULLABUF)
-	    return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__);
+        /* allow the buffer to be undefined */
+        if (ierr != NC_ENULLABUF)
+            return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__);
 
     /* If we are not forcing a flush, spread the usage to all IO
      * tasks. */
-    if (!force && file->iosystem->io_comm != MPI_COMM_NULL)
+    if (!force && file->iosystem->ioproc)
     {
         usage += addsize;
         if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &usage, 1,  MPI_OFFSET,  MPI_MAX,
                                     file->iosystem->io_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* Keep track of the maximum usage. */
     if (usage > maxusage)
         maxusage = usage;
 
+    LOG((2, "flush_output_buffer usage=%ld force=%d",usage, force));
     /* If the user forces it, or the buffer has exceeded the size
      * limit, then flush to disk. */
-    if (force || usage >= pio_buffer_size_limit)
+    if (force || (usage >= pio_buffer_size_limit))
     {
         int rcnt;
         int  maxreq;
@@ -1345,6 +1799,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
 #endif
             for (reqcnt = 0; reqcnt < vdesc->nreqs; reqcnt++)
                 request[rcnt++] = max(vdesc->request[reqcnt], NC_REQ_NULL);
+            LOG((3,"flush_output_buffer rcnt=%d",rcnt));
 
             if (vdesc->request != NULL)
                 free(vdesc->request);
@@ -1390,10 +1845,11 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
  *
  * @param ios pointer to the IO system structure
  * @param collective true if collective report is desired
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards
  */
-void cn_buffer_report(iosystem_desc_t *ios, bool collective)
+void
+cn_buffer_report(iosystem_desc_t *ios, bool collective)
 {
     int mpierr;  /* Return code from MPI functions. */
 
@@ -1410,10 +1866,10 @@ void cn_buffer_report(iosystem_desc_t *ios, bool collective)
         {
             LOG((3, "cn_buffer_report calling MPI_Reduce ios->comp_comm = %d", ios->comp_comm));
             if ((mpierr = MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios->comp_comm)))
-                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             LOG((3, "cn_buffer_report calling MPI_Reduce"));
             if ((mpierr = MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios->comp_comm)))
-                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             if (ios->compmaster == MPI_ROOT)
             {
                 LOG((1, "Currently allocated buffer space %ld %ld", bget_mins[0], bget_maxs[0]));
@@ -1439,10 +1895,11 @@ void cn_buffer_report(iosystem_desc_t *ios, bool collective)
  * non zero), this function does nothing.
  *
  * @param ios pointer to the IO system structure.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards
  */
-void free_cn_buffer_pool(iosystem_desc_t *ios)
+void
+free_cn_buffer_pool(iosystem_desc_t *ios)
 {
 #if !PIO_USE_MALLOC
     LOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool));
@@ -1465,10 +1922,11 @@ void free_cn_buffer_pool(iosystem_desc_t *ios)
  * @param wmb pointer to the wmulti_buffer structure.
  * @param flushtodisk if true, then flush data to disk.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_write_darray
+ * @ingroup PIO_write_darray_c
  * @author Jim Edwards, Ed Hartnett
  */
-int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk)
+int
+flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk)
 {
     file_desc_t *file;
     int ret;
@@ -1518,3 +1976,305 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk)
     return PIO_NOERR;
 }
 
+/**
+ * Sort the contents of an array.
+ *
+ * @param array pointer to the array
+ * @param sortedarray pointer that gets the sorted array.
+ * @param iodesc pointer to the iodesc.
+ * @param nvars number of variables.
+ * @param direction sort direction.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray_c
+ * @author Jim Edwards
+ */
+int
+pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc,
+                int nvars, int direction)
+{
+    int maplen = iodesc->maplen;
+
+    if (direction == 0){
+        switch (iodesc->piotype)
+        {
+        case PIO_BYTE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((signed char *)sortedarray)[m+maplen*v] = ((signed char *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_CHAR:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((char *)sortedarray)[m+maplen*v] = ((char *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_SHORT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((short *)sortedarray)[m+maplen*v] = ((short *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+
+            break;
+        case PIO_INT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((int *)sortedarray)[m+maplen*v] = ((int *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_FLOAT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((float *)sortedarray)[m+maplen*v] = ((float *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_DOUBLE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((double *)sortedarray)[m+maplen*v] = ((double *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_UBYTE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned char *)sortedarray)[m+maplen*v] = ((unsigned char *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_USHORT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned short *)sortedarray)[m+maplen*v] = ((unsigned short *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_UINT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned int *)sortedarray)[m+maplen*v] = ((unsigned int *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_INT64:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((long long *)sortedarray)[m+maplen*v] = ((long long *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_UINT64:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned long long *)sortedarray)[m+maplen*v] = ((unsigned long long *)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        case PIO_STRING:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((char **)sortedarray)[m+maplen*v] = ((char **)array)[iodesc->remap[m]+maplen*v];
+                }
+            }
+            break;
+        default:
+            return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__);
+        }
+    }
+    else
+    {
+        switch (iodesc->piotype)
+        {
+        case PIO_BYTE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((signed char *)sortedarray)[iodesc->remap[m]+maplen*v] = ((signed char *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_CHAR:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((char *)sortedarray)[iodesc->remap[m]+maplen*v] = ((char *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_SHORT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((short *)sortedarray)[iodesc->remap[m]+maplen*v] = ((short *)array)[m+maplen*v];
+                }
+            }
+
+            break;
+        case PIO_INT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((int *)sortedarray)[iodesc->remap[m]+maplen*v] = ((int *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_FLOAT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((float *)sortedarray)[iodesc->remap[m]+maplen*v] = ((float *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_DOUBLE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((double *)sortedarray)[iodesc->remap[m]+maplen*v] = ((double *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_UBYTE:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned char *)sortedarray)[iodesc->remap[m]+maplen*v] = ((unsigned char *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_USHORT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned short *)sortedarray)[iodesc->remap[m]+maplen*v] = ((unsigned short *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_UINT:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned int *)sortedarray)[iodesc->remap[m]+maplen*v] = ((unsigned int *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_INT64:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((long long *)sortedarray)[iodesc->remap[m]+maplen*v] = ((long long *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_UINT64:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((unsigned long long *)sortedarray)[iodesc->remap[m]+maplen*v] = ((unsigned long long *)array)[m+maplen*v];
+                }
+            }
+            break;
+        case PIO_STRING:
+            for (int v=0; v < nvars; v++)
+            {
+                for (int m=0; m < maplen; m++)
+                {
+                    ((char **)sortedarray)[iodesc->remap[m]+maplen*v] = ((char **)array)[m+maplen*v];
+                }
+            }
+            break;
+        default:
+            return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__);
+        }
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Compute the maximum aggregate number of bytes. This is called by
+ * subset_rearrange_create() and box_rearrange_create().
+ *
+ * @param ios pointer to the IO system structure.
+ * @param iodesc a pointer to decomposition description.
+ * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
+ */
+int
+compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int maxbytesoniotask = INT_MAX;
+    int maxbytesoncomputetask = INT_MAX;
+    int maxbytes;
+    int mpierr;  /* Return code from MPI functions. */
+
+    /* Check inputs. */
+    pioassert(iodesc, "invalid input", __FILE__, __LINE__);
+
+    LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d",
+         iodesc->maxiobuflen, iodesc->ndof));
+
+    /* Determine the max bytes that can be held on IO task. */
+    if (ios->ioproc && iodesc->maxiobuflen > 0)
+        maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen;
+
+    /* Determine the max bytes that can be held on computation task. */
+    if (ios->comp_rank >= 0 && iodesc->ndof > 0)
+        maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof;
+
+    /* Take the min of the max IO and max comp bytes. */
+    maxbytes = min(maxbytesoniotask, maxbytesoncomputetask);
+    LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d",
+         maxbytesoniotask, maxbytesoncomputetask));
+
+    /* Get the min value of this on all tasks. */
+    LOG((3, "before allreaduce maxbytes = %d", maxbytes));
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN,
+                                ios->union_comm)))
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "after allreaduce maxbytes = %d", maxbytes));
+
+    /* Remember the result. */
+    iodesc->maxbytes = maxbytes;
+
+    return PIO_NOERR;
+}
diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c
index 211d546426af..f9703a218353 100644
--- a/src/clib/pio_file.c
+++ b/src/clib/pio_file.c
@@ -5,6 +5,22 @@
 #include 
 #include 
 #include 
+#include 
+
+/**
+ * @defgroup PIO_open_file_c Open a File
+ * Open an existing netCDF file with PIO in C.
+ *
+ * @defgroup PIO_create_file_c Create a File
+ * Create a new netCDF file with PIO in C.
+ *
+ * @defgroup PIO_sync_file_c Sync a File
+ * Flush buffers and sync data to disk in C.
+ *
+ * @defgroup PIO_close_file_c Close a File
+ * Close a file in C.
+ *
+ */
 
 /* This is the next ncid that will be used when a file is opened or
    created. We start at 16 so that it will be easy for us to notice
@@ -27,7 +43,7 @@ int pio_next_ncid = 16;
  * @param filename : The filename to open
  * @param mode : The netcdf mode for the open operation
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_openfile
+ * @ingroup PIO_open_file_c
  * @author Jim Edwards, Ed Hartnett
  */
 int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
@@ -54,7 +70,7 @@ int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @param filename : The filename to open
  * @param mode : The netcdf mode for the open operation
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_openfile
+ * @ingroup PIO_open_file_c
  * @author Ed Hartnett
  */
 int PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *filename,
@@ -75,7 +91,7 @@ int PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @param mode The netcdf mode for the open operation
  * @param ncidp pointer to int where ncid will go
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_openfile
+ * @ingroup PIO_open_file_c
  * @author Ed Hartnett
  */
 int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
@@ -120,7 +136,7 @@ int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
  * @param filename The filename to create.
  * @param mode The netcdf mode for the create operation.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_createfile
+ * @ingroup PIO_create_file_c
  * @author Jim Edwards, Ed Hartnett
  */
 int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
@@ -164,7 +180,7 @@ int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @param filename : The filename to open
  * @param ncidp : A pio file descriptor (output)
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_create
+ * @ingroup PIO_create_file_c
  * @author Ed Hartnett
  */
 int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
@@ -195,6 +211,7 @@ int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
  *
  * @param ncid: the file pointer
  * @returns PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_close_file_c
  * @author Jim Edwards, Ed Hartnett
  */
 int PIOc_closefile(int ncid)
@@ -205,7 +222,6 @@ int PIOc_closefile(int ncid)
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
 
     LOG((1, "PIOc_closefile ncid = %d", ncid));
-
     /* Find the info about this file. */
     if ((ierr = pio_get_file(ncid, &file)))
         return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
@@ -236,9 +252,9 @@ int PIOc_closefile(int ncid)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -270,7 +286,7 @@ int PIOc_closefile(int ncid)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -322,9 +338,9 @@ int PIOc_deletefile(int iosysid, const char *filename)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "done hanlding errors mpierr = %d", mpierr));
     }
 
@@ -346,7 +362,7 @@ int PIOc_deletefile(int iosysid, const char *filename)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
 
@@ -363,6 +379,7 @@ int PIOc_deletefile(int iosysid, const char *filename)
  *
  * @param ncid the ncid of the file to sync.
  * @returns PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_sync_file_c
  * @author Jim Edwards, Ed Hartnett
  */
 int PIOc_sync(int ncid)
@@ -387,25 +404,17 @@ int PIOc_sync(int ncid)
             wmulti_buffer *wmb, *twmb;
 
             LOG((3, "PIOc_sync checking buffers"));
-            wmb = &file->buffer;
-            while (wmb)
-            {
+	    HASH_ITER(hh, file->buffer, wmb, twmb)
+	      {  
                 /* If there are any data arrays waiting in the
                  * multibuffer, flush it. */
                 if (wmb->num_arrays > 0)
                     flush_buffer(ncid, wmb, true);
-                twmb = wmb;
-                wmb = wmb->next;
-                if (twmb == &file->buffer)
-                {
-                    twmb->ioid = -1;
-                    twmb->next = NULL;
-                }
-                else
-                {
-                    brel(twmb);
-                }
-            }
+		HASH_DEL(file->buffer, wmb);
+		brel(wmb);
+                
+	      }
+	    file->buffer = NULL;
         }
     }
 
@@ -425,9 +434,9 @@ int PIOc_sync(int ncid)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* Call the sync function on IO tasks. */
@@ -450,7 +459,6 @@ int PIOc_sync(int ncid)
 #ifdef _PNETCDF
             case PIO_IOTYPE_PNETCDF:
                 flush_output_buffer(file, true, 0);
-                ierr = ncmpi_sync(file->fh);
                 break;
 #endif
             default:
@@ -462,7 +470,7 @@ int PIOc_sync(int ncid)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
 
diff --git a/src/clib/pio_get_nc.c b/src/clib/pio_get_nc.c
index cb3ddc5b1c16..589a3de11e2d 100644
--- a/src/clib/pio_get_nc.c
+++ b/src/clib/pio_get_nc.c
@@ -1,17 +1,22 @@
 /**
  * @file
- * PIO functions to get data (excluding varm functions).
+ * PIO functions to get data.
  *
  * @author Ed Hartnett
  * @date  2016
  *
  * @see http://code.google.com/p/parallelio/
  */
-
 #include 
 #include 
 #include 
 
+/**
+ * @addtogroup PIO_get_vars_c Read Strided Arrays
+ * Read strided arrays of data from a variable in C.
+ * @{
+ */
+
 /**
  * Get strided, muti-dimensional subset of a text variable.
  *
@@ -340,6 +345,45 @@ int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start,
     return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT64, buf);
 }
 
+/**
+ * Get strided, muti-dimensional subset of a variable of the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_NAT, buf);
+}
+
+/**
+ * @}
+ */
+
+/**
+ * @addtogroup PIO_get_vara_c Read Arrays
+ * Read arrays of data from a variable in C, specifying start and
+ * count arrays.
+ * @{
+ */
+
 /**
  * Get a muti-dimensional subset of a text variable.
  *
@@ -630,6 +674,41 @@ int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start,
     return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT64, buf);
 }
 
+/**
+ * Get a muti-dimensional subset of a variable the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_NAT, buf);
+}
+
+/**
+ * @}
+ */
+
+/**
+ * @addtogroup PIO_get_var_c Read Entire Variable
+ * Read the entire variable at one time into an array in C.
+ * @{
+ */
+
 /**
  * Get all data of a text variable.
  *
@@ -834,6 +913,34 @@ int PIOc_get_var_longlong(int ncid, int varid, long long *buf)
     return PIOc_get_var_tc(ncid, varid, NC_INT64, buf);
 }
 
+/**
+ * Get all data from a variable the same type as the variable in the
+ * file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_var(int ncid, int varid, void *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_NAT, buf);
+}
+
+/**
+ * @}
+ */
+
+/**
+ * @addtogroup PIO_get_var1_c Read One Value
+ * Read one value from a variable in C.
+ * @{
+ */
+
 /**
  * Get one value of a text variable.
  *
@@ -1078,24 +1185,6 @@ int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index,
     return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf);
 }
 
-/**
- * Get all data from a variable the same type as the variable in the
- * file.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
- */
-int PIOc_get_var(int ncid, int varid, void *buf)
-{
-    return PIOc_get_var_tc(ncid, varid, NC_NAT, buf);
-}
-
 /**
  * Get one value from a variable the same type as the variable in the
  * file.
@@ -1118,54 +1207,5 @@ int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf)
 }
 
 /**
- * Get a muti-dimensional subset of a variable the same type
- * as the variable in the file.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
- */
-int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                  void *buf)
-{
-    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_NAT, buf);
-}
-
-/**
- * Get strided, muti-dimensional subset of a variable of the same type
- * as the variable in the file.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
- * @param stride an array of strides (must have same number of
- * entries as variable has dimensions). If NULL, strides of 1 will be
- * used.
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
+ * @}
  */
-int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                  const PIO_Offset *stride, void *buf)
-{
-    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_NAT, buf);
-}
diff --git a/src/clib/pio_get_vard.c b/src/clib/pio_get_vard.c
new file mode 100644
index 000000000000..e0decf92dca2
--- /dev/null
+++ b/src/clib/pio_get_vard.c
@@ -0,0 +1,269 @@
+/**
+ * @file
+ * PIO functions to get data with distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date  2019
+ *
+ * @see https://github.com/NCAR/ParallelIO
+ */
+#include 
+#include 
+#include 
+
+/**
+ * @addtogroup PIO_read_darray_c
+ * Read distributed arrays from a variable in C.
+ * @{
+ */
+
+/**
+ * Get a muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_text(int ncid, int varid, int decompid,
+                       const PIO_Offset recnum, char *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_uchar(int ncid, int varid, int decompid,
+                        const PIO_Offset recnum, unsigned char *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_schar(int ncid, int varid, int decompid,
+                        const PIO_Offset recnum, signed char *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_BYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_ushort(int ncid, int varid, int decompid,
+                         const PIO_Offset recnum, unsigned short *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_USHORT,
+                            buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_short(int ncid, int varid, int decompid,
+                        const PIO_Offset recnum, short *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_SHORT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_uint(int ncid, int varid, int decompid,
+                       const PIO_Offset recnum, unsigned int *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UINT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_int(int ncid, int varid, int decompid,
+                      const PIO_Offset recnum, int *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_INT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_float(int ncid, int varid, int decompid,
+                        const PIO_Offset recnum, float *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_FLOAT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_double(int ncid, int varid, int decompid,
+                         const PIO_Offset recnum, double *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_DOUBLE,
+                            buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid,
+                            const PIO_Offset recnum, unsigned long long *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UINT64,
+                            buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard_longlong(int ncid, int varid, int decompid,
+                           const PIO_Offset recnum, long long *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_INT64, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a variable the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int PIOc_get_vard(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                  void *buf)
+{
+    return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_NAT, buf);
+}
+
+
+/**
+ * @}
+ */
diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c
index 6ae26141d6dd..2eb37064bb82 100644
--- a/src/clib/pio_getput_int.c
+++ b/src/clib/pio_getput_int.c
@@ -1,14 +1,12 @@
 /**
  * @file
- * Internal PIO functions to get and put attributes and data
- * (excluding varm functions).
+ * Internal PIO functions to get and put attributes and data.
  *
- * @author Ed Hartnett
- * @date  2016
+ * @see https://github.com/NCAR/ParallelIO
  *
- * @see http://code.google.com/p/parallelio/
+ * @author Ed Hartnett
+ * @date 2016
  */
-
 #include 
 #include 
 #include 
@@ -16,21 +14,23 @@
 /**
  * Write a netCDF attribute of any type, converting to any type.
  *
- * This routine is called collectively by all tasks in the communicator
- * ios.union_comm.
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
  * @param name the name of the attribute.
- * @param atttype the nc_type of the attribute.
+ * @param atttype the nc_type of the attribute in the file.
  * @param len the length of the attribute array.
+ * @param memtype the nc_type of the attribute data in memory.
  * @param op a pointer with the attribute data.
  * @return PIO_NOERR for success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
-                    PIO_Offset len, nc_type memtype, const void *op)
+int
+PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
+                PIO_Offset len, nc_type memtype, const void *op)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -109,15 +109,15 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "PIOc_put_att bcast from comproot = %d atttype_len = %d", ios->comproot,
              atttype_len, memtype_len));
     }
@@ -211,7 +211,7 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -236,7 +236,8 @@ int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip)
+int
+PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip)
 {
     iosystem_desc_t *ios;   /* Pointer to io system information. */
     file_desc_t *file;      /* Pointer to file information. */
@@ -326,19 +327,19 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "mpi errors handled"));
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         LOG((2, "PIOc_get_att_tc bcast from comproot = %d attlen = %d atttype_len = %d", ios->comproot, attlen, atttype_len));
         if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "PIOc_get_att_tc bcast complete attlen = %d atttype_len = %d memtype_len = %d", attlen, atttype_len,
              memtype_len));
     }
@@ -434,7 +435,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void
     /* Broadcast and check the return code. */
     LOG((2, "ierr = %d", ierr));
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -442,7 +443,7 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void
     LOG((2, "bcasting att values attlen = %d memtype_len = %d", attlen, memtype_len));
     if ((mpierr = MPI_Bcast(ip, (int)attlen * memtype_len, MPI_BYTE, ios->ioroot,
                             ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     LOG((2, "get_att_tc data bcast complete"));
     return PIO_NOERR;
@@ -483,8 +484,9 @@ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                     const PIO_Offset *stride, nc_type xtype, void *buf)
+int
+PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                 const PIO_Offset *stride, nc_type xtype, void *buf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -495,6 +497,8 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
     char start_present = start ? true : false;
     char count_present = count ? true : false;
     char stride_present = stride ? true : false;
+    PIO_Offset one = 1; /* For fake_stride. */
+    PIO_Offset *fake_stride = &one; /* Needed for NULL stride bug in netcdf-4.6.2. */
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
     int ierr;                           /* Return code. */
 
@@ -590,22 +594,39 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ndims)
+    {
+        if (!stride_present)
+        {
+            LOG((2, "stride not present "));
+            if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset))))
+                return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+            for (int d = 0; d < ndims; d++)
+                fake_stride[d] = 1;
+        }
+        else
+            fake_stride = (PIO_Offset *)stride;
     }
 
     /* If this is an IO task, then call the netCDF function. */
     if (ios->ioproc)
     {
+
         LOG((2, "file->iotype = %d xtype = %d file->do_io = %d", file->iotype, xtype, file->do_io));
 #ifdef _PNETCDF
         if (file->iotype == PIO_IOTYPE_PNETCDF)
@@ -653,72 +674,93 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
         }
 #endif /* _PNETCDF */
 
+        LOG((2, "duck ndims %d", ndims));
+        for (int d = 0; d < ndims; d++)
+        {
+            LOG((2, "start[%d]  %d", d, start[d]));
+            LOG((2, "count[%d]  %d", d, count[d]));
+            LOG((2, "fake_stride[%d]  %d", d, fake_stride[d]));
+        }
+
         if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
             switch(xtype)
             {
             case NC_BYTE:
                 ierr = nc_get_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_CHAR:
                 ierr = nc_get_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_SHORT:
                 ierr = nc_get_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_INT:
                 ierr = nc_get_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
-                                       (ptrdiff_t *)stride, buf);
+                                       (ptrdiff_t *)fake_stride, buf);
                 break;
             case PIO_LONG_INTERNAL:
                 ierr = nc_get_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_FLOAT:
                 ierr = nc_get_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_DOUBLE:
                 ierr = nc_get_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
-                                          (ptrdiff_t *)stride, buf);
+                                          (ptrdiff_t *)fake_stride, buf);
                 break;
 #ifdef _NETCDF4
             case NC_UBYTE:
                 ierr = nc_get_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_USHORT:
                 ierr = nc_get_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
-                                          (ptrdiff_t *)stride, buf);
+                                          (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_UINT:
                 ierr = nc_get_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_INT64:
                 LOG((3, "about to call nc_get_vars_longlong"));
                 ierr = nc_get_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
-                                            (ptrdiff_t *)stride, buf);
+                                            (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_UINT64:
                 ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
-                                             (ptrdiff_t *)stride, buf);
+                                             (ptrdiff_t *)fake_stride, buf);
                 break;
                 /* case NC_STRING: */
                 /*      ierr = nc_get_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
-                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*                                (ptrdiff_t *)fake_stride, (void *)buf); */
                 /*      break; */
 #endif /* _NETCDF4 */
             default:
                 return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
             }
+        }
+
     }
 
+    LOG((2, "howdy ndims %d", ndims));
+    for (int d = 0; d < ndims; d++)
+    {
+        LOG((2, "fake_stride[%d]  %d", d, fake_stride[d]));
+    }
+
+    /* Free malloced resources. */
+    if (ndims && !stride_present)
+        free(fake_stride);
+
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -726,7 +768,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
     LOG((2, "PIOc_get_vars_tc bcasting data num_elem = %d typelen = %d ios->ioroot = %d", num_elem,
          typelen, ios->ioroot));
     if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     LOG((2, "PIOc_get_vars_tc bcasting data complete"));
 
     return PIO_NOERR;
@@ -749,8 +791,9 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
-                     void *buf)
+int
+PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                 void *buf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -782,15 +825,13 @@ int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param index an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
  * @param xtype the netcdf type of the variable.
  * @param buf pointer that will get the data.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
+int
+PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -819,15 +860,24 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
         int dimids[ndims];
         if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids)))
             return pio_err(ios, file, ierr, __FILE__, __LINE__);
-	if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
-	    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
-	if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
-	    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
+        {
+            free(startp);
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        }
 
         /* Find the dimension lengths. */
         for (int d = 0; d < ndims; d++)
+        {
             if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &countp[d])))
+            {
+                free(startp);
+                free(countp);
                 return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            }
+        }
 
         /* Set up start array. */
         for (int d = 0; d < ndims; d++)
@@ -840,10 +890,10 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
     }
 
     ierr = PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf);
-    if(startp != NULL)
-	free(startp);
-    if(countp != NULL)
-	free(countp);
+    if (startp)
+        free(startp);
+    if (countp)
+        free(countp);
     return ierr;
 
 }
@@ -884,8 +934,9 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                     const PIO_Offset *stride, nc_type xtype, const void *buf)
+int
+PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                 const PIO_Offset *stride, nc_type xtype, const void *buf)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;  /* Pointer to file information. */
@@ -895,9 +946,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
     char start_present = start ? true : false;    /* Is start non-NULL? */
     char count_present = count ? true : false;    /* Is count non-NULL? */
     char stride_present = stride ? true : false;  /* Is stride non-NULL? */
-    var_desc_t *vdesc;
-    int *request;
     nc_type vartype;   /* The type of the var we are reading from. */
+    PIO_Offset *fake_stride;
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
     int ierr;          /* Return code from function calls. */
 
@@ -996,23 +1046,37 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr));
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         LOG((2, "PIOc_put_vars_tc bcast from comproot"));
         if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "PIOc_put_vars_tc complete bcast from comproot ndims = %d", ndims));
     }
 
     /* If this is an IO task, then call the netCDF function. */
     if (ios->ioproc)
     {
+        if (ndims)
+        {
+            if (!stride_present)
+            {
+                LOG((2, "stride not present"));
+                if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset))))
+                    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                for (int d = 0; d < ndims; d++)
+                    fake_stride[d] = 1;
+            }
+            else
+                fake_stride = (PIO_Offset *)stride;
+        }
+
 #ifdef _PNETCDF
         if (file->iotype == PIO_IOTYPE_PNETCDF)
         {
@@ -1067,18 +1131,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
             else
             {
                 /* This is not a scalar var. */
-                PIO_Offset *fake_stride;
-
-                if (!stride_present)
-                {
-                    LOG((2, "stride not present"));
-                    if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset))))
-                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
-                    for (int d = 0; d < ndims; d++)
-                        fake_stride[d] = 1;
-                }
-                else
-                    fake_stride = (PIO_Offset *)stride;
+                var_desc_t *vdesc;
+                int *request;
 
                 LOG((2, "PIOc_put_vars_tc calling pnetcdf function"));
                 /*vdesc = &file->varlist[varid];*/
@@ -1129,9 +1183,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
                 flush_output_buffer(file, false, 0);
                 LOG((2, "PIOc_put_vars_tc flushed output buffer"));
 
-                /* Free malloced resources. */
-                if (!stride_present)
-                    free(fake_stride);
             } /* endif ndims == 0 */
         }
 #endif /* _PNETCDF */
@@ -1144,56 +1195,56 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
             {
             case NC_BYTE:
                 ierr = nc_put_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_CHAR:
                 ierr = nc_put_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_SHORT:
                 ierr = nc_put_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_INT:
                 ierr = nc_put_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
-                                       (ptrdiff_t *)stride, buf);
+                                       (ptrdiff_t *)fake_stride, buf);
                 break;
             case PIO_LONG_INTERNAL:
                 ierr = nc_put_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_FLOAT:
                 ierr = nc_put_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_DOUBLE:
                 ierr = nc_put_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
-                                          (ptrdiff_t *)stride, buf);
+                                          (ptrdiff_t *)fake_stride, buf);
                 break;
 #ifdef _NETCDF4
             case NC_UBYTE:
                 ierr = nc_put_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
-                                         (ptrdiff_t *)stride, buf);
+                                         (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_USHORT:
                 ierr = nc_put_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
-                                          (ptrdiff_t *)stride, buf);
+                                          (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_UINT:
                 ierr = nc_put_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
-                                        (ptrdiff_t *)stride, buf);
+                                        (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_INT64:
                 ierr = nc_put_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
-                                            (ptrdiff_t *)stride, buf);
+                                            (ptrdiff_t *)fake_stride, buf);
                 break;
             case NC_UINT64:
                 ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
-                                             (ptrdiff_t *)stride, buf);
+                                             (ptrdiff_t *)fake_stride, buf);
                 break;
                 /* case NC_STRING: */
                 /*      ierr = nc_put_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
-                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*                                (ptrdiff_t *)fake_stride, (void *)buf); */
                 /*      break; */
 #endif /* _NETCDF4 */
             default:
@@ -1201,13 +1252,20 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
             }
             LOG((2, "PIOc_put_vars_tc io_rank 0 done with netcdf call, ierr=%d", ierr));
         }
+
+        /* Free malloced resources. */
+        if (ndims && !stride_present)
+            free(fake_stride);
+
+        if (ierr)
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
     }
 
     /* Broadcast and check the return code. */
-    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
-    if (ierr)
-        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    /* if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) */
+    /*     return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); */
+    /* if (ierr) */
+    /*     return check_netcdf(file, ierr, __FILE__, __LINE__); */
     LOG((2, "PIOc_put_vars_tc bcast netcdf return code %d complete", ierr));
 
     return PIO_NOERR;
@@ -1242,8 +1300,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
-                     const void *op)
+int
+PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                 const void *op)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1293,7 +1352,8 @@ int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
+int
+PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1319,10 +1379,14 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
     if (ndims)
     {
         int dimid[ndims];
-	if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
-	    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
-	if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
-	    return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(startp = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(countp = malloc(ndims * sizeof(PIO_Offset))))
+        {
+            free(startp);
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+
 
         /* Set up start array. */
         for (int d = 0; d < ndims; d++)
@@ -1330,19 +1394,147 @@ int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
 
         /* Get the dimids for this var. */
         if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid)))
+        {
+            free(startp);
+            free(countp);
             return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
 
         /* Count array are the dimlens. */
         for (int d = 0; d < ndims; d++)
+        {
             if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &countp[d])))
+            {
+                free(startp);
+                free(countp);
                 return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            }
+        }
 
     }
 
+    /* Call the vars function. */
     ierr = PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op);
-    if (startp != NULL)
-	free(startp);
-    if (countp != NULL)
-	free(countp);
+
+    /* Free any allocated resources. */
+    if (startp)
+        free(startp);
+    if (countp)
+        free(countp);
+
     return ierr;
 }
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * PIOc_get_vard() and related functions. This function gets
+ * distributed arrays of any type, converting them to any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from the type of the variable being
+ * read from to this type. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                 nc_type xtype, void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    var_desc_t *vdesc;     /* Pointer to var information. */
+    int ret;
+
+    LOG((1, "PIOc_get_vard_tc ncid %d varid %d decompid %d recnum %d "
+         "xtype %d", ncid, varid, decompid, recnum, xtype));
+
+    /* Get file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Set the value of the record dimension. */
+    if ((ret = PIOc_setframe(ncid, varid, recnum)))
+        return ret;
+
+    /* Get var info. */
+    if ((ret = get_var_desc(varid, &file->varlist, &vdesc)))
+        return pio_err(ios, file, ret, __FILE__, __LINE__);
+    LOG((2, "vdesc->pio_type %d", vdesc->pio_type));
+
+    /* Disallow type conversion for now. */
+    if (xtype != NC_NAT && xtype != vdesc->pio_type)
+        return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+
+    /* Read the distributed array. */
+    if ((ret = PIOc_read_darray(ncid, varid, decompid, 0, buf)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * PIOc_get_vard() and related functions. This function puts
+ * distributed arrays of any type, converting them to any type.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                 nc_type xtype, const void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    io_desc_t *iodesc;     /* The IO description. */
+    file_desc_t *file;     /* Pointer to file information. */
+    var_desc_t *vdesc;     /* Pointer to var information. */
+    int ret;
+
+    /* Get file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Set the value of the record dimension. */
+    if ((ret = PIOc_setframe(ncid, varid, recnum)))
+        return pio_err(ios, file, ret, __FILE__, __LINE__);
+
+    /* Get decomposition information. */
+    if (!(iodesc = pio_get_iodesc_from_id(decompid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Get var info. */
+    if ((ret = get_var_desc(varid, &file->varlist, &vdesc)))
+        return pio_err(ios, file, ret, __FILE__, __LINE__);
+
+    /* Disallow type conversion for now. */
+    if (xtype != NC_NAT && xtype != vdesc->pio_type)
+        return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+
+    /* Write the distributed array. */
+    if ((ret = PIOc_write_darray(ncid, varid, decompid, iodesc->ndof,
+                                 (void *)buf, NULL)))
+        return ret;
+
+    return PIO_NOERR;
+}
diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h
index a8947135a24f..26ea24176246 100644
--- a/src/clib/pio_internal.h
+++ b/src/clib/pio_internal.h
@@ -4,7 +4,7 @@
  * @author Jim Edwards, Ed Hartnett
  * @date  2014
  *
- * @see http://code.google.com/p/parallelio/
+ * @see https://github.com/NCAR/ParallelIO
  */
 
 #ifndef __PIO_INTERNAL__
@@ -12,14 +12,25 @@
 
 #include 
 #include 
+#include 
+#include 
+#include 
+#ifdef TIMING
+#include 
+#endif
+#include 
 
 /* These are the sizes of types in netCDF files. Do not replace these
  * constants with sizeof() calls for C types. They are not the
  * same. Even on a system where sizeof(short) is 4, the size of a
  * short in a netCDF file is 2 bytes. */
+/** Size (in bytes) of a char in a netCDF file. */
 #define NETCDF_CHAR_SIZE 1
+/** Size (in bytes) of a short in a netCDF file. */
 #define NETCDF_SHORT_SIZE 2
+/** Size (in bytes) of a int or float in a netCDF file. */
 #define NETCDF_INT_FLOAT_SIZE 4
+/** Size (in bytes) of a long long int or double in a netCDF file. */
 #define NETCDF_DOUBLE_INT64_SIZE 8
 
 /* It seems that some versions of openmpi fail to define
@@ -30,43 +41,43 @@
 #endif
 #endif
 #ifndef MPI_Offset
+/** This is the type used for PIO_Offset. */
 #define MPI_Offset long long
 #endif
 
+/** Some MPI implementations do not allow passing MPI_DATATYPE_NULL to
+ * comm functions even though the send or recv length is 0, in these
+ * cases we use MPI_CHAR */
 #if defined(MPT_VERSION) || defined(OPEN_MPI)
-/* Some MPI implementations do not allow passing MPI_DATATYPE_NULL to comm functions
- * even though the send or recv length is 0, in these cases we use MPI_CHAR */
 #define PIO_DATATYPE_NULL MPI_CHAR
 #else
 #define PIO_DATATYPE_NULL MPI_DATATYPE_NULL
 #endif
 
-#include 
-#include 
-#include 
-#ifdef TIMING
-#include 
-#endif
-#include 
-
 #if PIO_ENABLE_LOGGING
 void pio_log(int severity, const char *fmt, ...);
 #define LOG(e) pio_log e
 #else
+/** Logging macro for debugging. */
 #define LOG(e)
 #endif /* PIO_ENABLE_LOGGING */
 
+/** Find maximum. */
 #define max(a,b)                                \
     ({ __typeof__ (a) _a = (a);                 \
         __typeof__ (b) _b = (b);                \
         _a > _b ? _a : _b; })
 
+/** Find minimum. */
 #define min(a,b)                                \
     ({ __typeof__ (a) _a = (a);                 \
         __typeof__ (b) _b = (b);                \
         _a < _b ? _a : _b; })
 
+/** Block size of gathers. */
 #define MAX_GATHER_BLOCK_SIZE 0
+
+/** Request allocation size. */
 #define PIO_REQUEST_ALLOC_CHUNK 16
 
 /** This is needed to handle _long() functions. It may not be used as
@@ -83,38 +94,32 @@ extern "C" {
     /** Used to sort map points in the subset rearranger. */
     typedef struct mapsort
     {
-        int rfrom;
-        PIO_Offset soffset;
-        PIO_Offset iomap;
+        int rfrom; /**< from */
+        PIO_Offset soffset; /**< ??? */
+        PIO_Offset iomap; /**< ??? */
     } mapsort;
 
     /** swapm defaults. */
     typedef struct pio_swapm_defaults
     {
-        int nreqs;
-        bool handshake;
-        bool isend;
+        int nreqs; /**< number of requests */
+        bool handshake; /**< handshake */
+        bool isend; /**< is end? */
     } pio_swapm_defaults;
 
     /* Handle an error in the PIO library. */
     int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
                 int line);
 
-    /* Check return from MPI function and print error message. */
-    void CheckMPIReturn(int ierr, const char *file, int line);
-
     /* Print error message and abort. */
     void piodie(const char *msg, const char *fname, int line);
 
     /* Assert that an expression is true. */
-    void pioassert(bool exp, const char *msg, const char *fname, int line);
-
-    /* Check the return code from an MPI function call. */
-    int check_mpi(file_desc_t *file, int mpierr, const char *filename, int line);
+    void pioassert(_Bool expression, const char *msg, const char *fname, int line);
 
     /* Check the return code from an MPI function call. */
-    int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr, const char *filename,
-                   int line);
+    int check_mpi(iosystem_desc_t *ios, file_desc_t *file, int mpierr, const char *filename,
+                  int line);
 
     /* Check the return code from a netCDF call. */
     int check_netcdf(file_desc_t *file, int status, const char *fname, int line);
@@ -145,7 +150,7 @@ extern "C" {
     int pio_get_file(int ncid, file_desc_t **filep);
     int pio_delete_file_from_list(int ncid);
     void pio_add_to_file_list(file_desc_t *file);
-    
+
     /* List operations for var_desc_t list. */
     int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size,
                        MPI_Datatype mpi_type, int mpi_type_size, var_desc_t **varlist);
@@ -219,16 +224,6 @@ extern "C" {
     /* Free a region list. */
     void free_region_list(io_region *top);
 
-    /* Compare sets of rearranger options. */
-    bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts);
-
-    /* Check and reset, if needed, rearranger opts to default values. */
-    int check_and_reset_rearr_opts(rearr_opt_t *rearr_opt);
-
-    /* Compare rearranger flow control options. */
-    bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt,
-                                const rearr_comm_fc_opt_t *exp_opt);
-
     /* Create a subset rearranger. */
     int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, const int *gsize,
                                 int ndim, io_desc_t *iodesc);
@@ -249,6 +244,8 @@ extern "C" {
     /* Flush contents of multi-buffer to disk. */
     int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize);
 
+    int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc);
+
     /* Compute the size that the IO tasks will need to hold the data. */
     int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc);
 
@@ -258,9 +255,6 @@ extern "C" {
     /* Find greatest commond divisor for long long. */
     long long lgcd (long long a, long long b );
 
-    /* Find greatest commond divisor in an array. */
-    int gcd_array(int nain, int *ain);
-
     /* Convert a global coordinate value into a local array index. */
     PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count);
 
@@ -312,7 +306,7 @@ extern "C" {
 
     int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
     int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
-    int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc);    
+    int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc);
 
     /* Read atts with type conversion. */
     int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip);
@@ -327,6 +321,8 @@ extern "C" {
     int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
                          void *buf);
     int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf);
+    int PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                         nc_type xtype, void *buf);
 
     /* Generalized put functions. */
     int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
@@ -334,6 +330,8 @@ extern "C" {
     int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
                          const void *op);
     int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op);
+    int PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                         nc_type xtype, const void *buf);
 
     /* An internal replacement for a function pnetcdf does not
      * have. */
@@ -344,7 +342,7 @@ extern "C" {
     int pioc_change_def(int ncid, int is_enddef);
 
     /* Initialize and finalize logging. */
-    void pio_init_logging(void);
+    int pio_init_logging(void);
     void pio_finalize_logging(void );
 
     /* Write a netCDF decomp file. */
@@ -361,6 +359,17 @@ extern "C" {
     int determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp,
                         int **proc_list, int **my_proc_list);
 
+    int pio_sorted_copy(const void *array, void *tmparray, io_desc_t *iodesc, int nvars, int direction);
+
+    int PIOc_inq_att_eh(int ncid, int varid, const char *name, int eh,
+                        nc_type *xtypep, PIO_Offset *lenp);
+
+    /* Start a timer. */
+    int pio_start_timer(const char *name);
+
+    /* Stop a timer. */
+    int pio_stop_timer(const char *name);
+
 #if defined(__cplusplus)
 }
 #endif
@@ -369,6 +378,7 @@ extern "C" {
  * async is being used. */
 enum PIO_MSG
 {
+    PIO_MSG_NULL,
     PIO_MSG_OPEN_FILE,
     PIO_MSG_CREATE_FILE,
     PIO_MSG_INQ_ATT,
diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c
index c4edb5d5042e..2d236d5e57bc 100644
--- a/src/clib/pio_lists.c
+++ b/src/clib/pio_lists.c
@@ -7,6 +7,7 @@
 #include 
 #include 
 #include 
+#include 
 
 static io_desc_t *pio_iodesc_list = NULL;
 static io_desc_t *current_iodesc = NULL;
@@ -20,31 +21,16 @@ static file_desc_t *current_file = NULL;
  * @param file pointer to the file_desc_t struct for the new file.
  * @author Jim Edwards
  */
-void pio_add_to_file_list(file_desc_t *file)
+void
+pio_add_to_file_list(file_desc_t *file)
 {
-    file_desc_t *cfile;
-
     assert(file);
 
-    /* This file will be at the end of the list, and have no next. */
-    file->next = NULL;
-
-    /* Get a pointer to the global list of files. */
-    cfile = pio_file_list;
-
     /* Keep a global pointer to the current file. */
     current_file = file;
 
-    /* If there is nothing in the list, then file will be the first
-     * entry. Otherwise, move to end of the list. */
-    if (!cfile)
-        pio_file_list = file;
-    else
-    {
-        while (cfile->next)
-            cfile = cfile->next;
-        cfile->next = file;
-    }
+    /* Add file to list. */
+    HASH_ADD_INT(pio_file_list, pio_ncid, file);
 }
 
 /**
@@ -58,7 +44,8 @@ void pio_add_to_file_list(file_desc_t *file)
  * @returns 0 for success, error code otherwise.
  * @author Ed Hartnett
  */
-int pio_get_file(int ncid, file_desc_t **cfile1)
+int
+pio_get_file(int ncid, file_desc_t **cfile1)
 {
     file_desc_t *cfile = NULL;
 
@@ -72,17 +59,14 @@ int pio_get_file(int ncid, file_desc_t **cfile1)
     if (current_file && current_file->pio_ncid == ncid)
         cfile = current_file;
     else
-        for (cfile = pio_file_list; cfile; cfile = cfile->next)
-            if (cfile->pio_ncid == ncid)
-            {
-                current_file = cfile;
-                break;
-            }
+        HASH_FIND_INT(pio_file_list, &ncid, cfile);
 
     /* If not found, return error. */
     if (!cfile)
         return PIO_EBADID;
 
+    current_file = cfile;
+
     /* We depend on every file having a pointer to the iosystem. */
     if (!cfile->iosystem)
         return PIO_EINVAL;
@@ -103,35 +87,34 @@ int pio_get_file(int ncid, file_desc_t **cfile1)
  * @returns 0 for success, error code otherwise
  * @author Jim Edwards, Ed Hartnett
  */
-int pio_delete_file_from_list(int ncid)
+int
+pio_delete_file_from_list(int ncid)
 {
-    file_desc_t *cfile, *pfile = NULL;
+    file_desc_t *cfile = NULL;
     int ret;
 
-    /* Look through list of open files. */
-    for (cfile = pio_file_list; cfile; cfile = cfile->next)
+    /* Find the file pointer. */
+    if (current_file && current_file->pio_ncid == ncid)
+        cfile = current_file;
+    else
+        HASH_FIND_INT(pio_file_list, &ncid, cfile);
+
+    if (cfile)
     {
-        if (cfile->pio_ncid == ncid)
-        {
-            if (!pfile)
-                pio_file_list = cfile->next;
-            else
-                pfile->next = cfile->next;
+        HASH_DEL(pio_file_list, cfile);
 
-            if (current_file == cfile)
-                current_file = pfile;
+        if (current_file == cfile)
+            current_file = pio_file_list;
 
-            /* Free the varlist entries for this file. */
-            while (cfile->varlist)
-                if ((ret = delete_var_desc(cfile->varlist->varid, &cfile->varlist)))
-                    return pio_err(NULL, cfile, ret, __FILE__, __LINE__);
+        /* Free the varlist entries for this file. */
+        while (cfile->varlist)
+            if ((ret = delete_var_desc(cfile->varlist->varid, &cfile->varlist)))
+                return pio_err(NULL, cfile, ret, __FILE__, __LINE__);
 
-            /* Free the memory used for this file. */
-            free(cfile);
+        /* Free the memory used for this file. */
+        free(cfile);
 
-            return PIO_NOERR;
-        }
-        pfile = cfile;
+        return PIO_NOERR;
     }
 
     /* No file was found. */
@@ -145,7 +128,8 @@ int pio_delete_file_from_list(int ncid)
  * @returns 0 on success, error code otherwise
  * @author Jim Edwards
  */
-int pio_delete_iosystem_from_list(int piosysid)
+int
+pio_delete_iosystem_from_list(int piosysid)
 {
     iosystem_desc_t *ciosystem, *piosystem = NULL;
 
@@ -175,7 +159,8 @@ int pio_delete_iosystem_from_list(int piosysid)
  * @returns 0 on success, error code otherwise
  * @author Jim Edwards
  */
-int pio_add_to_iosystem_list(iosystem_desc_t *ios)
+int
+pio_add_to_iosystem_list(iosystem_desc_t *ios)
 {
     iosystem_desc_t *cios;
     int i = 1;
@@ -209,7 +194,8 @@ int pio_add_to_iosystem_list(iosystem_desc_t *ios)
  * @returns pointer to iosystem_desc_t, or NULL if not found.
  * @author Jim Edwards
  */
-iosystem_desc_t *pio_get_iosystem_from_id(int iosysid)
+iosystem_desc_t *
+pio_get_iosystem_from_id(int iosysid)
 {
     iosystem_desc_t *ciosystem;
 
@@ -229,7 +215,8 @@ iosystem_desc_t *pio_get_iosystem_from_id(int iosysid)
  * @returns 0 for success.
  * @author Jim Edwards
  */
-int pio_num_iosystem(int *niosysid)
+int
+pio_num_iosystem(int *niosysid)
 {
     int count = 0;
 
@@ -250,25 +237,15 @@ int pio_num_iosystem(int *niosysid)
 /**
  * Add an iodesc.
  *
- * @param io_desc_t pointer to data to add to list.
+ * @param iodesc io_desc_t pointer to data to add to list.
  * @returns 0 for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int pio_add_to_iodesc_list(io_desc_t *iodesc)
+int
+pio_add_to_iodesc_list(io_desc_t *iodesc)
 {
-    io_desc_t *ciodesc;
-
-    iodesc->next = NULL;
-    if (pio_iodesc_list == NULL)
-        pio_iodesc_list = iodesc;
-    else
-    {
-        for (ciodesc = pio_iodesc_list; ciodesc->next; ciodesc = ciodesc->next)
-            ;
-        ciodesc->next = iodesc;
-    }
+    HASH_ADD_INT(pio_iodesc_list, ioid, iodesc);
     current_iodesc = iodesc;
-
     return PIO_NOERR;
 }
 
@@ -279,22 +256,17 @@ int pio_add_to_iodesc_list(io_desc_t *iodesc)
  * @returns pointer to the iodesc struc.
  * @author Jim Edwards
  */
-io_desc_t *pio_get_iodesc_from_id(int ioid)
+io_desc_t *
+pio_get_iodesc_from_id(int ioid)
 {
-    io_desc_t *ciodesc = NULL;
-
-    /* Do we already have a pointer to it? */
+    io_desc_t *ciodesc=NULL;
     if (current_iodesc && current_iodesc->ioid == ioid)
-        return current_iodesc;
-
-    /* Find the decomposition in the list. */
-    for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
-        if (ciodesc->ioid == ioid)
-        {
-            current_iodesc = ciodesc;
-            break;
-        }
-
+        ciodesc = current_iodesc;
+    else
+    {
+        HASH_FIND_INT(pio_iodesc_list, &ioid, ciodesc);
+        current_iodesc = ciodesc;
+    }
     return ciodesc;
 }
 
@@ -305,25 +277,19 @@ io_desc_t *pio_get_iodesc_from_id(int ioid)
  * @returns 0 on success, error code otherwise.
  * @author Jim Edwards
  */
-int pio_delete_iodesc_from_list(int ioid)
+int
+pio_delete_iodesc_from_list(int ioid)
 {
-    io_desc_t *ciodesc, *piodesc = NULL;
+    io_desc_t *ciodesc;
 
-    for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
+    ciodesc = pio_get_iodesc_from_id(ioid);
+    if (ciodesc)
     {
-        if (ciodesc->ioid == ioid)
-        {
-            if (piodesc == NULL)
-                pio_iodesc_list = ciodesc->next;
-            else
-                piodesc->next = ciodesc->next;
-
-            if (current_iodesc == ciodesc)
-                current_iodesc = pio_iodesc_list;
-            free(ciodesc);
-            return PIO_NOERR;
-        }
-        piodesc = ciodesc;
+        HASH_DEL(pio_iodesc_list, ciodesc);
+        if (current_iodesc == ciodesc)
+            current_iodesc = pio_iodesc_list;
+        free(ciodesc);
+        return PIO_NOERR;
     }
     return PIO_EBADID;
 }
@@ -333,12 +299,17 @@ int pio_delete_iodesc_from_list(int ioid)
  *
  * @param varid the varid of the variable.
  * @param rec_var non-zero if this is a record var.
+ * @param pio_type the PIO type.
+ * @param pio_type_size size of the PIO type in bytes
+ * @param mpi_type the MPI type.
+ * @param mpi_type_size size of the MPI type in bytes.
  * @param varlist pointer to list to add to.
  * @returns 0 for success, error code otherwise.
  * @author Ed Hartnett
  */
-int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, MPI_Datatype mpi_type,
-                   int mpi_type_size, var_desc_t **varlist)
+int
+add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size,
+               MPI_Datatype mpi_type, int mpi_type_size, var_desc_t **varlist)
 {
     var_desc_t *var_desc;
 
@@ -358,18 +329,7 @@ int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, MPI_
     var_desc->mpi_type_size = mpi_type_size;
     var_desc->record = -1;
 
-    /* Add to list. */
-    if (*varlist)
-    {
-        var_desc_t *v;
-
-        /* Move to end of list. */
-        for (v = *varlist; v->next; v = v->next)
-            ;
-        v->next = var_desc;
-    }
-    else
-        *varlist = var_desc;
+    HASH_ADD_INT(*varlist, varid, var_desc);
 
     return PIO_NOERR;
 }
@@ -383,21 +343,19 @@ int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, MPI_
  * @returns 0 for success, error code otherwise.
  * @author Ed Hartnett
  */
-int get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc)
+int
+get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc)
 {
-    var_desc_t *my_var;
+    var_desc_t *my_var=NULL;
 
     /* Check inputs. */
-    pioassert(varlist, "invalid input", __FILE__, __LINE__);    
+    pioassert(varlist, "invalid input", __FILE__, __LINE__);
 
     /* Empty varlist. */
     if (!*varlist)
         return PIO_ENOTVAR;
 
-    /* Find the var_desc_t for this varid. */
-    for (my_var = *varlist; my_var; my_var = my_var->next)
-        if (my_var->varid == varid)
-            break;
+    HASH_FIND_INT( *varlist, &varid, my_var);
 
     /* Did we find it? */
     if (!my_var)
@@ -416,10 +374,11 @@ int get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc)
  * @returns 0 on success, error code otherwise.
  * @author Ed Hartnett
  */
-int delete_var_desc(int varid, var_desc_t **varlist)
+int
+delete_var_desc(int varid, var_desc_t **varlist)
 {
     var_desc_t *v;
-    var_desc_t *prev = NULL;
+    int ret;
 
     /* Check inputs. */
     pioassert(varid >= 0 && varlist, "invalid input", __FILE__, __LINE__);
@@ -428,27 +387,10 @@ int delete_var_desc(int varid, var_desc_t **varlist)
     if (!*varlist)
         return PIO_ENOTVAR;
 
-    /* Find the var_desc_t for this varid. */
-    for (v = *varlist; v->next; v = v->next)
-    {
-        LOG((3, "v->varid = %d", v->varid));
-        if (v->varid == varid)
-            break;
-        prev = v;
-    }
-
-    /* Did we find it? */
-    if (v->varid != varid)
-    {
-        LOG((3, "return notvar error"));
-        return PIO_ENOTVAR;
-    }
+    if ((ret = get_var_desc( varid, varlist, &v)))
+        return ret;
 
-    /* Adjust next pointer. */
-    if (prev)
-        prev->next = v->next;
-    else
-        *varlist = v->next;
+    HASH_DEL(*varlist, v);
 
     /* Free memory. */
     if (v->fillvalue)
diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
index eb5e09d9827c..7e4b5e7dc569 100644
--- a/src/clib/pio_msg.c
+++ b/src/clib/pio_msg.c
@@ -47,13 +47,13 @@ int inq_type_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Handle null pointer issues. */
     if (name_present)
@@ -91,9 +91,9 @@ int inq_format_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "inq_format_handler got parameters ncid = %d format_present = %d",
          ncid, format_present));
 
@@ -131,11 +131,11 @@ int set_fill_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "set_fill_handler got parameters ncid = %d fillmode = %d old_modep_present = %d",
          ncid, fillmode, old_modep_present));
 
@@ -174,17 +174,17 @@ int create_file_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Get space for the filename. */
     char filename[len + 1];
 
     if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "create_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
          len, filename, iotype, mode));
 
@@ -217,7 +217,7 @@ int close_file_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "close_file_handler got parameter ncid = %d", ncid));
 
     /* Call the close file function. */
@@ -251,15 +251,15 @@ int inq_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d",
          ndims_present, nvars_present, ngatts_present, unlimdimid_present));
 
@@ -306,11 +306,11 @@ int inq_unlimdims_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nunlimdimsp_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&unlimdimidsp_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "inq_unlimdims_handler nunlimdimsp_present = %d unlimdimidsp_present = %d",
          nunlimdimsp_present, unlimdimidsp_present));
 
@@ -355,13 +355,13 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "inq_handler name_present = %d len_present = %d", name_present,
          len_present));
 
@@ -401,13 +401,13 @@ int inq_dimid_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d",
          ncid, namelen, name, id_present));
 
@@ -425,7 +425,6 @@ int inq_dimid_handler(iosystem_desc_t *ios)
  * tasks.
  *
  * @param ios pointer to the iosystem_desc_t.
- * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -448,18 +447,18 @@ int inq_att_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster,
                             ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Match NULLs in collective function call. */
     if (xtype_present)
@@ -477,7 +476,6 @@ int inq_att_handler(iosystem_desc_t *ios)
  * tasks.
  *
  * @param ios pointer to the iosystem_desc_t.
- * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -498,13 +496,13 @@ int inq_attname_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "inq_attname_handler got ncid = %d varid = %d attnum = %d name_present = %d",
          ncid, varid, attnum, name_present));
 
@@ -522,7 +520,6 @@ int inq_attname_handler(iosystem_desc_t *ios)
  * tasks.
  *
  * @param ios pointer to the iosystem_desc_t.
- * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -544,15 +541,15 @@ int inq_attid_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "inq_attid_handler got ncid = %d varid = %d id_present = %d",
          ncid, varid, id_present));
 
@@ -569,7 +566,6 @@ int inq_attid_handler(iosystem_desc_t *ios)
 /** Handle attribute operations. This code only runs on IO tasks.
  *
  * @param ios pointer to the iosystem_desc_t.
- * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -595,23 +591,23 @@ int att_put_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Allocate memory for the attribute data. */
     if (!(op = malloc(attlen * memtype_len)))
@@ -619,7 +615,7 @@ int att_put_handler(iosystem_desc_t *ios)
     if ((mpierr = MPI_Bcast(op, attlen * memtype_len, MPI_BYTE, 0, ios->intercomm)))
     {
         free(op);
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     LOG((1, "att_put_handler ncid = %d varid = %d namelen = %d name = %s"
          "atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = 5d",
@@ -638,7 +634,6 @@ int att_put_handler(iosystem_desc_t *ios)
 /** Handle attribute operations. This code only runs on IO tasks.
  *
  * @param ios pointer to the iosystem_desc_t.
- * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -665,25 +660,25 @@ int att_get_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d"
          " atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = %d",
          ncid, varid, namelen, name, iotype, atttype, attlen, atttype_len, memtype, memtype_len));
@@ -732,37 +727,37 @@ int put_vars_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Now we know how big to make these arrays. */
     PIO_Offset start[ndims], count[ndims], stride[ndims];
 
     if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (start_present)
         if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims));
     if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (count_present)
         if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (stride_present)
         if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d "
          "start_present = %d count_present = %d stride_present = %d xtype = %d "
          "num_elem = %d typelen = %d", ncid, varid, ndims, start_present, count_present,
@@ -774,7 +769,7 @@ int put_vars_handler(iosystem_desc_t *ios)
 
     /* Get the data. */
     if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Set the non-NULL pointers. */
     if (start_present)
@@ -873,44 +868,44 @@ int get_vars_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (start_present)
     {
         if (!(start = malloc(ndims * sizeof(PIO_Offset))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (count_present)
     {
         if (!(count = malloc(ndims * sizeof(PIO_Offset))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (stride_present)
     {
         if (!(stride = malloc(ndims * sizeof(PIO_Offset))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "get_vars_handler ncid = %d varid = %d ndims = %d "
          "stride_present = %d xtype = %d num_elem = %d typelen = %d",
          ncid, varid, ndims, stride_present, xtype, num_elem, typelen));
@@ -1018,19 +1013,19 @@ int inq_var_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2,"inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d "
          "dimids_present = %d natts_present = %d",
          ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present));
@@ -1079,15 +1074,15 @@ int inq_var_chunking_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2,"inq_var_handler ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
          ncid, varid, storage_present, chunksizes_present));
 
@@ -1121,7 +1116,7 @@ int inq_var_fill_handler(iosystem_desc_t *ios)
     char fill_mode_present, fill_value_present;
     PIO_Offset type_size;
     int fill_mode, *fill_modep = NULL;
-    PIO_Offset *fill_value, *fill_valuep = NULL;
+    void *fill_value, *fill_valuep = NULL;
     int mpierr;
 
     assert(ios);
@@ -1130,15 +1125,15 @@ int inq_var_fill_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2,"inq_var_fill_handler ncid = %d varid = %d type_size = %lld, fill_mode_present = %d fill_value_present = %d",
          ncid, varid, type_size, fill_mode_present, fill_value_present));
 
@@ -1154,12 +1149,21 @@ int inq_var_fill_handler(iosystem_desc_t *ios)
         fill_valuep = fill_value;
 
     /* Call the inq function to get the values. */
+    LOG((3, "inq_var_fill_handlder about to call inq_var_fill"));
     PIOc_inq_var_fill(ncid, varid, fill_modep, fill_valuep);
+    if (fill_modep)
+        LOG((3, "after inq_var_fill fill_modep %d", *fill_modep));
 
     /* Free fill value storage if we allocated some. */
     if (fill_value_present)
+    {
+        LOG((3, "about to free fill_value"));
         free(fill_value);
+        LOG((3, "freed fill_value"));
+    }
 
+    if (fill_modep)
+        LOG((3, "done with inq_var_fill_handler", *fill_modep));
     return PIO_NOERR;
 }
 
@@ -1184,11 +1188,11 @@ int inq_var_endian_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2,"inq_var_endian_handler ncid = %d varid = %d endian_present = %d", ncid, varid,
          endian_present));
 
@@ -1216,9 +1220,9 @@ int inq_var_deflate_handler(iosystem_desc_t *ios)
     char shuffle_present;
     char deflate_present;
     char deflate_level_present;
-    int shuffle, *shufflep;
-    int deflate, *deflatep;
-    int deflate_level, *deflate_levelp;
+    int shuffle, *shufflep = NULL;
+    int deflate, *deflatep = NULL;
+    int deflate_level, *deflate_levelp = NULL;
     int mpierr;
 
     assert(ios);
@@ -1227,24 +1231,24 @@ int inq_var_deflate_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (shuffle_present && !mpierr)
         if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (deflate_present && !mpierr)
         if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (deflate_level_present && !mpierr)
         if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "inq_var_handler ncid = %d varid = %d shuffle_present = %d deflate_present = %d "
          "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present,
          deflate_level_present));
@@ -1285,11 +1289,11 @@ int inq_varid_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Call the inq_dimid function. */
     PIOc_inq_varid(ncid, name, &varid);
@@ -1317,7 +1321,7 @@ int sync_file_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "sync_file_handler got parameter ncid = %d", ncid));
 
     /* Call the sync file function. */
@@ -1350,11 +1354,11 @@ int setframe_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&frame, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "setframe_handler got parameter ncid = %d varid = %d frame = %d",
          ncid, varid, frame));
 
@@ -1387,9 +1391,9 @@ int advanceframe_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "advanceframe_handler got parameter ncid = %d varid = %d",
          ncid, varid));
 
@@ -1404,6 +1408,7 @@ int advanceframe_handler(iosystem_desc_t *ios)
  * This function is run on the IO tasks to enddef a netCDF file.
  *
  * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
  * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
  * from netCDF base function.
  * @internal
@@ -1420,7 +1425,7 @@ int change_def_file_handler(iosystem_desc_t *ios, int msg)
     /* Get the parameters for this function that the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Call the function. */
     if (msg == PIO_MSG_ENDDEF)
@@ -1459,21 +1464,21 @@ int def_var_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (!(dimids = malloc(ndims * sizeof(int))))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(dimids, ndims, MPI_INT, 0, ios->intercomm)))
     {
         free(dimids);
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     LOG((1, "def_var_handler got parameters namelen = %d "
          "name = %s ncid = %d", namelen, name, ncid));
@@ -1511,20 +1516,20 @@ int def_var_chunking_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&storage, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (chunksizes_present){
         if (!(chunksizesp = malloc(ndims* sizeof(PIO_Offset))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     LOG((1, "def_var_chunking_handler got parameters ncid = %d varid = %d storage = %d "
          "ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present));
@@ -1562,15 +1567,15 @@ int def_var_fill_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (fill_value_present)
     {
         if (!(fill_valuep = malloc(type_size)))
@@ -1578,7 +1583,7 @@ int def_var_fill_handler(iosystem_desc_t *ios)
         if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, 0, ios->intercomm)))
         {
             free(fill_valuep);
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         }
     }
     LOG((1, "def_var_fill_handler got parameters ncid = %d varid = %d fill_mode = %d "
@@ -1615,11 +1620,11 @@ int def_var_endian_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&endian, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "def_var_endian_handler got parameters ncid = %d varid = %d endain = %d ",
          ncid, varid, endian));
 
@@ -1652,15 +1657,15 @@ int def_var_deflate_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "def_var_deflate_handler got parameters ncid = %d varid = %d shuffle = %d ",
          "deflate = %d deflate_level = %d", ncid, varid, shuffle, deflate, deflate_level));
 
@@ -1693,15 +1698,15 @@ int set_var_chunk_cache_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "set_var_chunk_cache_handler got params ncid = %d varid = %d size = %d "
          "nelems = %d preemption = %g", ncid, varid, size, nelems, preemption));
 
@@ -1736,13 +1741,13 @@ int def_dim_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "def_dim_handler got parameters namelen = %d "
          "name = %s len = %d ncid = %d", namelen, name, len, ncid));
 
@@ -1777,13 +1782,13 @@ int rename_dim_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "rename_dim_handler got parameters namelen = %d "
          "name = %s ncid = %d dimid = %d", namelen, name, ncid, dimid));
 
@@ -1818,13 +1823,13 @@ int rename_var_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "rename_var_handler got parameters namelen = %d "
          "name = %s ncid = %d varid = %d", namelen, name, ncid, varid));
 
@@ -1859,17 +1864,17 @@ int rename_att_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "rename_att_handler got parameters namelen = %d name = %s ncid = %d varid = %d "
          "newnamelen = %d newname = %s", namelen, name, ncid, varid, newnamelen, newname));
 
@@ -1904,13 +1909,13 @@ int delete_att_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "delete_att_handler namelen = %d name = %s ncid = %d varid = %d ",
          namelen, name, ncid, varid));
 
@@ -1945,18 +1950,18 @@ int open_file_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "open_file_handler got parameter len = %d", len));
 
     /* Get space for the filename. */
     char filename[len + 1];
 
     if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     LOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
          len, filename, iotype, mode));
@@ -1989,13 +1994,13 @@ int delete_file_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Get space for the filename. */
     char filename[len + 1];
 
     if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "delete_file_handler got parameters len = %d filename = %s",
          len, filename));
 
@@ -2028,7 +2033,6 @@ int initdecomp_dof_handler(iosystem_desc_t *ios)
     char iocount_present;
     PIO_Offset *iocountp = NULL;
     int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
-    int ret; /* Return code. */
 
     LOG((1, "initdecomp_dof_handler called"));
     assert(ios);
@@ -2036,11 +2040,11 @@ int initdecomp_dof_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Now we know the size of these arrays. */
     int dims[ndims];
@@ -2048,35 +2052,37 @@ int initdecomp_dof_handler(iosystem_desc_t *ios)
     PIO_Offset iocount[ndims];
 
     if ((mpierr = MPI_Bcast(dims, ndims, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&maplen, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
-    PIO_Offset compmap[maplen];
+    PIO_Offset *compmap;
+    if (!(compmap = malloc(maplen * sizeof(PIO_Offset))))
+        return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Bcast(compmap, maplen, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if (rearranger_present)
         if ((mpierr = MPI_Bcast(&rearranger, 1, MPI_INT, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if (iostart_present)
         if ((mpierr = MPI_Bcast(iostart, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     if (iocount_present)
         if ((mpierr = MPI_Bcast(iocount, ndims, MPI_OFFSET, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     LOG((2, "initdecomp_dof_handler iosysid = %d pio_type = %d ndims = %d maplen = %d "
          "rearranger_present = %d iostart_present = %d iocount_present = %d ",
@@ -2090,10 +2096,12 @@ int initdecomp_dof_handler(iosystem_desc_t *ios)
         iocountp = iocount;
 
     /* Call the function. */
-    ret = PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp,
-                          iostartp, iocountp);
+    PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp,
+                    iostartp, iocountp);
 
-    LOG((1, "PIOc_InitDecomp returned %d", ret));
+    LOG((1, "PIOc_InitDecomp returned"));
+
+    free(compmap);
     return PIO_NOERR;
 }
 
@@ -2132,46 +2140,46 @@ int write_darray_multi_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nvars, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     int varids[nvars];
     if ((mpierr = MPI_Bcast(varids, nvars, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ioid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Get decomposition information. */
     if (!(iodesc = pio_get_iodesc_from_id(ioid)))
-        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (!(array = malloc(arraylen * iodesc->piotype_size)))
         return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(array, arraylen * iodesc->piotype_size, MPI_CHAR, 0,
                             ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&frame_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (frame_present)
     {
         if (!(frame = malloc(nvars * sizeof(int))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(frame, nvars, MPI_INT, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     if ((mpierr = MPI_Bcast(&fillvalue_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (fillvalue_present)
     {
         if (!(fillvalue = malloc(nvars * iodesc->piotype_size)))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(fillvalue, nvars * iodesc->piotype_size, MPI_CHAR, 0, ios->intercomm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     if ((mpierr = MPI_Bcast(&flushtodisk, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "write_darray_multi_handler ncid = %d nvars = %d ioid = %d arraylen = %d "
          "frame_present = %d fillvalue_present flushtodisk = %d", ncid, nvars,
          ioid, arraylen, frame_present, fillvalue_present, flushtodisk));
@@ -2249,9 +2257,9 @@ int seterrorhandling_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the he comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&method, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     LOG((1, "seterrorhandling_handler got parameters method = %d old_method_present = %d",
          method, old_method_present));
@@ -2290,15 +2298,15 @@ int set_chunk_cache_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "set_chunk_cache_handler got params iosysid = %d iotype = %d size = %d "
          "nelems = %d preemption = %g", iosysid, iotype, size, nelems, preemption));
 
@@ -2323,9 +2331,9 @@ int get_chunk_cache_handler(iosystem_desc_t *ios)
     int iosysid;
     int iotype;
     char size_present, nelems_present, preemption_present;
-    PIO_Offset size, *sizep;
-    PIO_Offset nelems, *nelemsp;
-    float preemption, *preemptionp;
+    PIO_Offset size, *sizep = NULL;
+    PIO_Offset nelems, *nelemsp = NULL;
+    float preemption, *preemptionp = NULL;
     int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
 
     LOG((1, "get_chunk_cache_handler called"));
@@ -2334,15 +2342,15 @@ int get_chunk_cache_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "get_chunk_cache_handler got params iosysid = %d iotype = %d size_present = %d "
          "nelems_present = %d preemption_present = %g", iosysid, iotype, size_present,
          nelems_present, preemption_present));
@@ -2376,9 +2384,9 @@ int get_var_chunk_cache_handler(iosystem_desc_t *ios)
     int ncid;
     int varid;
     char size_present, nelems_present, preemption_present;
-    PIO_Offset size, *sizep;
-    PIO_Offset nelems, *nelemsp;
-    float preemption, *preemptionp;
+    PIO_Offset size, *sizep = NULL;
+    PIO_Offset nelems, *nelemsp = NULL;
+    float preemption, *preemptionp = NULL;
     int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
 
     LOG((1, "get_var_chunk_cache_handler called"));
@@ -2387,15 +2395,15 @@ int get_var_chunk_cache_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "get_var_chunk_cache_handler got params ncid = %d varid = %d size_present = %d "
          "nelems_present = %d preemption_present = %g", ncid, varid, size_present,
          nelems_present, preemption_present));
@@ -2428,7 +2436,6 @@ int freedecomp_handler(iosystem_desc_t *ios)
     int iosysid;
     int ioid;
     int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
-    int ret; /* Return code. */
 
     LOG((1, "freedecomp_handler called"));
     assert(ios);
@@ -2436,15 +2443,15 @@ int freedecomp_handler(iosystem_desc_t *ios)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Bcast(&ioid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "freedecomp_handler iosysid = %d ioid = %d", iosysid, ioid));
 
     /* Call the function. */
-    ret = PIOc_freedecomp(iosysid, ioid);
+    PIOc_freedecomp(iosysid, ioid);
 
-    LOG((1, "PIOc_freedecomp returned %d", ret));
+    LOG((1, "PIOc_freedecomp returned"));
     return PIO_NOERR;
 }
 
@@ -2469,7 +2476,7 @@ int finalize_handler(iosystem_desc_t *ios, int index)
     /* Get the parameters for this function that the the comp master
      * task is broadcasting. */
     if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((1, "finalize_handler got parameter iosysid = %d", iosysid));
 
     /* Call the function. */
@@ -2494,7 +2501,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
                      MPI_Comm io_comm)
 {
     iosystem_desc_t *my_iosys;
-    int msg = 0;
+    int msg = PIO_MSG_NULL, messages[component_count];
     MPI_Request req[component_count];
     MPI_Status status;
     int index;
@@ -2514,9 +2521,9 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
         {
             my_iosys = iosys[cmp];
             LOG((1, "about to call MPI_Irecv union_comm = %d", my_iosys->union_comm));
-            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG,
+            if ((mpierr = MPI_Irecv(&(messages[cmp]), 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG,
                                     my_iosys->union_comm, &req[cmp])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             LOG((1, "MPI_Irecv req[%d] = %d", cmp, req[cmp]));
         }
     }
@@ -2536,8 +2543,9 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
             for (int c = 0; c < component_count; c++)
                 LOG((3, "req[%d] = %d", c, req[c]));
             if ((mpierr = MPI_Waitany(component_count, req, &index, &status)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             LOG((3, "Waitany returned index = %d req[%d] = %d", index, index, req[index]));
+	    msg = messages[index];
             for (int c = 0; c < component_count; c++)
                 LOG((3, "req[%d] = %d", c, req[c]));
         }
@@ -2546,7 +2554,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
          * originated the request to the rest of the IO tasks. */
         LOG((3, "About to do Bcast of index = %d io_comm = %d", index, io_comm));
         if ((mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, io_comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "index MPI_Bcast complete index = %d", index));
 
         /* Set the correct iosys depending on the index. */
@@ -2555,7 +2563,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
         /* Broadcast the msg value to the rest of the IO tasks. */
         LOG((3, "about to call msg MPI_Bcast io_comm = %d", io_comm));
         if ((mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, io_comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         LOG((1, "pio_msg_handler2 msg MPI_Bcast complete msg = %d", msg));
 
         /* Handle the message. This code is run on all IO tasks. */
@@ -2724,15 +2732,15 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
             my_iosys = iosys[index];
             LOG((3, "pio_msg_handler2 about to Irecv index = %d comproot = %d union_comm = %d",
                  index, my_iosys->comproot, my_iosys->union_comm));
-            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm,
+            if ((mpierr = MPI_Irecv(&(messages[index]), 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm,
                                     &req[index])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             LOG((3, "pio_msg_handler2 called MPI_Irecv req[%d] = %d", index, req[index]));
         }
 
         LOG((3, "pio_msg_handler2 done msg = %d open_components = %d",
              msg, open_components));
-
+	msg = PIO_MSG_NULL;
         /* If there are no more open components, exit. */
         if (finalize)
         {
diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c
index a258dc1abb42..7223025e0fca 100644
--- a/src/clib/pio_nc.c
+++ b/src/clib/pio_nc.c
@@ -19,7 +19,54 @@
 #include 
 
 /**
- * @ingroup PIO_inq
+ * @defgroup PIO_inq_c Learn About File
+ * Learn the number of variables, dimensions, and global atts, and the
+ * unlimited dimension in C.
+ *
+ * @defgroup PIO_typelen_c Learn Aboue a Data Type
+ * Learn the length of a data type in C.
+ *
+ * @defgroup PIO_inq_format_c Learn About Binary Format
+ * Learn about the binary format in C.
+ *
+ * @defgroup PIO_inq_dim_c Learn About a Dimension
+ * Learn dimension name and length in C.
+ *
+ * @defgroup PIO_inq_var_c Learn About a Variable
+ * Learn variable name, dimensions, and type in C.
+ *
+ * @defgroup PIO_inq_att_c Learn About an Attribute
+ * Learn length, type, and name of an attribute in C.
+ *
+ * @defgroup PIO_rename_dim_c Rename a Dimension
+ * Rename a dimension in C.
+ *
+ * @defgroup PIO_rename_var_c Rename a Variable
+ * Rename a variable in C.
+ *
+ * @defgroup PIO_rename_att_c Rename an Attribute
+ * Rename an attribute in C.
+ *
+ * @defgroup PIO_del_att_c Delete an Attribute
+ * Delete an attribute in C.
+ *
+ * @defgroup PIO_set_fill_c Set Fill Value
+ * Set the fill value for a variable in C.
+ *
+ * @defgroup PIO_enddef_c End Define Mode
+ * End define mode in C.
+ *
+ * @defgroup PIO_redef_c Re-enter Define Mode
+ * Re-enter Define Mode in C.
+ *
+ * @defgroup PIO_def_dim_c Define a Dimension
+ * Define a new dimension in the file in C.
+ *
+ * @defgroup PIO_def_var_c Define a Variable
+ * Define a new variable in the file in C.
+ */
+
+/**
  * The PIO-C interface for the NetCDF function nc_inq.
  *
  * This routine is called collectively by all tasks in the
@@ -30,12 +77,23 @@
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
+ * @param ndimsp a pointer that will get the number of
+ * dimensions. Ignored if NULL.
+ * @param nvarsp a pointer that will get the number of
+ * variables. Ignored if NULL.
+ * @param ngattsp a pointer that will get the number of
+ * attributes. Ignored if NULL.
+ * @param unlimdimidp a pointer that will the ID of the unlimited
+ * dimension, or -1 if there is no unlimited dimension. Ignored if
+ * NULL.
  *
  * @return PIO_NOERR for success, error code otherwise. See
- * PIOc_Set_File_Error_Handling
+ * PIOc_Set_File_Error_Handling().
+ * @ingroup PIO_inq_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
+int
+PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -79,9 +137,9 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -129,84 +187,89 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if (ndimsp)
         if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (nvarsp)
         if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (ngattsp)
         if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (unlimdimidp)
         if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_ndims
  * Find out how many dimensions are defined in the file.
  *
  * @param ncid the ncid of the open file.
- * @param ndimsp a pointer that will get the number of dimensions.
+ * @param ndimsp a pointer that will get the number of
+ * dimensions. Ignored if NULL.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_ndims(int ncid, int *ndimsp)
+int
+PIOc_inq_ndims(int ncid, int *ndimsp)
 {
     LOG((1, "PIOc_inq_ndims"));
     return PIOc_inq(ncid, ndimsp, NULL, NULL, NULL);
 }
 
 /**
- * @ingroup PIO_inq_nvars
  * Find out how many variables are defined in a file.
  *
  * @param ncid the ncid of the open file.
  * @param nvarsp a pointer that will get the number of variables.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_nvars(int ncid, int *nvarsp)
+int
+PIOc_inq_nvars(int ncid, int *nvarsp)
 {
     return PIOc_inq(ncid, NULL, nvarsp, NULL, NULL);
 }
 
 /**
- * @ingroup PIO_inq_natts
  * Find out how many global attributes are defined in a file.
  *
  * @param ncid the ncid of the open file.
- * @param nattsp a pointer that will get the number of attributes.
+ * @param ngattsp a pointer that will get the number of attributes.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_natts(int ncid, int *ngattsp)
+int
+PIOc_inq_natts(int ncid, int *ngattsp)
 {
     return PIOc_inq(ncid, NULL, NULL, ngattsp, NULL);
 }
 
 /**
- * @ingroup PIO_inq_unlimdim
  * Find out the dimension ids of the unlimited dimension.
  *
  * @param ncid the ncid of the open file.
  * @param unlimdimidp a pointer that will the ID of the unlimited
  * dimension, or -1 if there is no unlimited dimension.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_unlimdim(int ncid, int *unlimdimidp)
+int
+PIOc_inq_unlimdim(int ncid, int *unlimdimidp)
 {
     LOG((1, "PIOc_inq_unlimdim ncid = %d", ncid));
     return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp);
@@ -222,10 +285,11 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp)
  * @param unlimdimidsp a pointer that will get an array of unlimited
  * dimension IDs.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_inq_unlimdim
+ * @ingroup PIO_inq_unlimdim_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
+int
+PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -264,9 +328,9 @@ int PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     LOG((2, "file->iotype = %d", file->iotype));
@@ -331,27 +395,26 @@ int PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if ((mpierr = MPI_Bcast(&tmp_nunlimdims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (nunlimdimsp)
         if ((mpierr = MPI_Bcast(nunlimdimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (unlimdimidsp)
         if ((mpierr = MPI_Bcast(unlimdimidsp, tmp_nunlimdims, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_typelen
  * Learn the name and size of a type.
  *
  * @param ncid the ncid of the open file.
@@ -359,9 +422,11 @@ int PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
  * @param name pointer that will get the name of the type.
  * @param sizep pointer that will get the size of the type in bytes.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_typelen_c
  * @author Ed Hartnett
  */
-int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
+int
+PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -399,9 +464,9 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -419,7 +484,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -430,28 +495,29 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
         if (ios->iomaster == MPI_ROOT)
             slen = strlen(name);
         if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if (!mpierr)
             if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
-                return check_mpi(file, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
     if (sizep)
         if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_format
  * Learn the netCDF format of an open file.
  *
  * @param ncid the ncid of an open file.
  * @param formatp a pointer that will get the format.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_format_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_format(int ncid, int *formatp)
+int
+PIOc_inq_format(int ncid, int *formatp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -484,9 +550,9 @@ int PIOc_inq_format(int ncid, int *formatp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -504,20 +570,19 @@ int PIOc_inq_format(int ncid, int *formatp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if (formatp)
         if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_dim
  * The PIO-C interface for the NetCDF function nc_inq_dim.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -527,11 +592,17 @@ int PIOc_inq_format(int ncid, int *formatp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
+ * @param dimid the dimension ID.
+ * @param name a pointer that gets the name of the dimension. Igorned
+ * if NULL. Name will be PIO_MAX_NAME chars or fewer.
  * @param lenp a pointer that will get the number of values
- * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @return PIO_NOERR for success, error code otherwise.  See
+ * @ingroup PIO_inq_dim_c
+ * PIOc_Set_File_Error_Handling()
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
+int
+PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -571,9 +642,9 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -597,7 +668,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -609,38 +680,38 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
         if (ios->iomaster == MPI_ROOT)
             slen = strlen(name);
         if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (lenp)
         if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     LOG((2, "done with PIOc_inq_dim"));
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_dimname
  * Find the name of a dimension.
  *
  * @param ncid the ncid of an open file.
  * @param dimid the dimension ID.
  * @param name a pointer that gets the name of the dimension. Igorned
- * if NULL.
+ * if NULL. Name will be PIO_MAX_NAME chars or fewer.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_dim_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_dimname(int ncid, int dimid, char *name)
+int
+PIOc_inq_dimname(int ncid, int dimid, char *name)
 {
     LOG((1, "PIOc_inq_dimname ncid = %d dimid = %d", ncid, dimid));
     return PIOc_inq_dim(ncid, dimid, name, NULL);
 }
 
 /**
- * @ingroup PIO_inq_dimlen
  * Find the length of a dimension.
  *
  * @param ncid the ncid of an open file.
@@ -648,15 +719,16 @@ int PIOc_inq_dimname(int ncid, int dimid, char *name)
  * @param lenp a pointer that gets the length of the dimension. Igorned
  * if NULL.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_inq_dim_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp)
+int
+PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp)
 {
     return PIOc_inq_dim(ncid, dimid, NULL, lenp);
 }
 
 /**
- * @ingroup PIO_inq_dimid
  * The PIO-C interface for the NetCDF function nc_inq_dimid.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -666,11 +738,14 @@ int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
+ * @param name pointer taht gets the name of the dimension.
  * @param idp a pointer that will get the id of the variable or attribute.
  * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @ingroup PIO_inq_dim_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_dimid(int ncid, const char *name, int *idp)
+int
+PIOc_inq_dimid(int ncid, const char *name, int *idp)
 {
     iosystem_desc_t *ios;
     file_desc_t *file;
@@ -713,9 +788,9 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* IO tasks call the netCDF functions. */
@@ -733,20 +808,19 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results. */
     if (idp)
         if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_var
  * The PIO-C interface for the NetCDF function nc_inq_var.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -757,13 +831,23 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
- * @param xtypep a pointer that will get the type of the attribute.
- * @param nattsp a pointer that will get the number of attributes
+ * @param name a pointer that gets the name of the dimension. Igorned
+ * if NULL. Name will be PIO_MAX_NAME chars or fewer.
+ * @param xtypep a pointer that will get the type of the
+ * attribute. Ignored if NULL.
+ * @param ndimsp a pointer that will get the number of
+ * dimensions. Ignored if NULL.
+ * @param dimidsp a pointer that will get an array of dimids. Ignored
+ * if NULL.
+ * @param nattsp a pointer that will get the number of
+ * attributes. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
-                 int *dimidsp, int *nattsp)
+int
+PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
+             int *dimidsp, int *nattsp)
 {
     iosystem_desc_t *ios;
     file_desc_t *file;
@@ -814,9 +898,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* Call the netCDF layer. */
@@ -868,7 +952,7 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -879,53 +963,53 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
         if (ios->iomaster == MPI_ROOT)
             slen = strlen(name);
         if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
     if (xtypep)
         if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     if (ndimsp)
     {
         LOG((2, "PIOc_inq_var about to Bcast ndims = %d ios->ioroot = %d ios->my_comm = %d",
              *ndimsp, ios->ioroot, ios->my_comm));
         if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((2, "PIOc_inq_var Bcast ndims = %d", *ndimsp));
     }
     if (dimidsp)
     {
         if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
     if (nattsp)
         if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_varname
  * Get the name of a variable.
  *
  * @param ncid the ncid of the open file.
  * @param varid the variable ID.
  * @param name a pointer that will get the variable name.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_varname(int ncid, int varid, char *name)
+int
+PIOc_inq_varname(int ncid, int varid, char *name)
 {
     return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL);
 }
 
 /**
- * @ingroup PIO_inq_vartype
  * Find the type of a variable.
  *
  * @param ncid the ncid of the open file.
@@ -933,15 +1017,16 @@ int PIOc_inq_varname(int ncid, int varid, char *name)
  * @param xtypep a pointer that will get the type of the
  * attribute. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep)
+int
+PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep)
 {
     return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL);
 }
 
 /**
- * @ingroup PIO_inq_varndims
  * Find the number of dimensions of a variable.
  *
  * @param ncid the ncid of the open file.
@@ -949,15 +1034,16 @@ int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep)
  * @param ndimsp a pointer that will get the number of
  * dimensions. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_varndims(int ncid, int varid, int *ndimsp)
+int
+PIOc_inq_varndims(int ncid, int varid, int *ndimsp)
 {
     return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL);
 }
 
 /**
- * @ingroup PIO_inq_vardimid
  * Find the dimension IDs associated with a variable.
  *
  * @param ncid the ncid of the open file.
@@ -965,15 +1051,16 @@ int PIOc_inq_varndims(int ncid, int varid, int *ndimsp)
  * @param dimidsp a pointer that will get an array of dimids. Ignored
  * if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp)
+int
+PIOc_inq_vardimid(int ncid, int varid, int *dimidsp)
 {
     return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL);
 }
 
 /**
- * @ingroup PIO_inq_varnatts
  * Find the number of attributes associated with a variable.
  *
  * @param ncid the ncid of the open file.
@@ -981,15 +1068,16 @@ int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp)
  * @param nattsp a pointer that will get the number of attriburtes. Ignored
  * if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_varnatts(int ncid, int varid, int *nattsp)
+int
+PIOc_inq_varnatts(int ncid, int varid, int *nattsp)
 {
     return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp);
 }
 
 /**
- * @ingroup PIO_inq_varid
  * The PIO-C interface for the NetCDF function nc_inq_varid.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -999,12 +1087,14 @@ int PIOc_inq_varnatts(int ncid, int varid, int *nattsp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
- * @param varid the variable ID.
+ * @param name the variable name.
  * @param varidp a pointer that will get the variable id
  * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_varid(int ncid, const char *name, int *varidp)
+int
+PIOc_inq_varid(int ncid, const char *name, int *varidp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1043,9 +1133,9 @@ int PIOc_inq_varid(int ncid, const char *name, int *varidp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1062,20 +1152,19 @@ int PIOc_inq_varid(int ncid, const char *name, int *varidp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if (varidp)
         if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_att
  * The PIO-C interface for the NetCDF function nc_inq_att.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1085,14 +1174,19 @@ int PIOc_inq_varid(int ncid, const char *name, int *varidp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
- * @param varid the variable ID.
+ * @param varid the variable ID or NC_GLOBAL.
+ * @param name name of the attribute.
+ * @param eh non-zero to handle errors in the function. This will
+ * cause program to halt if PIO error handler is set to INTERNAL.
  * @param xtypep a pointer that will get the type of the attribute.
  * @param lenp a pointer that will get the number of values
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_att_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
-                 PIO_Offset *lenp)
+int
+PIOc_inq_att_eh(int ncid, int varid, const char *name, int eh,
+                nc_type *xtypep, PIO_Offset *lenp)
 {
     int msg = PIO_MSG_INQ_ATT;
     iosystem_desc_t *ios;
@@ -1139,9 +1233,9 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1159,23 +1253,51 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
-    if (ierr)
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (eh && ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
-    /* Broadcast results. */
-    if (xtypep)
-        if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
-    if (lenp)
-        if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+    /* Broadcast results if call succeeded. */
+    if (!ierr)
+    {
+        if (xtypep)
+            if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+                check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+        if (lenp)
+            if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+                check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    }
 
-    return PIO_NOERR;
+    return ierr;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_inq_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varid the variable ID or NC_GLOBAL.
+ * @param name name of the attribute.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param lenp a pointer that will get the number of values
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_att_c
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
+             PIO_Offset *lenp)
+{
+    return PIOc_inq_att_eh(ncid, varid, name, 1, xtypep, lenp);
 }
 
 /**
- * @ingroup PIO_inq_attlen
  * Get the length of an attribute.
  *
  * @param ncid the ID of an open file.
@@ -1184,15 +1306,16 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
  * @param lenp a pointer that gets the lenght of the attribute
  * array. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_attlen_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp)
+int
+PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp)
 {
     return PIOc_inq_att(ncid, varid, name, NULL, lenp);
 }
 
 /**
- * @ingroup PIO_inq_atttype
  * Get the type of an attribute.
  *
  * @param ncid the ID of an open file.
@@ -1201,15 +1324,16 @@ int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp)
  * @param xtypep a pointer that gets the type of the
  * attribute. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_atttype_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep)
+int
+PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep)
 {
     return PIOc_inq_att(ncid, varid, name, xtypep, NULL);
 }
 
 /**
- * @ingroup PIO_inq_attname
  * The PIO-C interface for the NetCDF function nc_inq_attname.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1221,10 +1345,13 @@ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep)
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
  * @param attnum the attribute ID.
+ * @param name the name of the attribute.
  * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @ingroup PIO_inq_attname_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
+int
+PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1262,9 +1389,9 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1282,7 +1409,7 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1291,17 +1418,16 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
     {
         int namelen = strlen(name);
         if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         /* Casting to void to avoid warnings on some compilers. */
         if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_inq_attid
  * The PIO-C interface for the NetCDF function nc_inq_attid.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1312,11 +1438,16 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
- * @param idp a pointer that will get the id of the variable or attribute.
+ * @param name a pointer that will get name of attribute. Ignored if
+ * NULL.
+ * @param idp a pointer that will get the id of the variable or
+ * attribute. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @ingroup PIO_inq_attid_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
+int
+PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1360,9 +1491,9 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1380,20 +1511,19 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results. */
     if (idp)
         if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_rename_dim
  * The PIO-C interface for the NetCDF function nc_rename_dim.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1403,10 +1533,15 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
- * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @param dimid the dimension ID.
+ * @param name the new name for the dimension.
+ * @return PIO_NOERR for success, error code otherwise. See
+ * @ingroup PIO_rename_dim_c
+ * PIOc_Set_File_Error_Handling().
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_rename_dim(int ncid, int dimid, const char *name)
+int
+PIOc_rename_dim(int ncid, int dimid, const char *name)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1449,9 +1584,9 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
 
@@ -1470,7 +1605,7 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1478,7 +1613,6 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name)
 }
 
 /**
- * @ingroup PIO_rename_var
  * The PIO-C interface for the NetCDF function nc_rename_var.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1489,10 +1623,14 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
- * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ * @param name the new name for the variable.
+ * @return PIO_NOERR for success, error code otherwise. See
+ * @ingroup PIO_rename_var_c
+ * PIOc_Set_File_Error_Handling().
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_rename_var(int ncid, int varid, const char *name)
+int
+PIOc_rename_var(int ncid, int varid, const char *name)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1535,9 +1673,9 @@ int PIOc_rename_var(int ncid, int varid, const char *name)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
 
@@ -1556,7 +1694,7 @@ int PIOc_rename_var(int ncid, int varid, const char *name)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1564,7 +1702,6 @@ int PIOc_rename_var(int ncid, int varid, const char *name)
 }
 
 /**
- * @ingroup PIO_rename_att
  * The PIO-C interface for the NetCDF function nc_rename_att.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1575,12 +1712,16 @@ int PIOc_rename_var(int ncid, int varid, const char *name)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
- * @return PIO_NOERR for success, error code otherwise.  See
- * PIOc_Set_File_Error_Handling
+ * @param name the name of the attribute.
+ * @param newname the new name for the attribute.
+ * @return PIO_NOERR for success, error code otherwise. See
+ * @ingroup PIO_rename_att_c
+ * PIOc_Set_File_Error_Handling().
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_rename_att(int ncid, int varid, const char *name,
-                    const char *newname)
+int
+PIOc_rename_att(int ncid, int varid, const char *name,
+                const char *newname)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1628,9 +1769,9 @@ int PIOc_rename_att(int ncid, int varid, const char *name,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1647,7 +1788,7 @@ int PIOc_rename_att(int ncid, int varid, const char *name,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1656,7 +1797,6 @@ int PIOc_rename_att(int ncid, int varid, const char *name,
 }
 
 /**
- * @ingroup PIO_del_att
  * The PIO-C interface for the NetCDF function nc_del_att.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1669,9 +1809,11 @@ int PIOc_rename_att(int ncid, int varid, const char *name,
  * @param varid the variable ID.
  * @param name of the attribute to delete.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_del_att_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_del_att(int ncid, int varid, const char *name)
+int
+PIOc_del_att(int ncid, int varid, const char *name)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1712,9 +1854,9 @@ int PIOc_del_att(int ncid, int varid, const char *name)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1731,7 +1873,7 @@ int PIOc_del_att(int ncid, int varid, const char *name)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1751,10 +1893,11 @@ int PIOc_del_att(int ncid, int varid, const char *name)
  * @param fillmode either NC_FILL or NC_NOFILL.
  * @param old_modep a pointer to an int that gets the old setting.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_set_fill
+ * @ingroup PIO_set_fill_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
+int
+PIOc_set_fill(int ncid, int fillmode, int *old_modep)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1792,9 +1935,9 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1814,7 +1957,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -1823,7 +1966,7 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
     {
         LOG((2, "old_mode = %d", *old_modep));
         if ((mpierr = MPI_Bcast(old_modep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     LOG((2, "PIOc_set_fill succeeded"));
@@ -1831,7 +1974,6 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
 }
 
 /**
- * @ingroup PIO_enddef
  * The PIO-C interface for the NetCDF function nc_enddef.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1842,15 +1984,16 @@ int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_enddef_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_enddef(int ncid)
+int
+PIOc_enddef(int ncid)
 {
     return pioc_change_def(ncid, 1);
 }
 
 /**
- * @ingroup PIO_redef
  * The PIO-C interface for the NetCDF function nc_redef.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1861,15 +2004,16 @@ int PIOc_enddef(int ncid)
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_redef_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_redef(int ncid)
+int
+PIOc_redef(int ncid)
 {
     return pioc_change_def(ncid, 0);
 }
 
 /**
- * @ingroup PIO_def_dim
  * The PIO-C interface for the NetCDF function nc_def_dim.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -1879,11 +2023,15 @@ int PIOc_redef(int ncid)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
+ * @param name name of the dimension.
+ * @param len length of the dimension.
  * @param idp a pointer that will get the id of the variable or attribute.
  * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_def_dim_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
+int
+PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1926,9 +2074,9 @@ int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1945,14 +2093,14 @@ int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if (idp)
         if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     LOG((2, "def_dim ierr = %d", ierr));
     return PIO_NOERR;
@@ -1968,14 +2116,18 @@ int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
  *
  * @param ncid the ncid of the open file, obtained from
  * PIOc_openfile() or PIOc_createfile().
- * @param varid the variable ID.
- * @param varidp a pointer that will get the variable id
+ * @param name the variable name.
+ * @param xtype the PIO_TYPE of the variable.
+ * @param ndims the number of dimensions.
+ * @param dimidsp pointer to array of dimension IDs.
+ * @param varidp a pointer that will get the variable ID.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
-                 const int *dimidsp, int *varidp)
+int
+PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
+             const int *dimidsp, int *varidp)
 {
     iosystem_desc_t *ios;      /* Pointer to io system information. */
     file_desc_t *file;         /* Pointer to file information. */
@@ -2014,10 +2166,13 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
         /* Get the MPI type corresponding with the PIO type. */
         if ((ierr = find_mpi_type(xtype, &mpi_type, NULL)))
             return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
-        
+
         /* Get the size of the MPI type. */
-        if ((mpierr = MPI_Type_size(mpi_type, &mpi_type_size)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        if(mpi_type == MPI_DATATYPE_NULL)
+            mpi_type_size = 0;
+        else
+            if ((mpierr = MPI_Type_size(mpi_type, &mpi_type_size)))
+                return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
         /* How many unlimited dims are present in the file? */
         if ((ierr = PIOc_inq_unlimdims(ncid, &nunlimdims, NULL)))
@@ -2086,21 +2241,21 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&rec_var, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&invalid_unlim_dim, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&pio_type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&mpi_type, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&mpi_type_size, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* Check that only one unlimited dim is specified, and that it is
@@ -2118,6 +2273,8 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
 
         if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
             ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, &varid);
+        LOG((3, "defined var ierr %d file->iotype %d", ierr, file->iotype));
+
 #ifdef _NETCDF4
         /* For netCDF-4 serial files, turn on compression for this variable. */
         if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C)
@@ -2131,13 +2288,13 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results. */
     if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        check_mpi(file, mpierr, __FILE__, __LINE__);
+        check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (varidp)
         *varidp = varid;
 
@@ -2172,12 +2329,13 @@ int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
  * @param ncid the ncid of the open file.
  * @param varid the ID of the variable to set chunksizes for.
  * @param fill_mode fill mode for this variable (NC_FILL or NC_NOFILL)
- * @param fill_value pointer to the fill value to be used if fill_mode is set to NC_FILL.
+ * @param fill_valuep pointer to the fill value to be used if fill_mode is set to NC_FILL.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_valuep)
+int
+PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_valuep)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -2208,8 +2366,8 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value
             return check_netcdf(file, ierr, __FILE__, __LINE__);
         if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &type_size)))
             return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_def_var_fill type_size = %d", type_size));
     }
-    LOG((2, "PIOc_def_var_fill type_size = %d", type_size));
 
     /* If async is in use, and this is not an IO task, bcast the parameters. */
     if (ios->async)
@@ -2241,15 +2399,15 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (ios->ioproc)
@@ -2264,7 +2422,11 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value
         {
             LOG((2, "defining fill value attribute for netCDF classic file"));
             if (file->do_io)
-                ierr = nc_put_att(file->fh, varid, _FillValue, xtype, 1, fill_valuep);
+            {
+                ierr = nc_set_fill(file->fh, NC_FILL, NULL);
+                if (!ierr)
+                    ierr = nc_put_att(file->fh, varid, _FillValue, xtype, 1, fill_valuep);
+            }
         }
         else
         {
@@ -2278,7 +2440,7 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -2302,10 +2464,11 @@ int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_value
  * @param fill_valuep pointer to space that gets the fill value for
  * this variable. Ignored if NULL.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_inq_var_fill
+ * @ingroup PIO_inq_var_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
+int
+PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -2363,15 +2526,15 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -2455,21 +2618,27 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. Ignore NULL parameters. */
     if (no_fill)
         if ((mpierr = MPI_Bcast(no_fill, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (fill_valuep)
         if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, ios->ioroot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
+/**
+ * @addtogroup PIO_get_att_c Get Attribute Values
+ * Get the values stored in an attribute in C.
+ * @{
+ */
+
 /**
  * Get the value of an attribute of any type, with no type conversion.
  *
@@ -2482,10 +2651,10 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
  * @param name the name of the attribute to get
  * @param ip a pointer that will get the attribute value.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_get_att
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att(int ncid, int varid, const char *name, void *ip)
+int
+PIOc_get_att(int ncid, int varid, const char *name, void *ip)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -2505,37 +2674,13 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip)
 
     /* Get the type of the attribute. */
     if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, NULL)))
-        return check_netcdf(file, ierr, __FILE__, __LINE__);
+        return ierr;
     LOG((2, "atttype = %d", atttype));
 
     return PIOc_get_att_tc(ncid, varid, name, atttype, ip);
 }
 
 /**
- * @ingroup PIO_put_att
- * Write a netCDF attribute of any type.
- *
- * This routine is called collectively by all tasks in the communicator
- * ios.union_comm.
- *
- * @param ncid the ncid of the open file, obtained from
- * PIOc_openfile() or PIOc_createfile().
- * @param varid the variable ID.
- * @param name the name of the attribute.
- * @param xtype the nc_type of the attribute.
- * @param len the length of the attribute array.
- * @param op a pointer with the attribute data.
- * @return PIO_NOERR for success, error code otherwise.
- * @author Jim Edwards, Ed Hartnett
- */
-int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype,
-                 PIO_Offset len, const void *op)
-{
-    return PIOc_put_att_tc(ncid, varid, name, xtype, len, xtype, op);
-}
-
-/**
- * @ingroup PIO_get_att
  * Get the value of an 64-bit floating point array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2549,13 +2694,13 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip)
+int
+PIOc_get_att_double(int ncid, int varid, const char *name, double *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_DOUBLE, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 8-bit unsigned char array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2569,13 +2714,13 @@ int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip)
+int
+PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_UBYTE, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 16-bit unsigned integer array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2589,7 +2734,8 @@ int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip)
+int
+PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_USHORT, (void *)ip);
 }
@@ -2606,16 +2752,15 @@ int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *i
  * @param name the name of the attribute to get
  * @param ip a pointer that will get the attribute value.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_get_att
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip)
+int
+PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_UINT, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 32-bit ingeger array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2629,7 +2774,8 @@ int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip)
+int
+PIOc_get_att_long(int ncid, int varid, const char *name, long *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_LONG_INTERNAL, (void *)ip);
 }
@@ -2648,16 +2794,15 @@ int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip)
  * @param name the name of the attribute to get
  * @param ip a pointer that will get the attribute value.
  * @return PIO_NOERR for success, error code otherwise.
- * @ingroup PIO_get_att
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip)
+int
+PIOc_get_att_text(int ncid, int varid, const char *name, char *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_CHAR, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 8-bit signed char array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2671,13 +2816,13 @@ int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip)
+int
+PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_BYTE, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 64-bit unsigned integer array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2691,13 +2836,13 @@ int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip)
+int
+PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_UINT64, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 16-bit integer array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2711,13 +2856,13 @@ int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip)
+int
+PIOc_get_att_short(int ncid, int varid, const char *name, short *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_SHORT, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 32-bit integer array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2731,13 +2876,13 @@ int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip)
+int
+PIOc_get_att_int(int ncid, int varid, const char *name, int *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_INT, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 64-bit integer array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2751,13 +2896,13 @@ int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip)
+int
+PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_INT64, (void *)ip);
 }
 
 /**
- * @ingroup PIO_get_att
  * Get the value of an 32-bit floating point array attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2771,13 +2916,46 @@ int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip)
+int
+PIOc_get_att_float(int ncid, int varid, const char *name, float *ip)
 {
     return PIOc_get_att_tc(ncid, varid, name, PIO_FLOAT, (void *)ip);
 }
 
 /**
- * @ingroup PIO_put_att
+ * @}
+ */
+
+/**
+ * @addtogroup PIO_put_att_c Write an Attribute
+ * Create an attribute in C.
+ * @{
+ */
+
+/**
+ * Write a netCDF attribute of any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype,
+             PIO_Offset len, const void *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, xtype, op);
+}
+
+/**
  * Write a netCDF attribute array of 8-bit signed chars.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2793,14 +2971,14 @@ int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip)
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype,
-                       PIO_Offset len, const signed char *op)
+int
+PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype,
+                   PIO_Offset len, const signed char *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_BYTE, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 32-bit signed integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2816,14 +2994,14 @@ int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype,
-                      PIO_Offset len, const long *op)
+int
+PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype,
+                  PIO_Offset len, const long *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_LONG_INTERNAL, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 32-bit signed integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2839,14 +3017,14 @@ int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype,
-                     PIO_Offset len, const int *op)
+int
+PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype,
+                 PIO_Offset len, const int *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 8-bit unsigned chars.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2862,14 +3040,14 @@ int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype,
-                       PIO_Offset len, const unsigned char *op)
+int
+PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype,
+                   PIO_Offset len, const unsigned char *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UBYTE, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 64-bit signed integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2885,14 +3063,14 @@ int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype,
-                          PIO_Offset len, const long long *op)
+int
+PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype,
+                      PIO_Offset len, const long long *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT64, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 32-bit unsigned integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2908,14 +3086,14 @@ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype,
-                      PIO_Offset len, const unsigned int *op)
+int
+PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype,
+                  PIO_Offset len, const unsigned int *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 32-bit floating points.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2931,14 +3109,14 @@ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype,
-                       PIO_Offset len, const float *op)
+int
+PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype,
+                   PIO_Offset len, const float *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_FLOAT, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 64-bit unsigned integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2954,14 +3132,14 @@ int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
-                           PIO_Offset len, const unsigned long long *op)
+int
+PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const unsigned long long *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT64, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 16-bit unsigned integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2977,14 +3155,14 @@ int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype,
-                        PIO_Offset len, const unsigned short *op)
+int
+PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype,
+                    PIO_Offset len, const unsigned short *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_USHORT, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF text attribute.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -2994,20 +3172,19 @@ int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype,
  * PIOc_openfile() or PIOc_createfile().
  * @param varid the variable ID.
  * @param name the name of the attribute.
- * @param xtype the nc_type of the attribute.
  * @param len the length of the attribute array.
  * @param op a pointer with the attribute data.
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_text(int ncid, int varid, const char *name,
-                      PIO_Offset len, const char *op)
+int
+PIOc_put_att_text(int ncid, int varid, const char *name,
+                  PIO_Offset len, const char *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, NC_CHAR, len, NC_CHAR, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 16-bit integers.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -3023,14 +3200,14 @@ int PIOc_put_att_text(int ncid, int varid, const char *name,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype,
-                       PIO_Offset len, const short *op)
+int
+PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype,
+                   PIO_Offset len, const short *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_SHORT, op);
 }
 
 /**
- * @ingroup PIO_put_att
  * Write a netCDF attribute array of 64-bit floating points.
  *
  * This routine is called collectively by all tasks in the communicator
@@ -3046,8 +3223,12 @@ int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype,
  * @return PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype,
-                        PIO_Offset len, const double *op)
+int
+PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype,
+                    PIO_Offset len, const double *op)
 {
     return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_DOUBLE, op);
 }
+/**
+ * @}
+ */
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 62ceb080c43f..b536a2da3382 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -28,11 +28,12 @@
  * @param deflate_level 1 to 9, with 1 being faster and 9 being more
  * compressed.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
-                         int deflate_level)
+int
+PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
+                     int deflate_level)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -72,9 +73,9 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 
         /* Handle MPI errors from computation tasks. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (ios->ioproc)
@@ -90,7 +91,7 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -118,11 +119,12 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
  * level (from 1-9) if deflation is in use for this variable.  Ignored
  * if NULL.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_inq_var
+ * @ingroup PIO_inq_var_c
  * @author Ed Hartnett
  */
-int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
-                         int *deflate_levelp)
+int
+PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
+                     int *deflate_levelp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -174,9 +176,9 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (ios->ioproc)
@@ -189,26 +191,25 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. */
     if (shufflep)
         if ((mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (deflatep)
         if ((mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (deflate_levelp)
         if ((mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * @ingroup PIO_def_var
  * Set chunksizes for a variable.
  *
  * This function only applies to netCDF-4 files. When used with netCDF
@@ -226,12 +227,14 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
  * @param ncid the ncid of the open file.
  * @param varid the ID of the variable to set chunksizes for.
  * @param storage NC_CONTIGUOUS or NC_CHUNKED.
- * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * @param chunksizesp an array of chunksizes. Must have a chunksize for
  * every variable dimension.
  * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp)
+int
+PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -289,13 +292,13 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *ch
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     LOG((2, "PIOc_def_var_chunking ndims = %d", ndims));
@@ -324,7 +327,7 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *ch
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -346,14 +349,15 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *ch
  * @param varid the ID of the variable to set chunksizes for.
  * @param storagep pointer to int which will be set to either
  * NC_CONTIGUOUS or NC_CHUNKED.
- * @param chunksizep pointer to memory where chunksizes will be
+ * @param chunksizesp pointer to memory where chunksizes will be
  * set. There are the same number of chunksizes as there are
  * dimensions.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_inq_var
+ * @ingroup PIO_inq_var_c
  * @author Ed Hartnett
  */
-int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp)
+int
+PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -410,13 +414,13 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* Broadcast values currently only known on computation tasks to IO tasks. */
         if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -444,19 +448,19 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. */
     if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (storagep)
         if ((mpierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (chunksizesp)
         if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
@@ -478,14 +482,13 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunks
  *
  * @param ncid the ncid of the open file.
  * @param varid the ID of the variable to set chunksizes for.
- * @param storage NC_CONTIGUOUS or NC_CHUNKED.
- * @param chunksizep an array of chunksizes. Must have a chunksize for
- * every variable dimension.
+ * @param endian NC_ENDIAN_NATIVE, NC_ENDIAN_LITTLE, or NC_ENDIAN_BIG.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_def_var_endian(int ncid, int varid, int endian)
+int
+PIOc_def_var_endian(int ncid, int varid, int endian)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -520,9 +523,9 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (ios->ioproc)
@@ -535,7 +538,7 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -558,10 +561,11 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
  * @param endianp pointer to int which will be set to
  * endianness. Ignored if NULL.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_inq_var
+ * @ingroup PIO_inq_var_c
  * @author Ed Hartnett
  */
-int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
+int
+PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -600,9 +604,9 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -616,14 +620,14 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. */
     if (endianp)
         if ((mpierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
@@ -645,16 +649,18 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
  * variable documentation for details about the operation of this
  * function.
  *
+ * @param iosysid the IO system ID.
  * @param iotype the iotype of files to be created or opened.
  * @param size size of file cache.
  * @param nelems number of elements in file cache.
  * @param preemption preemption setting for file cache.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
-                         float preemption)
+int
+PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
+                     float preemption)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     int ierr;              /* Return code from function calls. */
@@ -695,9 +701,9 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset ne
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -715,7 +721,7 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset ne
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if (ierr)
         check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
 
@@ -745,16 +751,18 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset ne
  * attempts to choose sensible chunk sizes by default, but for best
  * performance check chunking against access patterns.
  *
+ * @param iosysid the IO system ID.
  * @param iotype the iotype of files to be created or opened.
  * @param sizep gets the size of file cache.
  * @param nelemsp gets the number of elements in file cache.
  * @param preemptionp gets the preemption setting for file cache.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
-                         float *preemptionp)
+int
+PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                     float *preemptionp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     int ierr;              /* Return code from function calls. */
@@ -800,9 +808,9 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -820,7 +828,7 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "bcast complete ierr = %d sizep = %d", ierr, sizep));
     if (ierr)
         return check_netcdf(NULL, ierr, __FILE__, __LINE__);
@@ -829,19 +837,19 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset
     {
         LOG((2, "bcasting size = %d ios->ioroot = %d", *sizep, ios->ioroot));
         if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((2, "bcast size = %d", *sizep));
     }
     if (nelemsp)
     {
         if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((2, "bcast complete nelems = %d", *nelemsp));
     }
     if (preemptionp)
     {
         if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((2, "bcast complete preemption = %d", *preemptionp));
     }
 
@@ -865,15 +873,16 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset
  *
  * @param ncid the ncid of the open file.
  * @param varid the ID of the variable to set chunksizes for.
- * @param storage NC_CONTIGUOUS or NC_CHUNKED.
- * @param chunksizep an array of chunksizes. Must have a chunksize for
- * every variable dimension.
+ * @param size the size in bytes for the cache.
+ * @param nelems the number of elements in the cache.
+ * @param preemption the cache preemption value.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_def_var
+ * @ingroup PIO_def_var_c
  * @author Ed Hartnett
  */
-int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
-                             float preemption)
+int
+PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
+                         float preemption)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -913,9 +922,9 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     if (ios->ioproc)
@@ -928,7 +937,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
@@ -955,11 +964,12 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
  * @param nelemsp will get the number of elements in the cache. Ignored if NULL.
  * @param preemptionp will get the cache preemption value. Ignored if NULL.
  * @return PIO_NOERR for success, otherwise an error code.
- * @ingroup PIO_inq_var
+ * @ingroup PIO_inq_var_c
  * @author Ed Hartnett
  */
-int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
-                             float *preemptionp)
+int
+PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                         float *preemptionp)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1007,9 +1017,9 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -1024,20 +1034,20 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
 
     /* Broadcast results to all tasks. */
     if (sizep && !ierr)
         if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (nelemsp && !ierr)
         if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (preemptionp && !ierr)
         if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
diff --git a/src/clib/pio_put_nc.c b/src/clib/pio_put_nc.c
index 9abd69be7c8c..d6f87ae08f93 100644
--- a/src/clib/pio_put_nc.c
+++ b/src/clib/pio_put_nc.c
@@ -6,13 +6,18 @@
  * @date  2016
  * @see http://code.google.com/p/parallelio/
  */
-
 #include 
 #include 
 #include 
 
 /**
- * Get strided, muti-dimensional subset of a text variable.
+ * @addtogroup PIO_put_vars_c Write Strided Arrays
+ * Write strided arrays of data to a Variable in C.
+ * @{
+ */
+
+/**
+ * Put strided, muti-dimensional subset of a text variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -32,14 +37,15 @@
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                       const PIO_Offset *stride, const char *op)
+int
+PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                   const PIO_Offset *stride, const char *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of an unsigned char variable.
+ * Put strided, muti-dimensional subset of an unsigned char variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -59,15 +65,16 @@ int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_O
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const PIO_Offset *stride,
-                        const unsigned char *op)
+int
+PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const PIO_Offset *stride,
+                    const unsigned char *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a signed char variable.
+ * Put strided, muti-dimensional subset of a signed char variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -87,14 +94,15 @@ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                        const PIO_Offset *stride, const signed char *op)
+int
+PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                    const PIO_Offset *stride, const signed char *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of an unsigned 16-bit integer
+ * Put strided, muti-dimensional subset of an unsigned 16-bit integer
  * variable.
  *
  * This routine is called collectively by all tasks in the
@@ -115,14 +123,15 @@ int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                         const PIO_Offset *stride, const unsigned short *op)
+int
+PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, const unsigned short *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a 16-bit integer variable.
+ * Put strided, muti-dimensional subset of a 16-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -142,14 +151,15 @@ int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const PIO_Offset *stride, const short *op)
+int
+PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const PIO_Offset *stride, const short *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of an unsigned integer
+ * Put strided, muti-dimensional subset of an unsigned integer
  * variable.
  *
  * This routine is called collectively by all tasks in the
@@ -170,14 +180,15 @@ int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                       const PIO_Offset *stride, const unsigned int *op)
+int
+PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                   const PIO_Offset *stride, const unsigned int *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of an integer variable.
+ * Put strided, muti-dimensional subset of an integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -197,14 +208,15 @@ int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_O
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                      const PIO_Offset *stride, const int *op)
+int
+PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, const int *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a 64-bit integer variable.
+ * Put strided, muti-dimensional subset of a 64-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -224,14 +236,15 @@ int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Of
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                       const PIO_Offset *stride, const long *op)
+int
+PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                   const PIO_Offset *stride, const long *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a floating point variable.
+ * Put strided, muti-dimensional subset of a floating point variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -251,14 +264,15 @@ int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_O
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                        const PIO_Offset *stride, const float *op)
+int
+PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                    const PIO_Offset *stride, const float *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a 64-bit unsigned integer
+ * Put strided, muti-dimensional subset of a 64-bit unsigned integer
  * variable.
  *
  * This routine is called collectively by all tasks in the
@@ -279,14 +293,15 @@ int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                           const PIO_Offset *stride, const long long *op)
+int
+PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const long long *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of a 64-bit floating point
+ * Put strided, muti-dimensional subset of a 64-bit floating point
  * variable.
  *
  * This routine is called collectively by all tasks in the
@@ -307,14 +322,15 @@ int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const P
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                         const PIO_Offset *stride, const double *op)
+int
+PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, const double *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op);
 }
 
 /**
- * Get strided, muti-dimensional subset of an unsigned 64-bit integer
+ * Put strided, muti-dimensional subset of an unsigned 64-bit integer
  * variable.
  *
  * This routine is called collectively by all tasks in the
@@ -335,14 +351,15 @@ int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                            const PIO_Offset *stride, const unsigned long long *op)
+int
+PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                        const PIO_Offset *stride, const unsigned long long *op)
 {
     return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op);
 }
 
 /**
- * Get one value from an text variable.
+ * Write strided, muti-dimensional subset of a variable of any type.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -352,243 +369,323 @@ int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const
  * @param start an array of start indicies (must have same number of
  * entries as variable has dimensions). If NULL, indices of 0 will be
  * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+              const PIO_Offset *stride, const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_NAT, op);
+}
+
+/**
+ * @}
+ */
+/**
+ * @addtogroup PIO_put_var1_c Write One Value
+ * Write one value to a variable in C.
+ * @{
+ */
+
+/**
+ * Put one value from an text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op)
+int
+PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_CHAR, op);
 }
 
 /**
- * Get one value from an text variable.
+ * Put one value from an text variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
-                        const unsigned char *op)
+int
+PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
+                    const unsigned char *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_UBYTE, op);
 }
 
 /**
- * Get one value from an signed char variable.
+ * Put one value from an signed char variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index,
-                        const signed char *op)
+int
+PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index,
+                    const signed char *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_BYTE, op);
 }
 
 /**
- * Get one value from an unsigned 16-bit integer variable.
+ * Put one value from an unsigned 16-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
-                         const unsigned short *op)
+int
+PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
+                     const unsigned short *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_USHORT, op);
 }
 
 /**
- * Get one value from a 16-bit integer variable.
+ * Put one value from a 16-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index,
-                        const short *op)
+int
+PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index,
+                    const short *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_SHORT, op);
 }
 
 /**
- * Get one value from an unsigned integer variable.
+ * Put one value from an unsigned integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
-                       const unsigned int *op)
+int
+PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
+                   const unsigned int *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_UINT, op);
 }
 
 /**
- * Get one value from an integer variable.
+ * Put one value from an integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op)
+int
+PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_INT, op);
 }
 
 /**
- * Get one value from an floating point variable.
+ * Put one value from an floating point variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op)
+int
+PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_FLOAT, op);
 }
 
 /**
- * Get one value from an integer variable.
+ * Put one value from an integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op)
+int
+PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, PIO_LONG_INTERNAL, op);
 }
 
 /**
- * Get one value from an 64-bit floating point variable.
+ * Put one value from an 64-bit floating point variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index,
-                         const double *op)
+int
+PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index,
+                     const double *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_DOUBLE, op);
 }
 
 /**
- * Get one value from an unsigned 64-bit integer variable.
+ * Put one value from an unsigned 64-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
-                            const unsigned long long *op)
+int
+PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
+                        const unsigned long long *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_UINT64, op);
 }
 
 /**
- * Get one value from a 64-bit integer variable.
+ * Put one value from a 64-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index,
-                           const long long *op)
+int
+PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index,
+                       const long long *op)
 {
     return PIOc_put_var1_tc(ncid, varid, index, NC_INT64, op);
 }
 
+/**
+ * Put one value from a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of indicies where the data value will be
+ * written (must have same number of entries as variable has
+ * dimensions). If NULL, indices of 0 will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_NAT, op);
+}
+
+/**
+ * @}
+ */
+/**
+ * @addtogroup PIO_put_vara_c Write Arrays
+ * Write arrays of data to a Variable in C, specifying start and count
+ * arrays.
+ * @{
+ */
+
 /**
  * Put muti-dimensional subset of a text variable.
  *
@@ -607,8 +704,9 @@ int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
-                       const PIO_Offset *count, const char *op)
+int
+PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
+                   const PIO_Offset *count, const char *op)
 {
     return PIOc_put_vars_text(ncid, varid, start, count, NULL, op);
 }
@@ -631,8 +729,9 @@ int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const unsigned char *op)
+int
+PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const unsigned char *op)
 {
     return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op);
 }
@@ -655,8 +754,9 @@ int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const signed char *op)
+int
+PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const signed char *op)
 {
     return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op);
 }
@@ -679,8 +779,9 @@ int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
-                         const PIO_Offset *count, const unsigned short *op)
+int
+PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                     const PIO_Offset *count, const unsigned short *op)
 {
     return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op);
 }
@@ -703,8 +804,9 @@ int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const short *op)
+int
+PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const short *op)
 {
     return PIOc_put_vars_short(ncid, varid, start, count, NULL, op);
 }
@@ -727,8 +829,9 @@ int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
-                       const PIO_Offset *count, const unsigned int *op)
+int
+PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                   const PIO_Offset *count, const unsigned int *op)
 {
     return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op);
 }
@@ -751,8 +854,9 @@ int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start,
-                      const PIO_Offset *count, const int *op)
+int
+PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start,
+                  const PIO_Offset *count, const int *op)
 {
     return PIOc_put_vars_int(ncid, varid, start, count, NULL, op);
 }
@@ -775,8 +879,9 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start,
-                       const PIO_Offset *count, const long *op)
+int
+PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start,
+                   const PIO_Offset *count, const long *op)
 {
     return PIOc_put_vars_long(ncid, varid, start, count, NULL, op);
 }
@@ -799,14 +904,15 @@ int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
-                        const PIO_Offset *count, const float *op)
+int
+PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
+                    const PIO_Offset *count, const float *op)
 {
     return PIOc_put_vars_float(ncid, varid, start, count, NULL, op);
 }
 
 /**
- * Put muti-dimensional subset of an unsigned 64-bit integer variable.
+ * Put muti-dimensional subset of a 64-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -823,14 +929,15 @@ int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
-                            const PIO_Offset *count, const unsigned long long *op)
+int
+PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start,
+                     const PIO_Offset *count, const double *op)
 {
-    return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op);
+    return PIOc_put_vars_double(ncid, varid, start, count, NULL, op);
 }
 
 /**
- * Put muti-dimensional subset of a 64-bit integer variable.
+ * Put muti-dimensional subset of an unsigned 64-bit integer variable.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -847,10 +954,11 @@ int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
-                           const PIO_Offset *count, const long long *op)
+int
+PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const unsigned long long *op)
 {
-    return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op);
+    return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op);
 }
 
 /**
@@ -871,14 +979,15 @@ int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start,
-                         const PIO_Offset *count, const double *op)
+int
+PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const long long *op)
 {
-    return PIOc_put_vars_double(ncid, varid, start, count, NULL, op);
+    return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op);
 }
 
 /**
- * Put all data to a text variable.
+ * Put muti-dimensional subset of a variable of any type.
  *
  * This routine is called collectively by all tasks in the
  * communicator ios.union_comm.
@@ -895,7 +1004,36 @@ int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start,
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_text(int ncid, int varid, const char *op)
+int
+PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+              const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_NAT, op);
+}
+
+/**
+ * @}
+ */
+/**
+ * @addtogroup PIO_put_var_c Write Entire Variable
+ * Write the entire variable in C.
+ * @{
+ */
+
+/**
+ * Put all data to a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_var_text(int ncid, int varid, const char *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_CHAR, op);
 }
@@ -908,17 +1046,12 @@ int PIOc_put_var_text(int ncid, int varid, const char *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op)
+int
+PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_UBYTE, op);
 }
@@ -931,17 +1064,12 @@ int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_schar(int ncid, int varid, const signed char *op)
+int
+PIOc_put_var_schar(int ncid, int varid, const signed char *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_BYTE, op);
 }
@@ -954,17 +1082,12 @@ int PIOc_put_var_schar(int ncid, int varid, const signed char *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op)
+int
+PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op)
 {
     return PIOc_put_var_tc(ncid, varid, NC_USHORT, op);
 }
@@ -977,17 +1100,12 @@ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_short(int ncid, int varid, const short *op)
+int
+PIOc_put_var_short(int ncid, int varid, const short *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_SHORT, op);
 }
@@ -1000,17 +1118,12 @@ int PIOc_put_var_short(int ncid, int varid, const short *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op)
+int
+PIOc_put_var_uint(int ncid, int varid, const unsigned int *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_UINT, op);
 }
@@ -1023,17 +1136,12 @@ int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_int(int ncid, int varid, const int *op)
+int
+PIOc_put_var_int(int ncid, int varid, const int *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_INT, op);
 }
@@ -1046,17 +1154,12 @@ int PIOc_put_var_int(int ncid, int varid, const int *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_long(int ncid, int varid, const long *op)
+int
+PIOc_put_var_long(int ncid, int varid, const long *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_LONG_INTERNAL, op);
 }
@@ -1069,17 +1172,12 @@ int PIOc_put_var_long(int ncid, int varid, const long *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_float(int ncid, int varid, const float *op)
+int
+PIOc_put_var_float(int ncid, int varid, const float *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_FLOAT, op);
 }
@@ -1092,17 +1190,12 @@ int PIOc_put_var_float(int ncid, int varid, const float *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op)
+int
+PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_UINT64, op);
 }
@@ -1115,17 +1208,12 @@ int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_longlong(int ncid, int varid, const long long *op)
+int
+PIOc_put_var_longlong(int ncid, int varid, const long long *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_INT64, op);
 }
@@ -1138,17 +1226,12 @@ int PIOc_put_var_longlong(int ncid, int varid, const long long *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
  * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var_double(int ncid, int varid, const double *op)
+int
+PIOc_put_var_double(int ncid, int varid, const double *op)
 {
     return PIOc_put_var_tc(ncid, varid, PIO_DOUBLE, op);
 }
@@ -1161,82 +1244,16 @@ int PIOc_put_var_double(int ncid, int varid, const double *op)
  *
  * @param ncid identifies the netCDF file
  * @param varid the variable ID number
- * @param buf pointer that will get the data.
+ * @param op pointer to the data to be written.
  * @return PIO_NOERR on success, error code otherwise.
  * @author Ed Hartnett
  */
-int PIOc_put_var(int ncid, int varid, const void *op)
+int
+PIOc_put_var(int ncid, int varid, const void *op)
 {
     return PIOc_put_var_tc(ncid, varid, NC_NAT, op);
 }
 
 /**
- * Get one value from a variable of any type.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
- */
-int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *op)
-{
-    return PIOc_put_var1_tc(ncid, varid, index, NC_NAT, op);
-}
-
-/**
- * Put muti-dimensional subset of a variable of any type.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
- */
-int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                  const void *op)
-{
-    return PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_NAT, op);
-}
-
-/**
- * Write strided, muti-dimensional subset of a variable of any type.
- *
- * This routine is called collectively by all tasks in the
- * communicator ios.union_comm.
- *
- * @param ncid identifies the netCDF file
- * @param varid the variable ID number
- * @param start an array of start indicies (must have same number of
- * entries as variable has dimensions). If NULL, indices of 0 will be
- * used.
- * @param count an array of counts (must have same number of entries
- * as variable has dimensions). If NULL, counts matching the size of
- * the variable will be used.
- * @param stride an array of strides (must have same number of
- * entries as variable has dimensions). If NULL, strides of 1 will be
- * used.
- * @param buf pointer that will get the data.
- * @return PIO_NOERR on success, error code otherwise.
- * @author Ed Hartnett
+ * @}
  */
-int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
-                  const PIO_Offset *stride, const void *op)
-{
-    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_NAT, op);
-}
diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c
new file mode 100644
index 000000000000..866fd40e84c6
--- /dev/null
+++ b/src/clib/pio_put_vard.c
@@ -0,0 +1,299 @@
+/**
+ * @file
+ * PIO functions to write data with distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date 2019
+ * @see https://github.com/NCAR/ParallelIO
+ */
+#include 
+#include 
+#include 
+
+/**
+ * @addtogroup PIO_write_darray_c
+ * Write distributed arrays to a Variable in C.
+ * @{
+ */
+
+/**
+ * Put distributed array subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                   const char *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, op);
+}
+
+/**
+ * Put distributed array subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                    const unsigned char *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, op);
+}
+
+/**
+ * Put distributed array subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                    const signed char *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_BYTE, op);
+}
+
+/**
+ * Put distributed array subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                     const unsigned short *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_USHORT, op);
+}
+
+/**
+ * Put distributed array subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                    const short *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_SHORT, op);
+}
+
+/**
+ * Put distributed array subset of an unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                   const unsigned int *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UINT, op);
+}
+
+/**
+ * Put distributed array subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                  const int *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_INT, op);
+}
+
+/* /\** */
+/*  * Put distributed array subset of a 64-bit integer variable. */
+/*  * */
+/*  * This routine is called collectively by all tasks in the */
+/*  * communicator ios.union_comm. */
+/*  * */
+/*  * @param ncid identifies the netCDF file */
+/*  * @param varid the variable ID number */
+/*  * @param decompid the decomposition ID. */
+/*  * @param recnum the record number. */
+/*  * @param op pointer to the data to be written. */
+/*  * @return PIO_NOERR on success, error code otherwise. */
+/*  * @author Ed Hartnett */
+/*  *\/ */
+/* int */
+/* PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, */
+/*                    const long *op) */
+/* { */
+/*     return PIOc_put_vard_tc(ncid, varid, decompid, recnum, PIO_LONG_INTERNAL, op); */
+/* } */
+
+/**
+ * Put distributed array subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                    const float *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_FLOAT, op);
+}
+
+/**
+ * Put distributed array subset of a 64-bit unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                       const long long *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_INT64, op);
+}
+
+/**
+ * Put distributed array subset of a 64-bit floating point
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                     const double *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_DOUBLE, op);
+}
+
+/**
+ * Put distributed array subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
+                        const unsigned long long *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UINT64, op);
+}
+
+/**
+ * Write distributed array subset of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param decompid the decomposition ID.
+ * @param recnum the record number.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+PIOc_put_vard(int ncid, int varid, int decompid, const PIO_Offset recnum,
+              const void *op)
+{
+    return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_NAT, op);
+}
+
+/**
+ * @}
+ */
diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c
index 8ee13f411e62..4d8dd3328935 100644
--- a/src/clib/pio_rearrange.c
+++ b/src/clib/pio_rearrange.c
@@ -1,4 +1,5 @@
-/** @file
+/**
+ * @file
  * Code to map IO to model decomposition.
  *
  * @author Jim Edwards
@@ -7,6 +8,10 @@
 #include 
 #include 
 
+#if PIO_USE_MPISERIAL
+#define MPI_Type_create_hvector MPI_Type_hvector
+#endif
+
 /**
  * Convert a 1-D index into a coordinate value in an arbitrary
  * dimension space. E.g., for index 4 into a array defined as a[3][2],
@@ -23,8 +28,8 @@
  * corresponding to this index.
  * @author Jim Edwards
  */
-void idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx,
-                     PIO_Offset *dim_list)
+inline void idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx,
+                            PIO_Offset *dim_list)
 {
     /* Check inputs. */
     pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input",
@@ -193,7 +198,7 @@ PIO_Offset find_region(int ndims, const int *gdimlen, int maplen, const PIO_Offs
  * @returns the local array index.
  * @author Jim Edwards
  */
-PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count)
+inline PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count)
 {
     PIO_Offset lindex = 0;
     PIO_Offset stride = 1;
@@ -243,7 +248,7 @@ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc)
 
     /* Share the max io buffer size with all io tasks. */
     if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     pioassert(totiosize > 0, "totiosize <= 0", __FILE__, __LINE__);
     LOG((2, "after allreduce compute_maxIObuffersize got totiosize = %lld", totiosize));
 
@@ -338,7 +343,11 @@ int create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
         if (mcount[i] > 0)
         {
             int len = mcount[i] / blocksize;
-            int displace[len];
+            int *displace;
+
+            if (!(displace = malloc(sizeof(int) * len)))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
             LOG((3, "blocksize = %d i = %d mcount[%d] = %d len = %d", blocksize, i, i,
                  mcount[i], len));
             if (blocksize == 1)
@@ -376,9 +385,12 @@ int create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
             LOG((3, "calling MPI_Type_create_indexed_block len = %d blocksize = %d "
                  "mpitype = %d", len, blocksize, mpitype));
             /* Create an indexed datatype with constant-sized blocks. */
-            if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace,
-                                                        mpitype, &mtype[i])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            mpierr = MPI_Type_create_indexed_block(len, blocksize, displace,
+                                                   mpitype, &mtype[i]);
+
+            free(displace);
+            if (mpierr)
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
             if (mtype[i] == PIO_DATATYPE_NULL)
                 return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
@@ -386,7 +398,7 @@ int create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
             /* Commit the MPI data type. */
             LOG((3, "about to commit type"));
             if ((mpierr = MPI_Type_commit(&mtype[i])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             pos += mcount[i];
         }
     }
@@ -535,7 +547,8 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
     int ierr;
 
     /* Check inputs. */
-    pioassert(ios && iodesc && dest_ioproc && dest_ioindex &&
+    pioassert(ios && iodesc && (iodesc->ndof == 0 ||
+                                (iodesc->ndof > 0 && dest_ioproc && dest_ioindex)) &&
               iodesc->rearranger == PIO_REARR_BOX && ios->num_uniontasks > 0,
               "invalid input", __FILE__, __LINE__);
     LOG((1, "compute_counts ios->num_uniontasks = %d ios->compproc %d ios->ioproc %d",
@@ -549,8 +562,12 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
     int recv_displs[ios->num_uniontasks];
 
     /* The list of indeces on each compute task */
-    PIO_Offset s2rindex[iodesc->ndof];
-
+    PIO_Offset *s2rindex = NULL;
+    if (iodesc->ndof > 0)
+    {
+        if (!(s2rindex = malloc(sizeof(PIO_Offset) * iodesc->ndof)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
     /* Allocate memory for the array of counts and init to zero. */
     if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int))))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
@@ -767,6 +784,8 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
                           &iodesc->rearr_opts.comp2io)))
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
 
+    free(s2rindex);
+
     return PIO_NOERR;
 }
 
@@ -792,8 +811,10 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
     int ret;
 
 #ifdef TIMING
-    GPTLstart("PIO:rearrange_comp2io");
-#endif
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:rearrange_comp2io")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Caller must provide these. */
     pioassert(ios && iodesc && nvars > 0, "invalid input", __FILE__, __LINE__);
@@ -815,7 +836,7 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
 
     /* Get the number of tasks. */
     if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     /* These are parameters to pio_swapm to send data from compute to
      * IO tasks. */
@@ -864,19 +885,14 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
                      *  spaced blocks of the same size. The block size
                      *  is 1, the stride here is the length of the
                      *  collected array (llen). */
-#if PIO_USE_MPISERIAL
-                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size,
-                                                   iodesc->rtype[i], &recvtypes[i])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#else
                     if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size,
                                                           iodesc->rtype[i], &recvtypes[i])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#endif /* PIO_USE_MPISERIAL */
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
                     pioassert(recvtypes[i] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__);
 
                     if ((mpierr = MPI_Type_commit(&recvtypes[i])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 }
                 else
                 {
@@ -885,20 +901,15 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
                          "recvcounts[iodesc->rfrom[i]] = %d", i, iodesc->rfrom[i],
                          recvcounts[iodesc->rfrom[i]]));
 
-#if PIO_USE_MPISERIAL
-                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size,
-                                                   iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#else
                     if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size,
                                                           iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#endif /* PIO_USE_MPISERIAL */
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
                     pioassert(recvtypes[iodesc->rfrom[i]] != PIO_DATATYPE_NULL,  "bad mpi type",
                               __FILE__, __LINE__);
 
                     if ((mpierr = MPI_Type_commit(&recvtypes[iodesc->rfrom[i]])))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
                     rdispls[iodesc->rfrom[i]] = 0;
                 }
@@ -908,35 +919,32 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
 
     /* On compute tasks loop over iotasks and create a data type for
      * each exchange.  */
-    for (int i = 0; i < niotasks; i++)
+    if(!ios->async || ios->compproc)
     {
-        int io_comprank = ios->ioranks[i];
-        LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i]));
-        if (iodesc->rearranger == PIO_REARR_SUBSET)
-            io_comprank = 0;
-
-        LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i]));
-        if (iodesc->scount[i] > 0 && sbuf)
-        {
-            LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank));
-            sendcounts[io_comprank] = 1;
-#if PIO_USE_MPISERIAL
-            if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size,
-                                           iodesc->stype[i], &sendtypes[io_comprank])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#else
-            if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size,
-                                                  iodesc->stype[i], &sendtypes[io_comprank])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-#endif /* PIO_USE_MPISERIAL */
-            pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL,  "bad mpi type", __FILE__, __LINE__);
-
-            if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
-        }
-        else
+        for (int i = 0; i < niotasks; i++)
         {
-            sendcounts[io_comprank] = 0;
+            int io_comprank = ios->ioranks[i];
+            LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i]));
+            if (iodesc->rearranger == PIO_REARR_SUBSET)
+                io_comprank = 0;
+
+            LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i]));
+            if (iodesc->scount[i] > 0 && sbuf)
+            {
+                LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank));
+                sendcounts[io_comprank] = 1;
+                if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size,
+                                                      iodesc->stype[i], &sendtypes[io_comprank])))
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+                pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL,  "bad mpi type", __FILE__, __LINE__);
+
+                if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank])))
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+            }
+            else
+            {
+                sendcounts[io_comprank] = 0;
+            }
         }
     }
 
@@ -953,16 +961,17 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
         LOG((3, "freeing MPI types for task %d", i));
         if (sendtypes[i] != PIO_DATATYPE_NULL)
             if ((mpierr = MPI_Type_free(&sendtypes[i])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
         if (recvtypes[i] != PIO_DATATYPE_NULL)
             if ((mpierr = MPI_Type_free(&recvtypes[i])))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
 #ifdef TIMING
-    GPTLstop("PIO:rearrange_comp2io");
-#endif
+    if ((ret = pio_stop_timer("PIO:rearrange_comp2io")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     return PIO_NOERR;
 }
@@ -991,8 +1000,10 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
     pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
 
 #ifdef TIMING
-    GPTLstart("PIO:rearrange_io2comp");
-#endif
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:rearrange_io2comp")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
 
     /* Different rearrangers use different communicators and number of
      * IO tasks. */
@@ -1010,7 +1021,7 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
 
     /* Get the size of this communicator. */
     if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Define the MPI data types that will be used for this
      * io_desc_t. */
@@ -1085,7 +1096,8 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
 #ifdef TIMING
-    GPTLstop("PIO:rearrange_io2comp");
+    if ((ret = pio_stop_timer("PIO:rearrange_io2comp")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 #endif
 
     return PIO_NOERR;
@@ -1134,7 +1146,7 @@ int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gdimlen,
          totalllen, totalgridsize));
     if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM,
                                 ios->union_comm)))
-        check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "after allreduce totalllen = %d", totalllen));
 
     /* If the total size of the data provided to be written is < the
@@ -1199,8 +1211,9 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
          "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks));
 
     /* Allocate arrays needed for this function. */
-    int dest_ioproc[maplen]; /* Destination IO task for each data element on compute task. */
-    PIO_Offset dest_ioindex[maplen];    /* Offset into IO task array for each data element. */
+    int *dest_ioproc = NULL; /* Destination IO task for each data element on compute task. */
+    PIO_Offset *dest_ioindex = NULL;    /* Offset into IO task array for each data element. */
+    PIO_Offset **gcoord_map = NULL; /* Global coordinate value for each data element. */
     int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */
     int sdispls[ios->num_uniontasks];    /* Send displacements for swapm. */
     int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */
@@ -1208,12 +1221,349 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
     MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */
     PIO_Offset iomaplen[ios->num_iotasks];   /* Gets the llen of all IO tasks. */
 
+    /* sc_info msg = [iomaplen, starts_for_all_dims, count_for_all_dims] */
+    int sc_info_msg_maplen_sz = 1; /* The iomaplen, == 0 implies start/count are invalid */
+    int sc_info_msg_sc_sz = 2 * ndims; /* The (start + count) for all dims */
+    int sc_info_msg_sz = sc_info_msg_maplen_sz + sc_info_msg_sc_sz;
+    PIO_Offset sc_info_msg_send[sc_info_msg_sz];
+    PIO_Offset sc_info_msg_recv[ios->num_iotasks * sc_info_msg_sz];
+
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:box_rearrange_create")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
     /* This is the box rearranger. */
     iodesc->rearranger = PIO_REARR_BOX;
 
     /* Number of elements of data on compute node. */
     iodesc->ndof = maplen;
 
+    if (maplen > 0)
+    {
+        if (!(dest_ioproc = malloc(maplen * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(dest_ioindex = malloc(maplen * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(gcoord_map = malloc(maplen * sizeof(PIO_Offset*))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        for (int i = 0; i < maplen; i++)
+        {
+            if (!(gcoord_map[i] = calloc(ndims, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+    }
+
+    /* Initialize the sc_info send and recv messages */
+    for(int i=0; iioranks[i]). Each
+     * sc_info message contains [iomaplen, start_for_all_dims, count_for_all_dims]
+     */
+    for(int i=0; inum_iotasks * sc_info_msg_sz; i++)
+    {
+        sc_info_msg_recv[i] = 0;
+    }
+
+    /* Initialize array values. */
+    for (int i = 0; i < maplen; i++)
+    {
+        dest_ioproc[i] = -1;
+        dest_ioindex[i] = -1;
+    }
+
+    /* Initialize arrays used in swapm. */
+    for (int i = 0; i < ios->num_uniontasks; i++)
+    {
+        sendcounts[i] = 0;
+        sdispls[i] = 0;
+        recvcounts[i] = 0;
+        rdispls[i] = 0;
+        dtypes[i] = MPI_OFFSET;
+    }
+
+    /* For IO tasks, determine llen, the length of the data array on
+     * the IO task. For computation tasks, llen will remain at 0. Also
+     * set up arrays for the allgather which will give every IO task a
+     * complete list of llens for each IO task. */
+    LOG((3, "ios->ioproc = %d ios->num_uniontasks = %d", ios->ioproc,
+         ios->num_uniontasks));
+    pioassert(iodesc->llen == 0, "error", __FILE__, __LINE__);
+    if (ios->ioproc)
+    {
+        /* Determine llen, the length of the data array on this IO
+         * node, by multipliying the counts in the
+         * iodesc->firstregion. */
+        iodesc->llen = 1;
+        for (int i = 0; i < ndims; i++)
+        {
+            iodesc->llen *= iodesc->firstregion->count[i];
+            LOG((3, "iodesc->firstregion->start[%d] = %d iodesc->firstregion->count[%d] = %d",
+                 i, iodesc->firstregion->start[i], i, iodesc->firstregion->count[i]));
+        }
+        LOG((2, "iodesc->llen = %d", iodesc->llen));
+    }
+
+    /* Determine whether fill values will be needed. */
+    if ((ret = determine_fill(ios, iodesc, gdimlen, compmap)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((2, "iodesc->needsfill = %d ios->num_iotasks = %d", iodesc->needsfill,
+         ios->num_iotasks));
+
+    /* Set the iomaplen in the sc_info msg */
+    sc_info_msg_send[0] = iodesc->llen;
+
+    /* start/count array to be sent: 1st half for start, 2nd half for count */
+    for (int j = 0; j < ndims; j++)
+    {
+        /* The first data in sc_info_msg_send[] is the iomaplen */
+        sc_info_msg_send[j + 1] = iodesc->firstregion->start[j];
+        sc_info_msg_send[ndims + j + 1] = iodesc->firstregion->count[j];
+    }
+
+    /* Set the recvcounts/recv displs for the sc_info msg from each io task */
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        /* From each iotask all procs (compute and I/O procs) receive an
+         * sc_info message containing [iomaplen, start_for_all_dims,
+         * count_for_all_dims] and the size of this message is
+         * [sizeof(MPI_OFFSET) + ndims * sizeof(MPI_OFFSET) + ndims *
+         * sizeof(MPI_OFFSET)]
+         * Note: The displacements are in bytes
+         */
+        recvcounts[ios->ioranks[i]] = sc_info_msg_sz;
+        rdispls[ios->ioranks[i]] = i * sc_info_msg_sz * SIZEOF_MPI_OFFSET;
+    }
+
+    /* Set the sendcounts/send displs for the sc_info msg sent from each
+     * I/O task
+     */
+    for(int i=0; inum_uniontasks; i++){
+        sendcounts[i] = 0;
+        sdispls[i] = 0;
+    }
+    if(ios->ioproc){
+        /* Only I/O procs send sc_info messages */
+        for (int i = 0; i < ios->num_comptasks; i++)
+        {
+            sendcounts[ios->compranks[i]] = sc_info_msg_sz;
+            sdispls[ios->compranks[i]] = 0;
+        }
+        for (int i = 0; i < ios->num_iotasks; i++)
+        {
+            sendcounts[ios->ioranks[i]] = sc_info_msg_sz;
+            sdispls[ios->ioranks[i]] = 0;
+        }
+    }
+
+    /* Send sc_info msg from iotasks (all iotasks) to all procs(compute and I/O procs)*/
+    LOG((3, "about to call pio_swapm with start/count from iotask ndims = %d",
+         ndims));
+    if ((ret = pio_swapm(sc_info_msg_send, sendcounts, sdispls, dtypes, sc_info_msg_recv,
+                         recvcounts, rdispls, dtypes, ios->union_comm,
+                         &iodesc->rearr_opts.io2comp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+#if PIO_ENABLE_LOGGING
+    /* First entry in the sc_info msg for each iorank is the iomaplen */
+    for (int i = 0; i < ios->num_iotasks; i++)
+        LOG((3, "iomaplen[%d] = %d", i, sc_info_msg_recv[i * sc_info_msg_sz]));
+#endif /* PIO_ENABLE_LOGGING */
+
+    /* Convert a 1-D index into a global coordinate value for each data element */
+    for (int k = 0; k < maplen; k++)
+    {
+        /* The compmap array is 1 based but calculations are 0 based */
+        LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims));
+        idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord_map[k]);
+#if PIO_ENABLE_LOGGING
+        for (int d = 0; d < ndims; d++)
+            LOG((3, "gcoord_map[%d][%d] = %lld", k, d, gcoord_map[k][d]));
+#endif /* PIO_ENABLE_LOGGING */
+    }
+
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        /* First entry in the sc_info msg is the iomaplen */
+        iomaplen[i] = sc_info_msg_recv[i * sc_info_msg_sz];
+        if(iomaplen[i] > 0)
+        {
+            /* The rest of the entries in the sc_info msg are the start and
+             * count arrays
+             */
+            PIO_Offset *start = &(sc_info_msg_recv[i * sc_info_msg_sz + 1]);
+            PIO_Offset *count = &(sc_info_msg_recv[i * sc_info_msg_sz + 1 + ndims]);
+
+#if PIO_ENABLE_LOGGING
+            for (int d = 0; d < ndims; d++)
+                LOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d]));
+#endif /* PIO_ENABLE_LOGGING */
+
+            /* For each element of the data array on the compute task,
+             * find the IO task to send the data element to, and its
+             * offset into the global data array. */
+            for (int k = 0; k < maplen; k++)
+            {
+                /* An IO task has already been found for this element */
+                if (dest_ioproc[k] >= 0)
+                    continue;
+
+                PIO_Offset lcoord[ndims];
+                bool found = true;
+
+                /* Find a destination for each entry in the compmap. */
+                for (int j = 0; j < ndims; j++)
+                {
+                    if (gcoord_map[k][j] >= start[j] && gcoord_map[k][j] < start[j] + count[j])
+                    {
+                        lcoord[j] = gcoord_map[k][j] - start[j];
+                    }
+                    else
+                    {
+                        found = false;
+                        break;
+                    }
+                }
+
+                /* Did we find a destination IO task for this element
+                 * of the computation task data array? If so, remember
+                 * the destination IO task, and determine the index
+                 * for that element in the IO task data. */
+                if (found)
+                {
+                    dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count);
+                    dest_ioproc[k] = i;
+                    LOG((3, "found dest_ioindex[%d] = %d dest_ioproc[%d] = %d", k, dest_ioindex[k],
+                         k, dest_ioproc[k]));
+                }
+            }
+        }
+    }
+
+    for (int i = 0; i < maplen; i++)
+        free(gcoord_map[i]);
+    free(gcoord_map);
+    gcoord_map = NULL;
+
+    /* Check that a destination is found for each compmap entry. */
+    for (int k = 0; k < maplen; k++)
+        if (dest_ioproc[k] < 0 && compmap[k] > 0)
+        {
+            LOG((1, "Error: Found dest_ioproc[%d] = %d and compmap[%d] = %lld", k, dest_ioproc[k], k, compmap[k]));
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+        }
+
+    /* Completes the mapping for the box rearranger. */
+    LOG((2, "calling compute_counts maplen = %d", maplen));
+    if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    free(dest_ioproc);
+    free(dest_ioindex);
+    dest_ioproc = NULL;
+    dest_ioindex = NULL;
+
+    /* Compute the max io buffer size needed for an iodesc. */
+    if (ios->ioproc)
+    {
+        if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+        LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
+    }
+
+    /* Using maxiobuflen compute the maximum number of bytes that the
+     * io task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes));
+
+#ifdef TIMING
+    if ((ret = pio_stop_timer("PIO:box_rearrange_create")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif
+
+    return PIO_NOERR;
+}
+
+
+/**
+ * The box_rearrange_create algorithm optimized for the case where many
+ * iotasks have iomaplen == 0 (holes)
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param maplen the length of the map.
+ * @param compmap a 1 based array of offsets into the global space. A
+ * 0 in this array indicates a value which should not be transfered.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param ndims the number of dimensions.
+ * @param iodesc a pointer to the io_desc_t struct, which must be
+ * allocated before this function is called.
+ * @returns 0 on success, error code otherwise.
+ * @author Jim Edwards
+ */
+int box_rearrange_create_with_holes(iosystem_desc_t *ios, int maplen,
+                                    const PIO_Offset *compmap,
+                                    const int *gdimlen, int ndims,
+                                    io_desc_t *iodesc)
+{
+    int ret;
+
+#ifdef TIMING
+    /* Start timer if desired. */
+    if ((ret = pio_start_timer("PIO:box_rearrange_create_with_holes")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
+    /* Check inputs. */
+    pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc,
+              "invalid input", __FILE__, __LINE__);
+    LOG((1, "box_rearrange_create maplen = %d ndims = %d ios->num_comptasks = %d "
+         "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks));
+
+    /* Allocate arrays needed for this function. */
+    int *dest_ioproc = NULL; /* Destination IO task for each data element on compute task. */
+    PIO_Offset *dest_ioindex = NULL;    /* Offset into IO task array for each data element. */
+    PIO_Offset **gcoord_map = NULL; /* Global coordinate value for each data element. */
+    int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */
+    int sdispls[ios->num_uniontasks];    /* Send displacements for swapm. */
+    int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */
+    int rdispls[ios->num_uniontasks];    /* Receive displacements for swapm. */
+    MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */
+    PIO_Offset iomaplen[ios->num_iotasks];   /* Gets the llen of all IO tasks. */
+
+    /* This is the box rearranger. */
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    /* Number of elements of data on compute node. */
+    iodesc->ndof = maplen;
+
+    if (maplen > 0)
+    {
+        if (!(dest_ioproc = malloc(maplen * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(dest_ioindex = malloc(maplen * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(gcoord_map = malloc(maplen * sizeof(PIO_Offset*))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        for (int i = 0; i < maplen; i++)
+        {
+            if (!(gcoord_map[i] = calloc(ndims, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+    }
+
     /* Initialize array values. */
     for (int i = 0; i < maplen; i++)
     {
@@ -1289,6 +1639,18 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
         LOG((3, "iomaplen[%d] = %d", i, iomaplen[i]));
 #endif /* PIO_ENABLE_LOGGING */
 
+    /* Convert a 1-D index into a global coordinate value for each data element */
+    for (int k = 0; k < maplen; k++)
+    {
+        /* The compmap array is 1 based but calculations are 0 based */
+        LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims));
+        idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord_map[k]);
+#if PIO_ENABLE_LOGGING
+        for (int d = 0; d < ndims; d++)
+            LOG((3, "gcoord_map[%d][%d] = %lld", k, d, gcoord_map[k][d]));
+#endif /* PIO_ENABLE_LOGGING */
+    }
+
     /* For each IO task send starts/counts to all compute tasks. */
     for (int i = 0; i < ios->num_iotasks; i++)
     {
@@ -1300,8 +1662,15 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
          * compute tasks. */
         if (iomaplen[i] > 0)
         {
-            PIO_Offset start[ndims];
-            PIO_Offset count[ndims];
+            PIO_Offset start_count_send[ndims * 2];
+            PIO_Offset start_count_recv[ndims * 2];
+
+            /* start/count array to be sent: 1st half for start, 2nd half for count */
+            for (int j = 0; j < ndims; j++)
+            {
+                start_count_send[j] = iodesc->firstregion->start[j];
+                start_count_send[ndims + j] = iodesc->firstregion->count[j];
+            }
 
             /* Set up send/recv parameters for all to all gather of
              * counts and starts. */
@@ -1312,25 +1681,21 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
                 rdispls[j] = 0;
                 recvcounts[j] = 0;
                 if (ios->union_rank == ios->ioranks[i])
-                    sendcounts[j] = ndims;
+                    sendcounts[j] = ndims * 2;
             }
-            recvcounts[ios->ioranks[i]] = ndims;
+            recvcounts[ios->ioranks[i]] = ndims * 2;
 
-            /* The count array from iotask i is sent to all compute tasks. */
-            LOG((3, "about to call pio_swapm with count from iotask %d ndims = %d",
+            /* The start/count array from iotask i is sent to all compute tasks. */
+            LOG((3, "about to call pio_swapm with start/count from iotask %d ndims = %d",
                  i, ndims));
-            if ((ret = pio_swapm(iodesc->firstregion->count, sendcounts, sdispls, dtypes, count,
+            if ((ret = pio_swapm(start_count_send, sendcounts, sdispls, dtypes, start_count_recv,
                                  recvcounts, rdispls, dtypes, ios->union_comm,
                                  &iodesc->rearr_opts.io2comp)))
                 return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
-            /* The start array from iotask i is sent to all compute tasks. */
-            LOG((3, "about to call pio_swapm with start from iotask %d ndims = %d",
-                 i, ndims));
-            if ((ret = pio_swapm(iodesc->firstregion->start,  sendcounts, sdispls, dtypes,
-                                 start, recvcounts, rdispls, dtypes, ios->union_comm,
-                                 &iodesc->rearr_opts.io2comp)))
-                return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+            /* start/count array received: 1st half for start, 2nd half for count */
+            PIO_Offset *start = start_count_recv;
+            PIO_Offset *count = start_count_recv + ndims;
 
 #if PIO_ENABLE_LOGGING
             for (int d = 0; d < ndims; d++)
@@ -1342,23 +1707,19 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
              * offset into the global data array. */
             for (int k = 0; k < maplen; k++)
             {
-                PIO_Offset gcoord[ndims], lcoord[ndims];
-                bool found = true;
+                /* An IO task has already been found for this element */
+                if (dest_ioproc[k] >= 0)
+                    continue;
 
-                /* The compmap array is 1 based but calculations are 0 based */
-                LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims));
-                idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord);
-#if PIO_ENABLE_LOGGING
-                for (int d = 0; d < ndims; d++)
-                    LOG((3, "gcoord[%d] = %lld", d, gcoord[d]));
-#endif /* PIO_ENABLE_LOGGING */
+                PIO_Offset lcoord[ndims];
+                bool found = true;
 
                 /* Find a destination for each entry in the compmap. */
                 for (int j = 0; j < ndims; j++)
                 {
-                    if (gcoord[j] >= start[j] && gcoord[j] < start[j] + count[j])
+                    if (gcoord_map[k][j] >= start[j] && gcoord_map[k][j] < start[j] + count[j])
                     {
-                        lcoord[j] = gcoord[j] - start[j];
+                        lcoord[j] = gcoord_map[k][j] - start[j];
                     }
                     else
                     {
@@ -1382,6 +1743,11 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
         }
     }
 
+    for (int i = 0; i < maplen; i++)
+        free(gcoord_map[i]);
+    free(gcoord_map);
+    gcoord_map = NULL;
+
     /* Check that a destination is found for each compmap entry. */
     for (int k = 0; k < maplen; k++)
         if (dest_ioproc[k] < 0 && compmap[k] > 0)
@@ -1392,6 +1758,11 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
     if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
+    free(dest_ioproc);
+    free(dest_ioindex);
+    dest_ioproc = NULL;
+    dest_ioindex = NULL;
+
     /* Compute the max io buffer size needed for an iodesc. */
     if (ios->ioproc)
     {
@@ -1400,6 +1771,17 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
         LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
     }
 
+    /* Using maxiobuflen compute the maximum number of bytes that the
+     * io task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes));
+
+#ifdef TIMING
+    if ((ret = pio_stop_timer("PIO:box_rearrange_create_with_holes")))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+#endif /* TIMING */
+
     return PIO_NOERR;
 }
 
@@ -1554,7 +1936,7 @@ int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc)
 
     /* Create new communicators. */
     if ((mpierr = MPI_Comm_split(ios->comp_comm, color, key, &iodesc->subset_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
@@ -1638,9 +2020,9 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
 
     /* Get size of this subset communicator and rank of this task in it. */
     if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Check rank for correctness. */
     if (ios->ioproc)
@@ -1699,7 +2081,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
      * to its associated IO task. */
     if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt,
                              MPI_INT, 0, iodesc->subset_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     iodesc->llen = 0;
 
@@ -1744,7 +2126,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
     if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET,
                               srcindex, recvcounts, rdispls, PIO_OFFSET, 0,
                               iodesc->subset_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     /* On IO tasks which need it, allocate memory for the map and the
      * iomap. */
@@ -1778,7 +2160,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
      * put gathered results into iomap. */
     if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvcounts,
                               rdispls, PIO_OFFSET, 0, iodesc->subset_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     if (shrtmap != compmap)
         free(shrtmap);
@@ -1830,6 +2212,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
     }
 
     /* Handle fill values if needed. */
+    LOG((4, "ios->ioproc %d iodesc->needsfill %d", ios->ioproc, iodesc->needsfill));
     if (ios->ioproc && iodesc->needsfill)
     {
         /* we need the list of offsets which are not in the union of iomap */
@@ -1844,6 +2227,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
         thisgridsize[0] =  totalgridsize / ios->num_iotasks;
         thisgridmax[0] = thisgridsize[0];
         int xtra = totalgridsize - thisgridsize[0] * ios->num_iotasks;
+        LOG((4, "xtra %d", xtra));
 
         for (nio = 0; nio < ios->num_iotasks; nio++)
         {
@@ -1855,7 +2239,9 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
                 if (nio >= ios->num_iotasks - xtra)
                     thisgridsize[nio]++;
                 thisgridmin[nio] = thisgridmax[nio - 1] + 1;
-                thisgridmax[nio]= thisgridmin[nio] + thisgridsize[nio] - 1;
+                thisgridmax[nio] = thisgridmin[nio] + thisgridsize[nio] - 1;
+                LOG((4, "nio %d thisgridsize[nio] %d thisgridmin[nio] %d thisgridmax[nio] %d",
+                     nio, thisgridsize[nio], thisgridmin[nio], thisgridmax[nio]));
             }
             for (int i = 0; i < iodesc->llen; i++)
             {
@@ -1866,10 +2252,11 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
                         imin = i;
                 }
             }
+            LOG((4, "cnt %d", cnt));
 
             /* Gather cnt from all tasks in the IO communicator into array gcnt. */
             if ((mpierr = MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios->io_comm)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
             if (nio == ios->io_rank)
             {
@@ -1888,11 +2275,12 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
 
             if ((mpierr = MPI_Gatherv(&iomap[imin], cnt, PIO_OFFSET, myusegrid, gcnt,
                                       displs, PIO_OFFSET, nio, ios->io_comm)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         }
 
         /* Allocate and initialize a grid to fill in missing values. ??? */
         PIO_Offset grid[thisgridsize[ios->io_rank]];
+        LOG((4, "thisgridsize[ios->io_rank] %d", thisgridsize[ios->io_rank]));
         for (i = 0; i < thisgridsize[ios->io_rank]; i++)
             grid[i] = 0;
 
@@ -1902,6 +2290,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
             int j = myusegrid[i] - thisgridmin[ios->io_rank];
             pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index",
                       __FILE__, __LINE__);
+            LOG((4, "i %d myusegrid[i] %d j %d", i, myusegrid[i], j));
             if (j >= 0)
             {
                 grid[j] = 1;
@@ -1912,6 +2301,8 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
             free(myusegrid);
 
         iodesc->holegridsize = thisgridsize[ios->io_rank] - cnt;
+        LOG((3, "iodesc->holegridsize %d thisgridsize[%d] %d cnt %d", iodesc->holegridsize,
+             ios->io_rank, thisgridsize[ios->io_rank], cnt));
         if (iodesc->holegridsize > 0)
         {
             /* Allocate space for the fillgrid. */
@@ -1952,22 +2343,22 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
          * the IO communicator. */
         if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX,
                                     ios->io_comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         iodesc->maxfillregions = maxregions;
 
         /* Get the max maxholegridsize, and distribute it to all tasks
          * in the IO communicator. */
-	iodesc->maxholegridsize = iodesc->holegridsize;
+        iodesc->maxholegridsize = iodesc->holegridsize;
         if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT,
                                     MPI_MAX, ios->io_comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* Scatter values of srcindex to subset communicator. ??? */
     if ((mpierr = MPI_Scatterv((void *)srcindex, recvcounts, rdispls, PIO_OFFSET,
                                (void *)iodesc->sindex, iodesc->scount[0],  PIO_OFFSET,
                                0, iodesc->subset_comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     if (ios->ioproc)
     {
@@ -1980,7 +2371,7 @@ int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compma
         /* Get the max maxregions, and distribute it to all tasks in
          * the IO communicator. */
         if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         iodesc->maxregions = maxregions;
 
         /* Free resources. */
@@ -2022,9 +2413,11 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
     int mpierr; /* Return code for MPI calls. */
 
     assert(iodesc);
-
-    if ((mpierr = MPI_Type_size(iodesc->mpitype, &tsize)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if(iodesc->mpitype == MPI_DATATYPE_NULL)
+        tsize = 0;
+    else
+        if ((mpierr = MPI_Type_size(iodesc->mpitype, &tsize)))
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     cbuf = NULL;
     ibuf = NULL;
     if (iodesc->ndof > 0)
@@ -2041,25 +2434,24 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
         mycomm = iodesc->subset_comm;
 
     if ((mpierr = MPI_Comm_size(mycomm, &nprocs)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_rank(mycomm, &myrank)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     int log2 = log(nprocs) / log(2) + 1;
     if (!(wall = bget(2 * 4 * log2 * sizeof(double))))
         return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
     double mintime;
-    int k = 0;
 
     if ((mpierr = MPI_Barrier(mycomm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     GPTLstamp(&wall[0], &usr[0], &sys[0]);
     rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
     rearrange_io2comp(ios, iodesc, ibuf, cbuf);
     GPTLstamp(&wall[1], &usr[1], &sys[1]);
     mintime = wall[1]-wall[0];
     if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     handshake = iodesc->rearr_opts.comp2io.hs;
     isend = iodesc->isend;
@@ -2085,7 +2477,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
             {
                 iodesc->max_requests = nreqs;
                 if ((mpierr = MPI_Barrier(mycomm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 GPTLstamp(wall, usr, sys);
                 rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
                 rearrange_io2comp(ios, iodesc, ibuf, cbuf);
@@ -2093,7 +2485,7 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
                 wall[1] -= wall[0];
                 if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, wall + 1, 1, MPI_DOUBLE, MPI_MAX,
                                             mycomm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
                 if (wall[1] < mintime * 0.95)
                 {
diff --git a/src/clib/pio_spmd.c b/src/clib/pio_spmd.c
index da2eef333b08..358ee1cd6211 100644
--- a/src/clib/pio_spmd.c
+++ b/src/clib/pio_spmd.c
@@ -97,9 +97,9 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 
     /* Get my rank and size of communicator. */
     if ((mpierr = MPI_Comm_size(comm, &ntasks)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     LOG((2, "ntasks = %d my_rank = %d", ntasks, my_rank));
 
@@ -113,7 +113,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 #if PIO_ENABLE_LOGGING
     {
         for (int p = 0; p < ntasks; p++)
-            LOG((3, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d "
+            LOG((4, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d "
                  "rdispls[%d] = %d recvtypes[%d] = %d", p, sendcounts[p], p, sdispls[p], p,
                  sendtypes[p], p, recvcounts[p], p, rdispls[p], p, recvtypes[p]));
     }
@@ -127,7 +127,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
         LOG((3, "Calling MPI_Alltoallw without flow control."));
         if ((mpierr = MPI_Alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf,
                                     recvcounts, rdispls, recvtypes, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         return PIO_NOERR;
     }
 
@@ -155,17 +155,17 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
         if ((mpierr = MPI_Sendrecv(sptr, sendcounts[my_rank],sendtypes[my_rank],
                                    my_rank, tag, rptr, recvcounts[my_rank], recvtypes[my_rank],
                                    my_rank, tag, comm, &status)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 #else
         if ((mpierr = MPI_Irecv(rptr, recvcounts[my_rank], recvtypes[my_rank],
                                 my_rank, tag, comm, rcvids)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Send(sptr, sendcounts[my_rank], sendtypes[my_rank],
                                my_rank, tag, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
         if ((mpierr = MPI_Wait(rcvids, &status)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 #endif
     }
 
@@ -246,7 +246,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
             {
                 tag = my_rank + offset_t;
                 if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids + istep)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             }
         }
     }
@@ -262,11 +262,11 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 
             if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm,
                                     rcvids + istep)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
             if (fc->hs)
                 if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         }
     }
 
@@ -283,7 +283,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
             if (fc->hs)
             {
                 if ((mpierr = MPI_Wait(hs_rcvids + istep, &status)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 hs_rcvids[istep] = MPI_REQUEST_NULL;
             }
             ptr = (char *)sendbuf + sdispls[p];
@@ -301,23 +301,23 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 #ifdef USE_MPI_ISEND_FOR_FC
                 if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
                                         sndids + istep)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 #else
                 if ((mpierr = MPI_Irsend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
                                          sndids + istep)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 #endif
             }
             else if (fc->isend)
             {
                 if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
                                         sndids + istep)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             }
             else
             {
                 if ((mpierr = MPI_Send(ptr, sendcounts[p], sendtypes[p], p, tag, comm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             }
         }
 
@@ -329,7 +329,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
             if (rcvids[p] != MPI_REQUEST_NULL)
             {
                 if ((mpierr = MPI_Wait(rcvids + p, &status)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 rcvids[p] = MPI_REQUEST_NULL;
             }
             if (rstep < steps)
@@ -339,7 +339,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
                 {
                     tag = my_rank + offset_t;
                     if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep)))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 }
                 if (recvcounts[p] > 0)
                 {
@@ -347,10 +347,10 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 
                     ptr = (char *)recvbuf + rdispls[p];
                     if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm, rcvids + rstep)))
-                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                     if (fc->hs)
                         if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
-                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 }
                 rstep++;
             }
@@ -363,15 +363,33 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
     {
         LOG((2, "Waiting for outstanding msgs"));
         if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if (fc->isend)
             if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
     return PIO_NOERR;
 }
 
+/**
+ * Clean up internal data structures, and free MPI resources,
+ * associated with an IOSystem. This is the old name for
+ * PIOc_free_iosystem(). This function is maintained for backward
+ * compatibility. Use PIOc_free_iosystem() for new code.
+ *
+ * @param iosysid: the io system ID provided by PIOc_Init_Intracomm()
+ * or PIOc_init_async().
+ * @returns 0 for success or non-zero for error.
+ * @ingroup PIO_finalize_c
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_finalize(int iosysid)
+{
+    return PIOc_free_iosystem(iosysid);
+}
+
 /**
  * Provides the functionality of MPI_Gatherv with flow control
  * options. This function is not currently used, but we hope it will
@@ -421,9 +439,9 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 /*     if (fc_gather) */
 /*     { */
 /*         if ((mpierr = MPI_Comm_rank(comm, &mytask))) */
-/*             return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*         if ((mpierr = MPI_Comm_size(comm, &nprocs))) */
-/*             return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 
 /*         mtag = 2 * nprocs; */
 /*         hs = 1; */
@@ -437,7 +455,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 /*             MPI_Request rcvid[gather_block_size]; */
 
 /*             if ((mpierr = MPI_Type_size(recvtype, &dsize))) */
-/*                 return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                 return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 
 /*             for (int p = 0; p < nprocs; p++) */
 /*             { */
@@ -449,41 +467,41 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 /*                         if (count > preposts) */
 /*                         { */
 /*                             if ((mpierr = MPI_Wait(rcvid + tail, &status))) */
-/*                                 return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                                 return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*                             tail = (tail + 1) % preposts; */
 /*                         } */
 
 /*                         void *ptr = (void *)((char *)recvbuf + dsize * displs[p]); */
 
 /*                         if ((mpierr = MPI_Irecv(ptr, recvcnts[p], recvtype, p, mtag, comm, rcvid + head))) */
-/*                             return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*                         head = (head + 1) % preposts; */
 /*                         if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, mtag, comm))) */
-/*                             return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*                     } */
 /*                 } */
 /*             } */
 
 /*             /\* copy local data *\/ */
 /*             if ((mpierr = MPI_Type_size(sendtype, &dsize))) */
-/*                 return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                 return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*             if ((mpierr = MPI_Sendrecv(sendbuf, sendcnt, sendtype, mytask, 102, recvbuf, recvcnts[mytask], */
 /*                                        recvtype, mytask, 102, comm, &status))) */
-/*                 return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                 return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 
 /*             count = min(count, preposts); */
 /*             if (count > 0) */
 /*                 if ((mpierr = MPI_Waitall(count, rcvid, MPI_STATUSES_IGNORE))) */
-/*                     return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                     return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*         } */
 /*         else */
 /*         { */
 /*             if (sendcnt > 0) */
 /*             { */
 /*                 if ((mpierr = MPI_Recv(&hs, 1, MPI_INT, root, mtag, comm, &status))) */
-/*                     return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                     return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*                 if ((mpierr = MPI_Send(sendbuf, sendcnt, sendtype, root, mtag, comm))) */
-/*                     return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*                     return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*             } */
 /*         } */
 /*     } */
@@ -491,7 +509,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
 /*     { */
 /*         if ((mpierr = MPI_Gatherv(sendbuf, sendcnt, sendtype, recvbuf, recvcnts, */
 /*                                   displs, recvtype, root, comm))) */
-/*             return check_mpi(NULL, mpierr, __FILE__, __LINE__); */
+/*             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); */
 /*     } */
 
 /*     return PIO_NOERR; */
diff --git a/src/clib/pio_varm.c b/src/clib/pio_varm.c
deleted file mode 100644
index b37edc50cc50..000000000000
--- a/src/clib/pio_varm.c
+++ /dev/null
@@ -1,1824 +0,0 @@
-#include 
-#include 
-#include 
-
-///
-/// PIO interface to nc_put_varm
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_uchar
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_short
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-///
-/// PIO interface to nc_put_varm_text
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_ushort
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_ulonglong
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-///
-/// PIO interface to nc_put_varm_int
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_float
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-///
-/// PIO interface to nc_put_varm_long
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_uint
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_double
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-///
-/// PIO interface to nc_put_varm_schar
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-///
-/// PIO interface to nc_put_varm_longlong
-///
-/// This routine is called collectively by all tasks in the communicator ios.union_comm.
-///
-/// Refer to the  netcdf documentation. 
-///
-int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    var_desc_t *vdesc;
-    int *request;
-
-    ierr = PIO_NOERR;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
-            ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            if (ios->io_rank==0){
-                ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-            if ((ierr = get_var_desc(varid, &file->varlist, &vdesc)))
-                return pio_err(ios, file, ierr, __FILE__, __LINE__);
-
-            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
-                vdesc->request = realloc(vdesc->request,
-                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
-            }
-            request = vdesc->request+vdesc->nreqs;
-
-            if (ios->io_rank==0){
-                ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);;
-            }else{
-                *request = PIO_REQ_NULL;
-            }
-            vdesc->nreqs++;
-            flush_output_buffer(file, false, 0);
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    return ierr;
-}
-
-int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_UNSIGNED_CHAR;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_CHAR;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_DOUBLE;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_CHAR;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_INT;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_UNSIGNED;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibufcnt = bufcount;
-    ibuftype = buftype;
-    ierr = PIO_NOERR;
-
-    /* Sorry, but varm functions are not supported by the async interface. */
-    if (ios->async)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_FLOAT;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_LONG;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_UNSIGNED_SHORT;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_LONG_LONG;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_SHORT;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
-
-int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf)
-{
-    int ierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    MPI_Datatype ibuftype;
-    int ndims;
-    int ibufcnt;
-    bool bcast = false;
-
-    /* Get file info. */
-    if ((ierr = pio_get_file(ncid, &file)))
-        return ierr;
-    ios = file->iosystem;
-    ibuftype = MPI_UNSIGNED_LONG_LONG;
-    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
-    ibufcnt = 1;
-    for(int i=0;iasync)
-        return PIO_EINVAL;
-
-    if (ios->ioproc){
-        switch(file->iotype){
-#ifdef _NETCDF4
-        case PIO_IOTYPE_NETCDF4P:
-            ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            break;
-        case PIO_IOTYPE_NETCDF4C:
-#endif
-        case PIO_IOTYPE_NETCDF:
-            bcast = true;
-            if (ios->iomaster == MPI_ROOT){
-                ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
-            }
-            break;
-#ifdef _PNETCDF
-        case PIO_IOTYPE_PNETCDF:
-#ifdef PNET_READ_AND_BCAST
-            ncmpi_begin_indep_data(file->fh);
-            if (ios->iomaster == MPI_ROOT){
-                ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap,  buf);;
-            };
-            ncmpi_end_indep_data(file->fh);
-            bcast=true;
-#else
-            ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap,  buf);;
-#endif
-            break;
-#endif
-        default:
-            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
-        }
-    }
-
-    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
-
-    if (ios->async || bcast ||
-        (ios->num_iotasks < ios->num_comptasks)){
-        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
-    }
-
-    return ierr;
-}
diff --git a/src/clib/pioc.c b/src/clib/pioc.c
index c49a811a1ee5..b9cb05586777 100644
--- a/src/clib/pioc.c
+++ b/src/clib/pioc.c
@@ -4,13 +4,49 @@
  * @author Jim Edwards
  * @date  2014
  *
- * @see http://code.google.com/p/parallelio/
+ * @see https://github.com/NCAR/ParallelIO
  */
-
 #include 
 #include 
 #include 
 
+/**
+ * @defgroup PIO_init_c Initialize the IO System
+ * Initialize the IOSystem, including specifying number of IO and
+ * computation tasks in C.
+ *
+ * @defgroup PIO_finalize_c Shut Down the IO System
+ * Shut down an IOSystem, freeing all associated resources in C.
+ *
+ * @defgroup PIO_initdecomp_c Initialize a Decomposition
+ * Intiailize a decomposition of data into distributed arrays in C.
+ *
+ * @defgroup PIO_freedecomp_c Free a Decomposition
+ * Free a decomposition, and associated resources in C.
+ *
+ * @defgroup PIO_setframe_c Set the Record Number
+ * Set the record number for a future call to PIOc_write_darray() or
+ * PIOc_read_darray() in C.
+ *
+ * @defgroup PIO_set_hint_c Set a Hint
+ * Set an MPI Hint in C.
+ *
+ * @defgroup PIO_error_method_c Set Error Handling
+ * Set the error handling method in case error is encountered in C.
+ *
+ * @defgroup PIO_get_local_array_size_c Get the Local Size
+ * Get the local size of a distributed array in C.
+ *
+ * @defgroup PIO_iosystem_is_active_c Check IOSystem
+ * Is the IO system active (in C)?
+ *
+ * @defgroup PIO_getnumiotasks_c Get Number IO Tasks
+ * Get the Number of IO Tasks in C.
+ *
+ * @defgroup PIO_set_blocksize_c Set Blocksize
+ * Set the Blocksize in C.
+ */
+
 /** The default error handler used when iosystem cannot be located. */
 int default_error_handler = PIO_INTERNAL_ERROR;
 
@@ -18,9 +54,15 @@ int default_error_handler = PIO_INTERNAL_ERROR;
  * used (see pio_sc.c). */
 extern int blocksize;
 
-/* Used when assiging decomposition IDs. */
+/** Used when assiging decomposition IDs. */
 int pio_next_ioid = 512;
 
+/** Sort map. */
+struct sort_map {
+    int remap;
+    PIO_Offset map;
+};
+
 /**
  * Check to see if PIO has been initialized.
  *
@@ -28,9 +70,11 @@ int pio_next_ioid = 512;
  * @param active pointer that gets true if IO system is active, false
  * otherwise.
  * @returns 0 on success, error code otherwise
+ * @ingroup PIO_iosystem_is_active_c
  * @author Jim Edwards
  */
-int PIOc_iosystem_is_active(int iosysid, bool *active)
+int
+PIOc_iosystem_is_active(int iosysid, bool *active)
 {
     iosystem_desc_t *ios;
 
@@ -53,9 +97,11 @@ int PIOc_iosystem_is_active(int iosysid, bool *active)
  *
  * @param ncid the ncid of an open file
  * @returns 1 if file is open, 0 otherwise.
+ * @ingroup PIO_file_open_c
  * @author Jim Edwards
  */
-int PIOc_File_is_Open(int ncid)
+int
+PIOc_File_is_Open(int ncid)
 {
     file_desc_t *file;
 
@@ -79,10 +125,11 @@ int PIOc_File_is_Open(int ncid)
  * @param ncid the ncid of an open file
  * @param method the error handling method
  * @returns old error handler
- * @ingroup PIO_error_method
+ * @ingroup PIO_error_method_c
  * @author Jim Edwards
  */
-int PIOc_Set_File_Error_Handling(int ncid, int method)
+int
+PIOc_Set_File_Error_Handling(int ncid, int method)
 {
     file_desc_t *file;
     int oldmethod;
@@ -111,9 +158,11 @@ int PIOc_Set_File_Error_Handling(int ncid, int method)
  * @param ncid the ncid of the open file
  * @param varid the variable ID
  * @returns 0 on success, error code otherwise
+ * @ingroup PIO_setframe_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_advanceframe(int ncid, int varid)
+int
+PIOc_advanceframe(int ncid, int varid)
 {
     iosystem_desc_t *ios;     /* Pointer to io system information. */
     file_desc_t *file;        /* Pointer to file information. */
@@ -121,7 +170,7 @@ int PIOc_advanceframe(int ncid, int varid)
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
     int ret;
 
-    LOG((1, "PIOc_advanceframe ncid = %d varid = %d"));
+    LOG((1, "PIOc_advanceframe ncid = %d varid = %d", ncid, varid));
 
     /* Get the file info. */
     if ((ret = pio_get_file(ncid, &file)))
@@ -150,9 +199,9 @@ int PIOc_advanceframe(int ncid, int varid)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* Increment the record number. */
@@ -170,10 +219,11 @@ int PIOc_advanceframe(int ncid, int varid)
  * @param frame the value of the unlimited dimension.  In c 0 for the
  * first record, 1 for the second
  * @return PIO_NOERR for no error, or error code.
- * @ingroup PIO_setframe
+ * @ingroup PIO_setframe_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_setframe(int ncid, int varid, int frame)
+int
+PIOc_setframe(int ncid, int varid, int frame)
 {
     iosystem_desc_t *ios;     /* Pointer to io system information. */
     file_desc_t *file;        /* Pointer to file information. */
@@ -213,14 +263,13 @@ int PIOc_setframe(int ncid, int varid, int frame)
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* Set the record dimension value for this variable. This will be
      * used by the write_darray functions. */
-    /* file->varlist[varid].record = frame; */
     vdesc->record = frame;
 
     return PIO_NOERR;
@@ -233,9 +282,11 @@ int PIOc_setframe(int ncid, int varid, int frame)
  * @param numiotasks a pointer taht gets the number of IO
  * tasks. Ignored if NULL.
  * @returns 0 on success, error code otherwise
+ * @ingroup PIO_getnumiotasks_c
  * @author Ed Hartnett
  */
-int PIOc_get_numiotasks(int iosysid, int *numiotasks)
+int
+PIOc_get_numiotasks(int iosysid, int *numiotasks)
 {
     iosystem_desc_t *ios;
 
@@ -253,9 +304,11 @@ int PIOc_get_numiotasks(int iosysid, int *numiotasks)
  *
  * @param ioid IO descrption ID.
  * @returns the size of the array.
+ * @ingroup PIO_get_local_array_size_c
  * @author Jim Edwards
  */
-int PIOc_get_local_array_size(int ioid)
+int
+PIOc_get_local_array_size(int ioid)
 {
     io_desc_t *iodesc;
 
@@ -274,10 +327,11 @@ int PIOc_get_local_array_size(int ioid)
  * @param iosysid the IO system ID
  * @param method the error handling method
  * @returns old error handler
- * @ingroup PIO_error_method
+ * @ingroup PIO_error_method_c
  * @author Jim Edwards
  */
-int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
+int
+PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
 {
     iosystem_desc_t *ios;
     int oldmethod;
@@ -304,10 +358,11 @@ int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
  * @param old_method pointer to int that will get old method. Ignored
  * if NULL.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_error_method
+ * @ingroup PIO_error_method_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
+int
+PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
 {
     iosystem_desc_t *ios = NULL;
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
@@ -345,9 +400,9 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
 
             /* Handle MPI errors. */
             if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-                check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+                check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
             if (mpierr)
-                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         }
 
     /* Return the current handler. */
@@ -363,6 +418,27 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
     return PIO_NOERR;
 }
 
+/**
+ * Compare.
+ *
+ * @param a pointer to a
+ * @param b pointer to b
+ * @return -1 if a.map < b.map, 1 if a.map > b.map, 0 if equal
+ * @author Jim Edwards
+ */
+int
+compare( const void* a, const void* b)
+{
+    struct sort_map l_a = * ( (struct sort_map *) a );
+    struct sort_map l_b = * ( (struct sort_map *) b );
+
+    if ( l_a.map < l_b.map )
+        return -1;
+    else if ( l_a.map > l_b.map )
+        return 1;
+    return 0;
+}
+
 /**
  * Initialize the decomposition used with distributed arrays. The
  * decomposition describes how the data will be distributed between
@@ -404,12 +480,13 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
  * rearranger is used. If NULL and SUBSET rearranger is used, the
  * iostarts are generated.
  * @returns 0 on success, error code otherwise
- * @ingroup PIO_initdecomp
+ * @ingroup PIO_initdecomp_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
-                    const PIO_Offset *compmap, int *ioidp, const int *rearranger,
-                    const PIO_Offset *iostart, const PIO_Offset *iocount)
+int
+PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                const PIO_Offset *compmap, int *ioidp, const int *rearranger,
+                const PIO_Offset *iostart, const PIO_Offset *iocount)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     io_desc_t *iodesc;     /* The IO description. */
@@ -478,9 +555,9 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* Allocate space for the iodesc info. This also allocates the
@@ -496,9 +573,48 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
     /* Remember the map. */
     if (!(iodesc->map = malloc(sizeof(PIO_Offset) * maplen)))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    iodesc->needssort = false;
+    iodesc->remap = NULL;
     for (int m = 0; m < maplen; m++)
-        iodesc->map[m] = compmap[m];
+    {
+        if(m > 0 && compmap[m] > 0 && compmap[m] < compmap[m-1])
+        {
+            iodesc->needssort = true;
+            LOG((2, "compmap[%d] = %ld compmap[%d]= %ld", m, compmap[m], m-1, compmap[m-1]));
+            break;
+        }
+    }
+    if (iodesc->needssort)
+    {
+        struct sort_map *tmpsort;
 
+        if (!(tmpsort = malloc(sizeof(struct sort_map) * maplen)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(iodesc->remap = malloc(sizeof(int) * maplen)))
+        {
+            free(tmpsort);
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+        for (int m=0; m < maplen; m++)
+        {
+            tmpsort[m].remap = m;
+            tmpsort[m].map = compmap[m];
+        }
+        qsort( tmpsort, maplen, sizeof(struct sort_map), compare );
+        for (int m=0; m < maplen; m++)
+        {
+            iodesc->map[m] = compmap[tmpsort[m].remap];
+            iodesc->remap[m] = tmpsort[m].remap;
+        }
+        free(tmpsort);
+    }
+    else
+    {
+        for (int m=0; m < maplen; m++)
+        {
+            iodesc->map[m] = compmap[m];
+        }
+    }
     /* Remember the dim sizes. */
     if (!(iodesc->dimlen = malloc(sizeof(int) * ndims)))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
@@ -518,7 +634,7 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
         iodesc->num_aiotasks = ios->num_iotasks;
         LOG((2, "creating subset rearranger iodesc->num_aiotasks = %d",
              iodesc->num_aiotasks));
-        if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, gdimlen,
+        if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)iodesc->map, gdimlen,
                                             ndims, iodesc)))
             return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
     }
@@ -559,12 +675,12 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
          * of io tasks used may vary. */
         if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot,
                                 ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "iodesc->num_aiotasks = %d", iodesc->num_aiotasks));
 
         /* Compute the communications pattern for this decomposition. */
         if (iodesc->rearranger == PIO_REARR_BOX)
-            if ((ierr = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+            if ((ierr = box_rearrange_create(ios, maplen, iodesc->map, gdimlen, ndims, iodesc)))
                 return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
     }
 
@@ -573,7 +689,7 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
     {
         LOG((3, "createfile bcasting pio_next_ioid %d", pio_next_ioid));
         if ((mpierr = MPI_Bcast(&pio_next_ioid, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "createfile bcast pio_next_ioid %d", pio_next_ioid));
     }
 
@@ -628,15 +744,17 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
  * @param iocount An array of count values for block cyclic
  * decompositions. If NULL ???
  * @returns 0 on success, error code otherwise
- * @ingroup PIO_initdecomp
+ * @ingroup PIO_initdecomp_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
-                     const PIO_Offset *compmap, int *ioidp, int rearranger,
-                     const PIO_Offset *iostart, const PIO_Offset *iocount)
+int
+PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                 const PIO_Offset *compmap, int *ioidp, int rearranger,
+                 const PIO_Offset *iostart, const PIO_Offset *iocount)
 {
-    PIO_Offset compmap_1_based[maplen];
+    PIO_Offset *compmap_1_based;
     int *rearrangerp = NULL;
+    int ret;
 
     LOG((1, "PIOc_init_decomp iosysid = %d pio_type = %d ndims = %d maplen = %d",
          iosysid, pio_type, ndims, maplen));
@@ -645,6 +763,10 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
     if (rearranger)
         rearrangerp = &rearranger;
 
+    /* Allocate storage for compmap that's one-based. */
+    if (!(compmap_1_based = malloc(sizeof(PIO_Offset) * maplen)))
+        return PIO_ENOMEM;
+
     /* Add 1 to all elements in compmap. */
     for (int e = 0; e < maplen; e++)
     {
@@ -653,8 +775,12 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
     }
 
     /* Call the legacy version of the function. */
-    return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based,
-                           ioidp, rearrangerp, iostart, iocount);
+    ret = PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based,
+                          ioidp, rearrangerp, iostart, iocount);
+
+    free(compmap_1_based);
+
+    return ret;
 }
 
 /**
@@ -669,13 +795,14 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
  * dimensions.
  * @param start start array
  * @param count count array
- * @param pointer that gets the IO ID.
+ * @param ioidp pointer that gets the IO ID.
  * @returns 0 for success, error code otherwise
- * @ingroup PIO_initdecomp
+ * @ingroup PIO_initdecomp_c
  * @author Jim Edwards
  */
-int PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen,
-                       const long int *start, const long int *count, int *ioidp)
+int
+PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen,
+                   const long int *start, const long int *count, int *ioidp)
 
 {
     iosystem_desc_t *ios;
@@ -774,11 +901,12 @@ int PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen,
  * until the decomposition is initialized.
  * @param iosysidp index of the defined system descriptor.
  * @return 0 on success, otherwise a PIO error code.
- * @ingroup PIO_init
+ * @ingroup PIO_init_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base,
-                        int rearr, int *iosysidp)
+int
+PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base,
+                    int rearr, int *iosysidp)
 {
     iosystem_desc_t *ios;
     int ustride;
@@ -789,11 +917,12 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
     int ret;           /* Return code for function calls. */
 
     /* Turn on the logging system. */
-    pio_init_logging();
+    if ((ret = pio_init_logging()))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
 
     /* Find the number of computation tasks. */
     if ((mpierr = MPI_Comm_size(comp_comm, &num_comptasks)))
-        return check_mpi2(NULL, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     /* Check the inputs. */
     if (!iosysidp || num_iotasks < 1 || num_iotasks * stride > num_comptasks)
@@ -822,11 +951,11 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
 
     /* Copy the computation communicator into union_comm. */
     if ((mpierr = MPI_Comm_dup(comp_comm, &ios->union_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Copy the computation communicator into comp_comm. */
     if ((mpierr = MPI_Comm_dup(comp_comm, &ios->comp_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "union_comm = %d comp_comm = %d", ios->union_comm, ios->comp_comm));
 
     ios->my_comm = ios->comp_comm;
@@ -834,7 +963,7 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
 
     /* Find MPI rank in comp_comm communicator. */
     if ((mpierr = MPI_Comm_rank(ios->comp_comm, &ios->comp_rank)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* With non-async, all tasks are part of computation component. */
     ios->compproc = true;
@@ -873,16 +1002,16 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
 
     /* Create a group for the computation tasks. */
     if ((mpierr = MPI_Comm_group(ios->comp_comm, &compgroup)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Create a group for the IO tasks. */
     if ((mpierr = MPI_Group_incl(compgroup, ios->num_iotasks, ios->ioranks,
                                  &iogroup)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);    
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Create an MPI communicator for the IO tasks. */
     if ((mpierr = MPI_Comm_create(ios->comp_comm, iogroup, &ios->io_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Free the MPI groups. */
     if (compgroup != MPI_GROUP_NULL)
@@ -897,7 +1026,7 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
     if (ios->ioproc)
     {
         if ((mpierr = MPI_Comm_rank(ios->io_comm, &ios->io_rank)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     }
     else
         ios->io_rank = -1;
@@ -929,12 +1058,14 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas
  * @param rearr_opts the rearranger options
  * @param iosysidp a pointer that gets the IO system ID
  * @returns 0 for success, error code otherwise
+ * @ingroup PIO_init_c
  * @author Jim Edwards
  */
-int PIOc_Init_Intracomm_from_F90(int f90_comp_comm,
-                                 const int num_iotasks, const int stride,
-                                 const int base, const int rearr,
-                                 rearr_opt_t *rearr_opts, int *iosysidp)
+int
+PIOc_Init_Intracomm_from_F90(int f90_comp_comm,
+                             const int num_iotasks, const int stride,
+                             const int base, const int rearr,
+                             rearr_opt_t *rearr_opts, int *iosysidp)
 {
     int ret = PIO_NOERR;
     ret = PIOc_Init_Intracomm(MPI_Comm_f2c(f90_comp_comm), num_iotasks,
@@ -968,9 +1099,11 @@ int PIOc_Init_Intracomm_from_F90(int f90_comp_comm,
  * @param hint the hint for MPI
  * @param hintval the value of the hint
  * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ * @ingroup PIO_set_hint_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_set_hint(int iosysid, const char *hint, const char *hintval)
+int
+PIOc_set_hint(int iosysid, const char *hint, const char *hintval)
 {
     iosystem_desc_t *ios;
     int mpierr; /* Return value for MPI calls. */
@@ -988,26 +1121,28 @@ int PIOc_set_hint(int iosysid, const char *hint, const char *hintval)
     /* Make sure we have an info object. */
     if (ios->info == MPI_INFO_NULL)
         if ((mpierr = MPI_Info_create(&ios->info)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Set the MPI hint. */
     if (ios->ioproc)
-        if ((mpierr = MPI_Info_set(ios->info, hint, hintval)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Info_set(ios->info, (char *)hint, (char *)hintval)))
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     return PIO_NOERR;
 }
 
 /**
- * Clean up internal data structures, free MPI resources, and exit the
- * pio library.
+ * Clean up internal data structures, and free MPI resources,
+ * associated with an IOSystem.
  *
- * @param iosysid: the io system ID provided by PIOc_Init_Intracomm().
+ * @param iosysid: the io system ID provided by PIOc_Init_Intracomm()
+ * or PIOc_init_async().
  * @returns 0 for success or non-zero for error.
- * @ingroup PIO_finalize
+ * @ingroup PIO_finalize_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_finalize(int iosysid)
+int
+PIOc_free_iosystem(int iosysid)
 {
     iosystem_desc_t *ios;
     int niosysid;          /* The number of currently open IO systems. */
@@ -1048,9 +1183,9 @@ int PIOc_finalize(int iosysid)
         /* Handle MPI errors. */
         LOG((3, "handling async errors mpierr = %d my_comm = %d", mpierr, ios->my_comm));
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
         LOG((3, "async errors bcast"));
     }
 
@@ -1077,7 +1212,8 @@ int PIOc_finalize(int iosysid)
     /* Free the MPI communicators. my_comm is just a copy (but not an
      * MPI copy), so does not have to have an MPI_Comm_free()
      * call. comp_comm and io_comm are MPI duplicates of the comms
-     * handed into init_intercomm. So they need to be freed by MPI. */
+     * handed into PIOc_init_async(). So they need to be freed by
+     * MPI. */
     if (ios->intercomm != MPI_COMM_NULL)
         MPI_Comm_free(&ios->intercomm);
     if (ios->union_comm != MPI_COMM_NULL)
@@ -1115,9 +1251,11 @@ int PIOc_finalize(int iosysid)
  * @param ioproc a pointer that gets 1 if task is an IO task, 0
  * otherwise. Ignored if NULL.
  * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ * @ingroup PIO_iosystem_is_active_c
  * @author Jim Edwards
  */
-int PIOc_iam_iotask(int iosysid, bool *ioproc)
+int
+PIOc_iam_iotask(int iosysid, bool *ioproc)
 {
     iosystem_desc_t *ios;
 
@@ -1138,9 +1276,11 @@ int PIOc_iam_iotask(int iosysid, bool *ioproc)
  * @param iorank a pointer that gets the io rank, or -1 if task is not
  * in the IO communicator. Ignored if NULL.
  * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ * @ingroup PIO_iosystem_is_active_c
  * @author Jim Edwards
  */
-int PIOc_iotask_rank(int iosysid, int *iorank)
+int
+PIOc_iotask_rank(int iosysid, int *iorank)
 {
     iosystem_desc_t *ios;
 
@@ -1160,7 +1300,8 @@ int PIOc_iotask_rank(int iosysid, int *iorank)
  * @returns 1 if iotype is in build, 0 if not.
  * @author Jim Edwards
  */
-int PIOc_iotype_available(int iotype)
+int
+PIOc_iotype_available(int iotype)
 {
     switch(iotype)
     {
@@ -1257,16 +1398,17 @@ int PIOc_iotype_available(int iotype)
  * gets the iosysid for each component.
  *
  * @return PIO_NOERR on success, error code otherwise.
- * @ingroup PIO_init
+ * @ingroup PIO_init_c
  * @author Ed Hartnett
  */
-int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
-                    int component_count, int *num_procs_per_comp, int **proc_list,
-                    MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int rearranger,
-                    int *iosysidp)
+int
+PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
+                int component_count, int *num_procs_per_comp, int **proc_list,
+                MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int rearranger,
+                int *iosysidp)
 {
     int my_rank;          /* Rank of this task. */
-    int *my_proc_list[component_count];   /* Array of arrays of procs for comp components. */
+    int **my_proc_list;   /* Array of arrays of procs for comp components. */
     int my_io_proc_list[num_io_procs]; /* List of processors in IO component. */
     int mpierr;           /* Return code from MPI functions. */
     int ret;              /* Return code. */
@@ -1276,8 +1418,11 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         (rearranger != PIO_REARR_BOX))
         return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
+    my_proc_list = (int**) malloc(component_count * sizeof(int*));
+
     /* Turn on the logging system for PIO. */
-    pio_init_logging();
+    if ((ret = pio_init_logging()))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
     LOG((1, "PIOc_init_async num_io_procs = %d component_count = %d", num_io_procs,
          component_count));
 
@@ -1288,11 +1433,11 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
     /* Determine which tasks to use for each computational component. */
     if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp,
                                proc_list, my_proc_list)))
-        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);        
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
 
     /* Get rank of this task in world. */
     if ((ret = MPI_Comm_rank(world, &my_rank)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
     /* Is this process in the IO component? */
     int pidx;
@@ -1311,7 +1456,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
     /* Create group for world. */
     MPI_Group world_group;
     if ((ret = MPI_Comm_group(world, &world_group)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
     LOG((3, "world group created"));
 
     /* We will create a group for the IO component. */
@@ -1329,12 +1474,12 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
 
     /* Create a group for the IO component. */
     if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
     LOG((3, "created IO group - io_group = %d MPI_GROUP_EMPTY = %d", io_group, MPI_GROUP_EMPTY));
 
     /* There is one shared IO comm. Create it. */
     if ((ret = MPI_Comm_create(world, io_group, &io_comm)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
     LOG((3, "created io comm io_comm = %d", io_comm));
 
     /* Does the user want a copy of the IO communicator? */
@@ -1343,7 +1488,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         *user_io_comm = MPI_COMM_NULL;
         if (in_io)
             if ((mpierr = MPI_Comm_dup(io_comm, user_io_comm)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
     /* For processes in the IO component, get their rank within the IO
@@ -1352,7 +1497,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
     {
         LOG((3, "about to get io rank"));
         if ((ret = MPI_Comm_rank(io_comm, &io_rank)))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
         iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL;
         LOG((3, "intracomm created for io_comm = %d io_rank = %d IO %s",
              io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT"));
@@ -1401,7 +1546,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         /* Create a group for this component. */
         if ((ret = MPI_Group_incl(world_group, num_procs_per_comp[cmp], my_proc_list[cmp],
                                   &group[cmp])))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
         LOG((3, "created component MPI group - group[%d] = %d", cmp, group[cmp]));
 
         /* For all the computation components create a union group
@@ -1454,7 +1599,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
 
         /* Create the union group. */
         if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, &union_group[cmp])))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
         LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp,
              union_group[cmp], nprocs_union));
 
@@ -1463,18 +1608,18 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
          * call. */
         LOG((3, "creating intracomm cmp = %d from group[%d] = %d", cmp, cmp, group[cmp]));
         if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm)))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
         if (in_cmp)
         {
             /* Does the user want a copy? */
             if (user_comp_comm)
                 if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp])))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
             /* Get the rank in this comp comm. */
             if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank)))
-                return check_mpi(NULL, ret, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
             /* Set comp_rank 0 to be the compmaster. It will have a
              * setting of MPI_ROOT, all other tasks will have a
@@ -1492,7 +1637,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
         {
             LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank));
             if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm)))
-                return check_mpi(NULL, ret, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
             LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank));
             my_iosys->iomaster = iomaster;
             my_iosys->io_rank = io_rank;
@@ -1512,15 +1657,15 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
          * are part of the union_comm. */
         LOG((3, "before creating union_comm my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp]));
         if ((ret = MPI_Comm_create(world, union_group[cmp], &my_iosys->union_comm)))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
         LOG((3, "created union comm for cmp %d my_iosys->union_comm %d", cmp, my_iosys->union_comm));
 
         if (in_io || in_cmp)
         {
             if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank)))
-                return check_mpi(NULL, ret, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
             LOG((3, "my_iosys->union_rank %d", my_iosys->union_rank));
-            
+
             /* Set my_comm to union_comm for async. */
             my_iosys->my_comm = my_iosys->union_comm;
             LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d",
@@ -1534,7 +1679,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
                      "my_iosys->io_comm = %d", cmp, my_iosys->io_comm));
                 if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm,
                                                 my_iosys->num_iotasks, cmp, &my_iosys->intercomm)))
-                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
             }
             else
             {
@@ -1543,7 +1688,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
                      my_iosys->comp_comm));
                 if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm,
                                                 0, cmp, &my_iosys->intercomm)))
-                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
             }
             LOG((3, "intercomm created for cmp = %d", cmp));
         }
@@ -1568,28 +1713,30 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
     /* Free resources if needed. */
     if (in_io)
         if ((mpierr = MPI_Comm_free(&io_comm)))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
     /* Free the arrays of processor numbers. */
     for (int cmp = 0; cmp < component_count; cmp++)
         free(my_proc_list[cmp]);
 
+    free(my_proc_list);
+
     /* Free MPI groups. */
     if ((ret = MPI_Group_free(&io_group)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
     for (int cmp = 0; cmp < component_count; cmp++)
     {
         if ((ret = MPI_Group_free(&group[cmp])))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
         if ((ret = MPI_Group_free(&union_group[cmp])))
-            return check_mpi(NULL, ret, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
     }
 
     if ((ret = MPI_Group_free(&world_group)))
-        return check_mpi(NULL, ret, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, ret, __FILE__, __LINE__);
 
-    LOG((2, "successfully done with PIO_Init_Async"));
+    LOG((2, "successfully done with PIOc_init_async"));
     return PIO_NOERR;
 }
 
@@ -1598,10 +1745,11 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
  *
  * @param newblocksize the new blocksize.
  * @returns 0 for success.
- * @ingroup PIO_set_blocksize
+ * @ingroup PIO_set_blocksize_c
  * @author Jim Edwards
  */
-int PIOc_set_blocksize(int newblocksize)
+int
+PIOc_set_blocksize(int newblocksize)
 {
     if (newblocksize > 0)
         blocksize = newblocksize;
diff --git a/src/clib/pioc_sc.c b/src/clib/pioc_sc.c
index 1efae6c8c3ef..e5c5db4840ff 100644
--- a/src/clib/pioc_sc.c
+++ b/src/clib/pioc_sc.c
@@ -9,12 +9,12 @@
 #include 
 #include 
 
-/** The default target blocksize for each io task when the box
+/** The default target blocksize in bytes for each io task when the box
  * rearranger is used. */
 #define DEFAULT_BLOCKSIZE 1024
 
-/** The target blocksize for each io task when the box rearranger is
- * used. */
+/** The target blocksize in bytes for each io task when the box
+ * rearranger is used. */
 int blocksize = DEFAULT_BLOCKSIZE;
 
 /**
@@ -22,8 +22,7 @@ int blocksize = DEFAULT_BLOCKSIZE;
  *
  * @param a
  * @param b
- * @returns greates common divisor.
- * @author Jim Edwards
+ * @returns greatest common divisor.
  */
 int gcd(int a, int b )
 {
@@ -38,8 +37,7 @@ int gcd(int a, int b )
  *
  * @param a
  * @param b
- * @returns greates common divisor.
- * @author Jim Edwards
+ * @returns greatest common divisor.
  */
 long long lgcd(long long a, long long b)
 {
@@ -54,7 +52,6 @@ long long lgcd(long long a, long long b)
  * @param nain number of elements in ain.
  * @param ain array of length nain.
  * @returns GCD of elements in ain.
- * @author Jim Edwards
  */
 long long lgcd_array(int nain, long long *ain)
 {
@@ -85,7 +82,6 @@ long long lgcd_array(int nain, long long *ain)
  * @param rank IO rank of this task.
  * @param start pointer to PIO_Offset that will get the start value.
  * @param count pointer to PIO_Offset that will get the count value.
- * @author Jim Edwards
  */
 void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start,
                      PIO_Offset *count)
@@ -126,105 +122,57 @@ void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start,
 
 /**
  * Look for the largest block of data for io which can be expressed in
- * terms of start and count.
+ * terms of start and count (ignore gaps).
  *
  * @param arrlen
  * @param arr_in
  * @returns the size of the block
- * @author Jim Edwards
  */
 PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
 {
-    int numblks = 0;  /* Number of blocks. */
-    int numtimes = 0; /* Number of times adjacent arr_in elements differ by != 1. */
-    int numgaps = 0;  /* Number of gaps. */
-    int j;  /* Loop counter. */
-    int ii; /* Loop counter. */
-    int n;
-    PIO_Offset bsize;     /* Size of the block. */
-    PIO_Offset bsizeg;    /* Size of gap block. */
-    PIO_Offset blklensum; /* Sum of all block lengths. */
-    PIO_Offset del_arr[arrlen - 1]; /* Array of deltas between adjacent elements in arr_in. */
-    PIO_Offset loc_arr[arrlen - 1];
-
     /* Check inputs. */
-    pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__);
+    pioassert(arrlen > 0 && arr_in && arr_in[0] >= 0, "invalid input", __FILE__, __LINE__);
 
-    /* Count the number of contiguous blocks in arr_in. If any if
-       these blocks is of size 1, we are done and can return.
-       Otherwise numtimes is the number of blocks. */
-    for (int i = 0; i < arrlen - 1; i++)
-    {
-        del_arr[i] = arr_in[i + 1] - arr_in[i];
-        if (del_arr[i] != 1)
-        {
-            numtimes++;
-            if ( i > 0 && del_arr[i - 1] > 1)
-                return(1);
-        }
-    }
+    /* If theres is only one contiguous block with length 1,
+     * the result must be 1 and we can return. */
+    if (arrlen == 1)
+        return 1;
 
-    /* If numtimes is 0 the all of the data in arr_in is contiguous
-     * and numblks=1. Not sure why I have three different variables
-     * here, seems like n,numblks and numtimes could be combined. */
-    numblks = numtimes + 1;
-    if (numtimes == 0)
-        n = numblks;
-    else
-        n = numtimes;
+    /* We can use the array length as the initial value.
+     * Suppose we have n contiguous blocks with lengths
+     * b1, b2, ..., bn, then gcd(b1, b2, ..., bn) =
+     * gcd(b1 + b2 + ... + bn, b1, b2, ..., bn) =
+     * gcd(arrlen, b1, b2, ..., bn) */
+    PIO_Offset bsize = arrlen;
+
+    /* The minimum length of a block is 1. */
+    PIO_Offset blk_len = 1;
 
-    /* If numblks==1 then the result is arrlen and you can return. */
-    bsize = (PIO_Offset)arrlen;
-    if (numblks > 1)
+    for (int i = 0; i < arrlen - 1; i++)
     {
-        PIO_Offset blk_len[numblks];
-        PIO_Offset gaps[numtimes];
+        pioassert(arr_in[i + 1] >= 0, "invalid input", __FILE__, __LINE__);
 
-        /* If numblks > 1 then numtimes must be > 0 and this if block
-         * isn't needed. */
-        if (numtimes > 0)
+        if ((arr_in[i + 1] - arr_in[i]) == 1)
         {
-            ii = 0;
-            for (int i = 0; i < arrlen - 1; i++)
-                if (del_arr[i] > 1)
-                    gaps[ii++] = del_arr[i] - 1;
-            numgaps = ii;
+            /* Still in a contiguous block. */
+            blk_len++;
         }
-
-        j = 0;
-        for (int i = 0; i < n; i++)
-            loc_arr[i] = 1;
-
-        for (int i = 0; i < arrlen - 1; i++)
-            if(del_arr[i] != 1)
-                loc_arr[j++] = i;
-
-        blk_len[0] = loc_arr[0];
-        blklensum = blk_len[0];
-        for(int i = 1; i < numblks - 1; i++)
+        else
         {
-            blk_len[i] = loc_arr[i] - loc_arr[i - 1];
-            blklensum += blk_len[i];
-        }
-        blk_len[numblks - 1] = arrlen - blklensum;
+            /* The end of a block has been reached. */
+            if (blk_len == 1)
+                return 1;
 
-        /* Get the GCD in blk_len array. */
-        bsize = lgcd_array(numblks, blk_len);
+            bsize = lgcd(bsize, blk_len);
+            if (bsize == 1)
+              return 1;
 
-        /* I don't recall why i needed these next two blocks, I
-         * remember struggling to get this right in all cases and I'm
-         * afraid that the end result is that bsize is almost always
-         * 1. */
-        if (numgaps > 0)
-        {
-            bsizeg = lgcd_array(numgaps, gaps);
-            bsize = lgcd(bsize, bsizeg);
+            /* Continue to find next block. */
+            blk_len = 1;
         }
-
-        /* ??? */
-        if (arr_in[0] > 0)
-            bsize = lgcd(bsize, arr_in[0]);
     }
+    /* Handle the last block. */
+    bsize = lgcd(bsize, blk_len);
 
     return bsize;
 }
@@ -243,7 +191,6 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
  * @param count array of length ndims with data count values.
  * @param num_aiotasks the number of IO tasks used(?)
  * @returns 0 for success, error code otherwise.
- * @author Jim Edwards
  */
 int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs,
                       int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks)
@@ -275,9 +222,8 @@ int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_proc
     /* We are trying to find start and count indices for each iotask
      * such that each task has approximately blocksize data to write
      * (read). The number of iotasks participating in the operation is
-     * blocksize/global_size. */
+     * global_size/blocksize. */
     minbytes = blocksize - 256;
-    maxbytes = blocksize + 256;
 
     /* Determine the size of the data type. */
     if ((ret = find_mpi_type(pio_type, NULL, &basesize)))
@@ -295,6 +241,8 @@ int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_proc
      * blocksize data on each iotask*/
     use_io_procs = max(1, min((int)((float)pgdims / (float)minblocksize + 0.5), num_io_procs));
 
+    maxbytes = max(blocksize, pgdims * basesize / use_io_procs) + 256;
+
     /* Initialize to 0. */
     converged = 0;
     for (i = 0; i < ndims; i++)
diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 9bee1ec65a2e..c7811dab2c0b 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -11,8 +11,13 @@
 
 #include 
 
+/** This is used with text decomposition files. */
 #define VERSNO 2001
 
+/** In decomposition files, backtraces are included. This is the max
+ * number of trace levels that will be used. */
+#define MAX_BACKTRACE 10
+
 /* Some logging constants. */
 #if PIO_ENABLE_LOGGING
 #define MAX_LOG_MSG 1024
@@ -34,6 +39,38 @@ extern int pio_next_ncid;
 /** The default error handler used when iosystem cannot be located. */
 extern int default_error_handler;
 
+/**
+ * Start the PIO timer.
+ *
+ * @param name name of the timer.
+ * @return 0 for success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+pio_start_timer(const char *name)
+{
+#ifdef TIMING
+    GPTLstart(name);
+#endif /* TIMING */
+    return PIO_NOERR;
+}
+
+/**
+ * Stop the PIO timer.
+ *
+ * @param name name of the timer.
+ * @return 0 for success, error code otherwise.
+ * @author Ed Hartnett
+ */
+int
+pio_stop_timer(const char *name)
+{
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_comp2io");
+#endif /* TIMING */
+    return PIO_NOERR;
+}
+
 /**
  * Return a string description of an error code. If zero is passed,
  * the errmsg will be "No error".
@@ -42,8 +79,10 @@ extern int default_error_handler;
  * @param errmsg Pointer that will get the error message. The message
  * will be PIO_MAX_NAME chars or less.
  * @return 0 on success.
+ * @author Jim Edwards
  */
-int PIOc_strerror(int pioerr, char *errmsg)
+int
+PIOc_strerror(int pioerr, char *errmsg)
 {
     LOG((1, "PIOc_strerror pioerr = %d", pioerr));
 
@@ -63,11 +102,11 @@ int PIOc_strerror(int pioerr, char *errmsg)
         strcpy(errmsg, "No error");
 #if defined(_NETCDF)
     else if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR)     /* NetCDF error? */
-        strncpy(errmsg, nc_strerror(pioerr), NC_MAX_NAME);
+        strncpy(errmsg, nc_strerror(pioerr), PIO_MAX_NAME);
 #endif /* endif defined(_NETCDF) */
 #if defined(_PNETCDF)
     else if (pioerr > PIO_FIRST_ERROR_CODE)     /* pNetCDF error? */
-        strncpy(errmsg, ncmpi_strerror(pioerr), NC_MAX_NAME);
+        strncpy(errmsg, ncmpi_strerror(pioerr), PIO_MAX_NAME);
 #endif /* defined( _PNETCDF) */
     else
         /* Handle PIO errors. */
@@ -76,6 +115,9 @@ int PIOc_strerror(int pioerr, char *errmsg)
         case PIO_EBADIOTYPE:
             strcpy(errmsg, "Bad IO type");
             break;
+        case PIO_EVARDIMMISMATCH:
+            strcpy(errmsg, "Variable dim mismatch in multivar call");
+            break;
         default:
             strcpy(errmsg, "Unknown Error: Unrecognized error code");
         }
@@ -98,8 +140,10 @@ int PIOc_strerror(int pioerr, char *errmsg)
  * @param level the logging level, 0 for errors only, 5 for max
  * verbosity.
  * @returns 0 on success, error code otherwise.
+ * @author Ed Hartnett
  */
-int PIOc_set_log_level(int level)
+int
+PIOc_set_log_level(int level)
 {
 
 #if PIO_ENABLE_LOGGING
@@ -122,20 +166,28 @@ int PIOc_set_log_level(int level)
 /**
  * Initialize logging.  Open log file, if not opened yet, or increment
  * ref count if already open.
+ *
+ * @author Jayesh Krishna, Ed Hartnett
  */
-void pio_init_logging(void)
+int
+pio_init_logging(void)
 {
+    int mpierr;
+    int ret = PIO_NOERR;
+
 #if PIO_ENABLE_LOGGING
-    char log_filename[NC_MAX_NAME];
+    char log_filename[PIO_MAX_NAME];
 
     if (!LOG_FILE)
     {
         /* Create a filename with the rank in it. */
-        MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+        if ((mpierr = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         sprintf(log_filename, "pio_log_%d.log", my_rank);
 
         /* Open a file for this rank to log messages. */
-        LOG_FILE = fopen(log_filename, "w");
+        if (!(LOG_FILE = fopen(log_filename, "w")))
+            return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__);
 
         pio_log_ref_cnt = 1;
     }
@@ -144,12 +196,15 @@ void pio_init_logging(void)
         pio_log_ref_cnt++;
     }
 #endif /* PIO_ENABLE_LOGGING */
+
+    return ret;
 }
 
 /**
  * Finalize logging - close log files, if open.
  */
-void pio_finalize_logging(void)
+void
+pio_finalize_logging(void)
 {
 #if PIO_ENABLE_LOGGING
     pio_log_ref_cnt -= 1;
@@ -186,8 +241,10 @@ void pio_finalize_logging(void)
  * then increasing levels of verbosity.
  * @param fmt the format string.
  * @param ... the arguments used in format string.
+ * @author Ed Hartnett
  */
-void pio_log(int severity, const char *fmt, ...)
+void
+pio_log(int severity, const char *fmt, ...)
 {
     va_list argp;
     int t;
@@ -213,6 +270,7 @@ void pio_log(int severity, const char *fmt, ...)
         ptr += strlen(ERROR_PREFIX);
         rem_len -= strlen(ERROR_PREFIX);
     }
+
     for (t = 0; t < severity; t++)
     {
         strncpy(ptr++, "\t", (rem_len > 0) ? rem_len : 0);
@@ -225,6 +283,12 @@ void pio_log(int severity, const char *fmt, ...)
     ptr += strlen(rank_str);
     rem_len -= strlen(rank_str);
 
+    /* /\* Show the severity. *\/ */
+    /* snprintf(rank_str, MAX_RANK_STR, ":%d ", severity); */
+    /* strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0); */
+    /* ptr += strlen(rank_str); */
+    /* rem_len -= strlen(rank_str); */
+
     /* Print out the variable list of args with vprintf. */
     va_start(argp, fmt);
     vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp);
@@ -266,8 +330,10 @@ void pio_log(int severity, const char *fmt, ...)
  * automated process or tools.
  *
  * @param fp file pointer to send output to
+ * @author Jim Edwards
  */
-void print_trace(FILE *fp)
+void
+print_trace(FILE *fp)
 {
     void *array[10];
     size_t size;
@@ -295,8 +361,10 @@ void print_trace(FILE *fp)
  * @param msg an error message
  * @param fname name of code file where error occured
  * @param line the line of code where the error occurred.
+ * @author Jim Edwards
  */
-void piodie(const char *msg, const char *fname, int line)
+void
+piodie(const char *msg, const char *fname, int line)
 {
     fprintf(stderr,"Abort with message %s in file %s at line %d\n",
             msg ? msg : "_", fname ? fname : "_", line);
@@ -317,8 +385,10 @@ void piodie(const char *msg, const char *fname, int line)
  * @param msg an error message
  * @param fname name of code file where error occured
  * @param line the line of code where the error occurred.
+ * @author Jim Edwards
  */
-void pioassert(_Bool expression, const char *msg, const char *fname, int line)
+void
+pioassert(_Bool expression, const char *msg, const char *fname, int line)
 {
 #ifndef NDEBUG
     if (!expression)
@@ -330,33 +400,17 @@ void pioassert(_Bool expression, const char *msg, const char *fname, int line)
  * Handle MPI errors. An error message is sent to stderr, then the
  * check_netcdf() function is called with PIO_EIO.
  *
- * @param file pointer to the file_desc_t info. Ignored if NULL.
- * @param mpierr the MPI return code to handle
- * @param filename the name of the code file where error occured.
- * @param line the line of code where error occured.
- * @return PIO_NOERR for no error, otherwise PIO_EIO.
- */
-int check_mpi(file_desc_t *file, int mpierr, const char *filename,
-              int line)
-{
-    return check_mpi2(NULL, file, mpierr, filename, line);
-}
-
-/**
- * Handle MPI errors. An error message is sent to stderr, then the
- * check_netcdf() function is called with PIO_EIO. This version of the
- * function accepts an ios parameter, for the (rare) occasions where
- * we have an ios but not a file.
- *
  * @param ios pointer to the iosystem_info_t. May be NULL.
- * @param file pointer to the file_desc_t info. May be NULL.
+ * @param file pointer to the file_desc_t info. Ignored if NULL.
  * @param mpierr the MPI return code to handle
  * @param filename the name of the code file where error occured.
  * @param line the line of code where error occured.
  * @return PIO_NOERR for no error, otherwise PIO_EIO.
+ * @author Ed Hartnett
  */
-int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr,
-               const char *filename, int line)
+int
+check_mpi(iosystem_desc_t *ios, file_desc_t *file, int mpierr,
+          const char *filename, int line)
 {
     if (mpierr)
     {
@@ -383,8 +437,10 @@ int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr,
  * @param fname the name of the code file.
  * @param line the line number of the netCDF call in the code.
  * @return the error code
+ * @author Ed Hartnett
  */
-int check_netcdf(file_desc_t *file, int status, const char *fname, int line)
+int
+check_netcdf(file_desc_t *file, int status, const char *fname, int line)
 {
     return check_netcdf2(NULL, file, status, fname, line);
 }
@@ -401,18 +457,25 @@ int check_netcdf(file_desc_t *file, int status, const char *fname, int line)
  * @param fname the name of the code file.
  * @param line the line number of the netCDF call in the code.
  * @return the error code
+ * @author Ed Hartnett
  */
-int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
-                  const char *fname, int line)
+int
+check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
+              const char *fname, int line)
 {
     int eh = default_error_handler; /* Error handler that will be used. */
-
+    int rbuf;
     /* User must provide this. */
     pioassert(fname, "code file name must be provided", __FILE__, __LINE__);
 
-    /* No harm, no foul. */
-    if (status == PIO_NOERR)
-        return PIO_NOERR;
+    if (file && file->iosystem->ioproc &&
+        (file->iotype == PIO_IOTYPE_PNETCDF || file->iotype == PIO_IOTYPE_NETCDF4P))
+    {
+        if (file->iosystem->io_rank == 0)
+            MPI_Reduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
+        else
+            MPI_Reduce(&status, &rbuf, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
+    }
 
     LOG((1, "check_netcdf2 status = %d fname = %s line = %d", status, fname, line));
 
@@ -426,7 +489,7 @@ int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
     LOG((2, "check_netcdf2 chose error handler = %d", eh));
 
     /* Decide what to do based on the error handler. */
-    if (eh == PIO_INTERNAL_ERROR)
+    if (eh == PIO_INTERNAL_ERROR && status != PIO_NOERR)
     {
         char errmsg[PIO_MAX_NAME + 1];  /* Error message. */
         PIOc_strerror(status, errmsg);
@@ -434,10 +497,10 @@ int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
     }
     else if (eh == PIO_BCAST_ERROR)
     {
-	if (ios)
-	    MPI_Bcast(&status, 1, MPI_INT, ios->ioroot, ios->my_comm);
-	else if (file)
-	    MPI_Bcast(&status, 1, MPI_INT, file->iosystem->ioroot, file->iosystem->my_comm);
+        if (ios)
+            MPI_Bcast(&status, 1, MPI_INT, ios->ioroot, ios->my_comm);
+        else if (file)
+            MPI_Bcast(&status, 1, MPI_INT, file->iosystem->ioroot, file->iosystem->my_comm);
     }
 
     /* For PIO_RETURN_ERROR, just return the error. */
@@ -464,9 +527,11 @@ int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
  * @param fname name of code file where error occured.
  * @param line the line of code where the error occurred.
  * @returns err_num if abort is not called.
+ * @author Jim Edwards
  */
-int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
-            int line)
+int
+pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
+        int line)
 {
     char err_msg[PIO_MAX_NAME + 1];
     int err_handler = default_error_handler; /* Default error handler. */
@@ -518,11 +583,13 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fn
  * @param ios pointer to the IO system info, used for error
  * handling. Ignored if NULL.
  * @param ndims the number of dimensions for the data in this region.
- * @param a pointer that gets a pointer to the newly allocated
+ * @param regionp a pointer that gets a pointer to the newly allocated
  * io_region struct.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp)
+int
+alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp)
 {
     io_region *region;
 
@@ -558,8 +625,10 @@ int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp)
  * @param type_size a pointer to int that will get the size of the
  * type, in bytes. (For example, 4 for PIO_INT). Ignored if NULL.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size)
+int
+find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size)
 {
     MPI_Datatype my_mpi_type;
     int my_type_size;
@@ -641,9 +710,11 @@ int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size)
  * @param ndims the number of dimensions.
  * @param iodesc pointer that gets the newly allocated io_desc_t.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
-                  io_desc_t **iodesc)
+int
+malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
+              io_desc_t **iodesc)
 {
     MPI_Datatype mpi_type;
     PIO_Offset type_size;
@@ -676,8 +747,11 @@ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
     (*iodesc)->mpitype = mpi_type;
 
     /* Get the size of the type. */
-    if ((mpierr = MPI_Type_size((*iodesc)->mpitype, &(*iodesc)->mpitype_size)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (mpi_type == MPI_DATATYPE_NULL)
+        (*iodesc)->mpitype_size = 0;
+    else
+        if ((mpierr = MPI_Type_size((*iodesc)->mpitype, &(*iodesc)->mpitype_size)))
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Initialize some values in the struct. */
     (*iodesc)->maxregions = 1;
@@ -698,8 +772,10 @@ int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
  * Free a region list.
  *
  * top a pointer to the start of the list to free.
+ * @author Jim Edwards
  */
-void free_region_list(io_region *top)
+void
+free_region_list(io_region *top)
 {
     io_region *ptr, *tptr;
 
@@ -722,9 +798,11 @@ void free_region_list(io_region *top)
  * @param iosysid the IO system ID.
  * @param ioid the ID of the decomposition map to free.
  * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_freedecomp_c
  * @author Jim Edwards
  */
-int PIOc_freedecomp(int iosysid, int ioid)
+int
+PIOc_freedecomp(int iosysid, int ioid)
 {
     iosystem_desc_t *ios;
     io_desc_t *iodesc;
@@ -758,10 +836,10 @@ int PIOc_freedecomp(int iosysid, int ioid)
         /* Handle MPI errors. */
         LOG((3, "handline error mpierr %d ios->comproot %d", mpierr, ios->comproot));
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(NULL, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__);
         LOG((3, "handline error mpierr2 %d", mpierr2));
         if (mpierr)
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
     LOG((3, "freeing map, dimlen"));
@@ -771,6 +849,9 @@ int PIOc_freedecomp(int iosysid, int ioid)
     /* Free the dimlens. */
     free(iodesc->dimlen);
 
+    if (iodesc->remap)
+        free(iodesc->remap);
+
     LOG((3, "freeing rfrom, rtype"));
     if (iodesc->rfrom)
         free(iodesc->rfrom);
@@ -780,7 +861,7 @@ int PIOc_freedecomp(int iosysid, int ioid)
         for (int i = 0; i < iodesc->nrecvs; i++)
             if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
                 if ((mpierr = MPI_Type_free(&iodesc->rtype[i])))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
         free(iodesc->rtype);
     }
@@ -791,7 +872,7 @@ int PIOc_freedecomp(int iosysid, int ioid)
         for (int i = 0; i < iodesc->num_stypes; i++)
             if (iodesc->stype[i] != PIO_DATATYPE_NULL)
                 if ((mpierr = MPI_Type_free(iodesc->stype + i)))
-                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
         iodesc->num_stypes = 0;
         free(iodesc->stype);
@@ -818,7 +899,7 @@ int PIOc_freedecomp(int iosysid, int ioid)
 
     if (iodesc->rearranger == PIO_REARR_SUBSET)
         if ((mpierr = MPI_Comm_free(&iodesc->subset_comm)))
-            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     return pio_delete_iodesc_from_list(ioid);
 }
@@ -834,9 +915,11 @@ int PIOc_freedecomp(int iosysid, int ioid)
  * @param map
  * @param comm
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
-                 PIO_Offset **map, MPI_Comm comm)
+int
+PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
+             PIO_Offset **map, MPI_Comm comm)
 {
     int npes, myrank;
     int rnpes, rversno;
@@ -852,9 +935,9 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
         return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Comm_size(comm, &npes)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_rank(comm, &myrank)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     if (myrank == 0)
     {
@@ -862,8 +945,8 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
         if (!fp)
             pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
-        fscanf(fp,"version %d npes %d ndims %d\n",&rversno, &rnpes, ndims);
-
+        if (fscanf(fp,"version %d npes %d ndims %d\n", &rversno, &rnpes, ndims) != 3)
+            pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
         if (rversno != VERSNO)
             return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
@@ -871,33 +954,36 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
             return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
         if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if (!(tdims = calloc(*ndims, sizeof(int))))
             return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         for (int i = 0; i < *ndims; i++)
-            fscanf(fp,"%d ", tdims + i);
+            if (fscanf(fp,"%d ", tdims + i) != 1)
+                pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
         if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
         for (int i = 0; i < rnpes; i++)
         {
-            fscanf(fp, "%d %lld", &j, &maplen);
+            if (fscanf(fp, "%d %lld", &j, &maplen) != 2)
+                pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
             if (j != i)  // Not sure how this could be possible
                 return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
             if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
                 return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
             for (j = 0; j < maplen; j++)
-                fscanf(fp, "%lld ", tmap+j);
+                if (fscanf(fp, "%lld ", tmap + j) != 1)
+                    pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
             if (i > 0)
             {
                 if ((mpierr = MPI_Send(&maplen, 1, PIO_OFFSET, i, i + npes, comm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 if ((mpierr = MPI_Send(tmap, maplen, PIO_OFFSET, i, i, comm)))
-                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
                 free(tmap);
             }
             else
@@ -911,22 +997,22 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
     else
     {
         if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         if (!(tdims = calloc(*ndims, sizeof(int))))
             return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
         if (myrank < rnpes)
         {
             if ((mpierr = MPI_Recv(&maplen, 1, PIO_OFFSET, 0, myrank + npes, comm, &status)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
                 return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
             if ((mpierr = MPI_Recv(tmap, maplen, PIO_OFFSET, 0, myrank, comm, &status)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             *map = tmap;
         }
         else
@@ -950,9 +1036,11 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
  * @param map pointer to the map array
  * @param f90_comm
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset *maplen,
-                          PIO_Offset **map, int f90_comm)
+int
+PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset *maplen,
+                      PIO_Offset **map, int f90_comm)
 {
     return PIOc_readmap(file, ndims, gdims, maplen, map, MPI_Comm_f2c(f90_comm));
 }
@@ -966,19 +1054,23 @@ int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset
  * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE.
  * @param ioid the ID of the IO description.
  * @param title optial title attribute for the file. Must be less than
- * NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * PIO_MAX_NAME + 1 if provided. Ignored if NULL.
  * @param history optial history attribute for the file. Must be less
- * than NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * than PIO_MAX_NAME + 1 if provided. Ignored if NULL.
  * @param fortran_order set to non-zero if fortran array ordering is
  * used, or to zero if C array ordering is used.
  * @returns 0 for success, error code otherwise.
+ * @author Ed Hartnett
  */
-int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
-                         char *title, char *history, int fortran_order)
+int
+PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
+                     char *title, char *history, int fortran_order)
 {
     iosystem_desc_t *ios; /* IO system info. */
     io_desc_t *iodesc;    /* Decomposition info. */
     int max_maplen;       /* The maximum maplen used for any task. */
+    int *full_map;        /* 2D array holds all map info for all tasks. */
+    int *my_map;          /* 1D array holds all map info for this task. */
     int mpierr;
     int ret;
 
@@ -1012,22 +1104,23 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
      * task_maplen array on all tasks. */
     if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT,
                                 ios->comp_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
     /* Find the max maplen. */
     if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX,
                                 ios->comp_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((3, "max_maplen = %d", max_maplen));
 
-    /* 2D array that will hold all the map information for all
-     * tasks. */
-    int full_map[ios->num_comptasks][max_maplen];
+    if (!(full_map = malloc(sizeof(int) * ios->num_comptasks * max_maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(my_map = malloc(sizeof(int) * max_maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
     /* Fill local array with my map. Use the fill value for unused */
     /* elements at the end if max_maplen is longer than maplen. Also
      * subtract 1 because the iodesc->map is 1-based. */
-    int my_map[max_maplen];
     for (int e = 0; e < max_maplen; e++)
     {
         my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT;
@@ -1035,20 +1128,23 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
     }
 
     /* Gather my_map from all computation tasks and fill the full_map array. */
-    if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen,
+    if ((mpierr = MPI_Allgather(my_map, max_maplen, MPI_INT, full_map, max_maplen,
                                 MPI_INT, ios->comp_comm)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    free(my_map);
 
     for (int p = 0; p < ios->num_comptasks; p++)
         for (int e = 0; e < max_maplen; e++)
-            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e]));
+            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p * max_maplen + e]));
 
     /* Write the netCDF decomp file. */
     if ((ret = pioc_write_nc_decomp_int(ios, filename, cmode, iodesc->ndims, iodesc->dimlen,
-                                        ios->num_comptasks, task_maplen, (int *)full_map, title,
+                                        ios->num_comptasks, task_maplen, full_map, title,
                                         history, fortran_order)))
         return ret;
 
+    free(full_map);
     return PIO_NOERR;
 }
 
@@ -1058,22 +1154,24 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
  *
  * @param iosysid the IO system ID.
  * @param filename the name of the decomp file.
- * @param ioid pointer that will get the newly-assigned ID of the IO
+ * @param ioidp pointer that will get the newly-assigned ID of the IO
  * description. The ioid is needed to later free the decomposition.
  * @param comm an MPI communicator.
  * @param pio_type the PIO type to be used as the type for the data.
  * @param title pointer that will get optial title attribute for the
- * file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * file. Will be less than PIO_MAX_NAME + 1 if provided. Ignored if
  * NULL.
  * @param history pointer that will get optial history attribute for
- * the file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * the file. Will be less than PIO_MAX_NAME + 1 if provided. Ignored if
  * NULL.
  * @param fortran_order pointer that gets set to 1 if fortran array
  * ordering is used, or to zero if C array ordering is used.
  * @returns 0 for success, error code otherwise.
+ * @author Ed Hartnett
  */
-int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm comm,
-                        int pio_type, char *title, char *history, int *fortran_order)
+int
+PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm comm,
+                    int pio_type, char *title, char *history, int *fortran_order)
 {
     iosystem_desc_t *ios; /* Pointer to the IO system info. */
     int ndims;            /* The number of data dims (except unlim). */
@@ -1102,9 +1200,9 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
 
     /* Get the communicator size and task rank. */
     if ((mpierr = MPI_Comm_size(comm, &size)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
-        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "size = %d my_rank = %d", size, my_rank));
 
     /* Read the file. This allocates three arrays that we have to
@@ -1124,7 +1222,10 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
     /* Now initialize the iodesc on each task for this decomposition. */
     if (!ret)
     {
-        PIO_Offset compmap[task_maplen[my_rank]];
+        PIO_Offset *compmap;
+
+        if (!(compmap = malloc(sizeof(PIO_Offset) * task_maplen[my_rank])))
+            return PIO_ENOMEM;
 
         /* Copy array into PIO_Offset array. Make it 1 based. */
         for (int e = 0; e < task_maplen[my_rank]; e++)
@@ -1133,6 +1234,8 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
         /* Initialize the decomposition. */
         ret = PIOc_InitDecomp(iosysid, pio_type, ndims, global_dimlen, task_maplen[my_rank],
                               compmap, ioidp, NULL, NULL, NULL);
+
+        free(compmap);
     }
 
     /* Free resources. */
@@ -1143,7 +1246,8 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
     return ret;
 }
 
-/* Write the decomp information in netCDF. This is an internal
+/**
+ * Write the decomp information in netCDF. This is an internal
  * function.
  *
  * @param ios pointer to io system info.
@@ -1166,10 +1270,12 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
  * @param fortran_order set to non-zero if using fortran array
  * ordering, 0 for C array ordering.
  * @returns 0 for success, error code otherwise.
+ * @author Ed Hartnett
  */
-int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims,
-                             int *global_dimlen, int num_tasks, int *task_maplen, int *map,
-                             const char *title, const char *history, int fortran_order)
+int
+pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims,
+                         int *global_dimlen, int num_tasks, int *task_maplen, int *map,
+                         const char *title, const char *history, int fortran_order)
 {
     int max_maplen = 0;
     int ncid;
@@ -1234,7 +1340,6 @@ int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmo
 
     /* Write an attribute with the stack trace. This can be helpful
      * for debugging. */
-#define MAX_BACKTRACE 10
     void *bt[MAX_BACKTRACE];
     size_t bt_size;
     char **bt_strings;
@@ -1246,8 +1351,8 @@ int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmo
     for (int b = 0; b < bt_size; b++)
         if (strlen(bt_strings[b]) > max_bt_size)
             max_bt_size = strlen(bt_strings[b]);
-    if (max_bt_size > NC_MAX_NAME)
-        max_bt_size = NC_MAX_NAME;
+    if (max_bt_size > PIO_MAX_NAME)
+        max_bt_size = PIO_MAX_NAME;
 
     /* Copy the backtrace into one long string. */
     char full_bt[max_bt_size * bt_size + bt_size + 1];
@@ -1327,7 +1432,8 @@ int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmo
     return PIO_NOERR;
 }
 
-/* Read the decomp information from a netCDF decomp file. This is an
+/**
+ * Read the decomp information from a netCDF decomp file. This is an
  * internal function.
  *
  * @param iosysid the IO system ID.
@@ -1363,10 +1469,12 @@ int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmo
  * decomposition file uses C array ordering, 1 if it uses Fortran
  * array ordering.
  * @returns 0 for success, error code otherwise.
+ * @author Ed Hartnett
  */
-int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
-                            int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
-                            char *history, char *source, char *version, int *fortran_order)
+int
+pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
+                        int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
+                        char *history, char *source, char *version, int *fortran_order)
 {
     iosystem_desc_t *ios;
     int ncid;
@@ -1418,7 +1526,7 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
         *max_maplen = max_maplen_in;
 
     /* Read title attribute, if it is in the file. */
-    char title_in[NC_MAX_NAME + 1];
+    char title_in[PIO_MAX_NAME + 1];
     ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_TITLE_ATT_NAME, title_in);
     if (ret == PIO_NOERR)
     {
@@ -1436,7 +1544,7 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
     /* Read history attribute, if it is in the file. */
-    char history_in[NC_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
     ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_HISTORY_ATT_NAME, history_in);
     if (ret == PIO_NOERR)
     {
@@ -1454,7 +1562,7 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
     /* Read source attribute. */
-    char source_in[NC_MAX_NAME + 1];
+    char source_in[PIO_MAX_NAME + 1];
     if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_SOURCE_ATT_NAME, source_in)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
     if (source)
@@ -1515,10 +1623,14 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
 
     /* Read the map. */
     int map_varid;
-    int map_in[num_tasks_in][max_maplen_in];
+    int *map_in;
+
+    if (!(map_in = malloc(sizeof(int) * num_tasks_in * max_maplen_in)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
     if ((ret = PIOc_inq_varid(ncid, DECOMP_MAP_VAR_NAME, &map_varid)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
-    if ((ret = PIOc_get_var_int(ncid, map_varid, (int *)map_in)))
+    if ((ret = PIOc_get_var_int(ncid, map_varid, map_in)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
     if (map)
     {
@@ -1526,8 +1638,9 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         for (int t = 0; t < num_tasks_in; t++)
             for (int l = 0; l < max_maplen_in; l++)
-                (*map)[t * max_maplen_in + l] = map_in[t][l];
+                (*map)[t * max_maplen_in + l] = map_in[t * max_maplen_in + l];
     }
+    free(map_in);
 
     /* Close the netCDF decomp file. */
     LOG((2, "pioc_read_nc_decomp_int about to close file ncid = %d", ncid));
@@ -1546,8 +1659,10 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
  * @param ioid the ID of the IO description.
  * @param comm an MPI communicator.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm)
+int
+PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm)
 {
     iosystem_desc_t *ios;
     io_desc_t *iodesc;
@@ -1574,9 +1689,11 @@ int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm)
  * @param map the map array
  * @param comm an MPI communicator.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
-                  PIO_Offset *map, MPI_Comm comm)
+int
+PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
+              PIO_Offset *map, MPI_Comm comm)
 {
     int npes, myrank;
     PIO_Offset *nmaplen = NULL;
@@ -1588,9 +1705,9 @@ int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset mapl
     LOG((1, "PIOc_writemap file = %s ndims = %d maplen = %d", file, ndims, maplen));
 
     if ((mpierr = MPI_Comm_size(comm, &npes)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     if ((mpierr = MPI_Comm_rank(comm, &myrank)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     LOG((2, "npes = %d myrank = %d", npes, myrank));
 
     /* Allocate memory for the nmaplen. */
@@ -1599,7 +1716,7 @@ int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset mapl
             return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
     if ((mpierr = MPI_Gather(&maplen, 1, PIO_OFFSET, nmaplen, 1, PIO_OFFSET, 0, comm)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
     /* Only rank 0 writes the file. */
     if (myrank == 0)
@@ -1628,9 +1745,9 @@ int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset mapl
             nmap = (PIO_Offset *)malloc(nmaplen[i] * sizeof(PIO_Offset));
 
             if ((mpierr = MPI_Send(&i, 1, MPI_INT, i, npes + i, comm)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             if ((mpierr = MPI_Recv(nmap, nmaplen[i], PIO_OFFSET, i, i, comm, &status)))
-                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
             LOG((2,"MPI_Recv map complete"));
 
             fprintf(fp, "%d %lld\n", i, nmaplen[i]);
@@ -1653,10 +1770,10 @@ int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset mapl
     {
         LOG((2,"ready to MPI_Recv..."));
         if ((mpierr = MPI_Recv(&i, 1, MPI_INT, 0, npes+myrank, comm, &status)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         LOG((2,"MPI_Recv got %d", i));
         if ((mpierr = MPI_Send(map, maplen, PIO_OFFSET, 0, myrank, comm)))
-            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         LOG((2,"MPI_Send map complete"));
     }
 
@@ -1671,11 +1788,13 @@ int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset mapl
  * @param gdims an array of dimension ids
  * @param maplen the length of the map
  * @param map the map array
- * @param comm an MPI communicator.
+ * @param f90_comm an MPI communicator.
  * @returns 0 for success, error code otherwise.
+ * @author Jim Edwards
  */
-int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
-                           PIO_Offset maplen, const PIO_Offset *map, int f90_comm)
+int
+PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
+                       PIO_Offset maplen, const PIO_Offset *map, int f90_comm)
 {
     return PIOc_writemap(file, ndims, gdims, maplen, (PIO_Offset *)map,
                          MPI_Comm_f2c(f90_comm));
@@ -1687,7 +1806,7 @@ int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
  * parameters are read on comp task 0 and ignored elsewhere.
  *
  * @param iosysid A defined pio system ID, obtained from
- * PIOc_Init_Intercomm() or PIOc_InitAsync().
+ * PIOc_Init_Intracomm() or PIOc_InitAsync().
  * @param ncidp A pointer that gets the ncid of the newly created
  * file.
  * @param iotype A pointer to a pio output format. Must be one of
@@ -1696,10 +1815,12 @@ int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
  * @param filename The filename to create.
  * @param mode The netcdf mode for the create operation.
  * @returns 0 for success, error code otherwise.
- * @ingroup PIO_createfile
+ * @ingroup PIO_createfile_c
+ * @author Ed Hartnett
  */
-int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
-                        int mode)
+int
+PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
+                    int mode)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -1711,7 +1832,7 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
 
     /* User must provide valid input for these parameters. */
-    if (!ncidp || !iotype || !filename || strlen(filename) > NC_MAX_NAME)
+    if (!ncidp || !iotype || !filename || strlen(filename) > PIO_MAX_NAME)
         return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
     /* A valid iotype must be specified. */
@@ -1729,7 +1850,7 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
     file->fh = -1;
     file->iosystem = ios;
     file->iotype = *iotype;
-    file->buffer.ioid = -1;
+    file->buffer = NULL;
     file->writable = 1;
 
     /* Set to true if this task should participate in IO (only true for
@@ -1770,9 +1891,9 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
         /* Handle MPI errors. */
         LOG((2, "handling mpi errors mpierr = %d", mpierr));
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this task is in the IO component, do the IO. */
@@ -1811,7 +1932,7 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
 
     /* Broadcast and check the return code. */
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     /* If there was an error, free the memory we allocated and handle error. */
     if (ierr)
@@ -1822,7 +1943,7 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
 
     /* Broadcast writablility to all tasks. */
     if ((mpierr = MPI_Bcast(&file->writable, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     /* Broadcast next ncid to all tasks from io root, necessary
      * because files may be opened on mutilple iosystems, causing the
@@ -1832,7 +1953,7 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
     {
         LOG((3, "createfile bcasting pio_next_ncid %d", pio_next_ncid));
         if ((mpierr = MPI_Bcast(&pio_next_ncid, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         LOG((3, "createfile bcast pio_next_ncid %d", pio_next_ncid));
     }
 
@@ -1863,7 +1984,8 @@ int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filena
  * @returns 0 if file is OK, error code otherwise.
  * @author Ed Hartnett
  */
-int check_unlim_use(int ncid)
+int
+check_unlim_use(int ncid)
 {
 #ifdef _NETCDF4
     int nunlimdims; /* Number of unlimited dims in file. */
@@ -1933,13 +2055,14 @@ int check_unlim_use(int ncid)
  * caller.
  *
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_openfile
+ * @ingroup PIO_openfile_c
  * @author Ed Hartnett
  */
-int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec_var,
-                      int **pio_type, int **pio_type_size, MPI_Datatype **mpi_type, int **mpi_type_size)
+int
+inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec_var,
+                  int **pio_type, int **pio_type_size, MPI_Datatype **mpi_type, int **mpi_type_size)
 {
-    int nunlimdims;        /* The number of unlimited dimensions. */
+    int nunlimdims = 0;        /* The number of unlimited dimensions. */
     int unlimdimid;
     int *unlimdimids;
     int mpierr;
@@ -1999,7 +2122,7 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
     if (nunlimdims)
     {
         if (!(unlimdimids = malloc(nunlimdims * sizeof(int))))
-            return pio_err(NULL, file, PIO_ENOMEM, __FILE__, __LINE__);        
+            return pio_err(NULL, file, PIO_ENOMEM, __FILE__, __LINE__);
         if (iotype == PIO_IOTYPE_PNETCDF || iotype == PIO_IOTYPE_NETCDF)
         {
             unlimdimids[0] = unlimdimid;
@@ -2023,21 +2146,21 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
          * learn about type. */
         if (iotype == PIO_IOTYPE_PNETCDF)
         {
-            PIO_Offset type_size;
-            
 #ifdef _PNETCDF
+            PIO_Offset type_size;
+
             if ((ret = ncmpi_inq_var(ncid, v, NULL, &my_type, &var_ndims, NULL, NULL)))
                 return pio_err(NULL, file, ret, __FILE__, __LINE__);
             (*pio_type)[v] = (int)my_type;
             if ((ret = pioc_pnetcdf_inq_type(ncid, (*pio_type)[v], NULL, &type_size)))
                 return check_netcdf(file, ret, __FILE__, __LINE__);
             (*pio_type_size)[v] = type_size;
-#endif /* _PNETCDF */            
+#endif /* _PNETCDF */
         }
         else
         {
             size_t type_size;
-            
+
             if ((ret = nc_inq_var(ncid, v, NULL, &my_type, &var_ndims, NULL, NULL)))
                 return pio_err(NULL, file, ret, __FILE__, __LINE__);
             (*pio_type)[v] = (int)my_type;
@@ -2051,8 +2174,11 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
             return pio_err(NULL, file, ret, __FILE__, __LINE__);
 
         /* Get the size of the MPI type. */
-        if ((mpierr = MPI_Type_size((*mpi_type)[v], &(*mpi_type_size)[v])))
-            return check_mpi2(NULL, file, mpierr, __FILE__, __LINE__);
+        if ((*mpi_type)[v] == MPI_DATATYPE_NULL)
+            (*mpi_type_size)[v] = 0;
+        else
+            if ((mpierr = MPI_Type_size((*mpi_type)[v], &(*mpi_type_size)[v])))
+                return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
         /* What are the dimids associated with this var? */
         if (var_ndims)
@@ -2063,14 +2189,14 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
 #ifdef _PNETCDF
                 if ((ret = ncmpi_inq_vardimid(ncid, v, var_dimids)))
                     return pio_err(NULL, file, ret, __FILE__, __LINE__);
-#endif /* _PNETCDF */                
+#endif /* _PNETCDF */
             }
             else
             {
                 if ((ret = nc_inq_vardimid(ncid, v, var_dimids)))
                     return pio_err(NULL, file, ret, __FILE__, __LINE__);
             }
-            
+
             /* Check against each variable dimid agains each unlimited
              * dimid. */
             for (int d = 0; d < var_ndims; d++)
@@ -2094,19 +2220,19 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
                         (*rec_var)[v] = 1;
                     else
                         return pio_err(NULL, file, PIO_EINVAL, __FILE__, __LINE__);
-                    
+
                 }
                 else
                     (*rec_var)[v] = 0;
-                    
+
             }
         }
     } /* next var */
-    
+
     /* Free resources. */
     if (nunlimdims)
         free(unlimdimids);
-    
+
     return PIO_NOERR;
 }
 
@@ -2123,20 +2249,21 @@ int inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int *
  *
  * Input parameters are read on comp task 0 and ignored elsewhere.
  *
- * @param iosysid: A defined pio system descriptor (input)
- * @param ncidp: A pio file descriptor (output)
- * @param iotype: A pio output format (input)
- * @param filename: The filename to open
- * @param mode: The netcdf mode for the open operation
- * @param retry: non-zero to automatically retry with netCDF serial
+ * @param iosysid a defined pio system descriptor.
+ * @param ncidp a pio file descriptor.
+ * @param iotype a pio output format.
+ * @param filename the filename to open
+ * @param mode the netcdf mode for the open operation
+ * @param retry non-zero to automatically retry with netCDF serial
  * classic.
  *
  * @return 0 for success, error code otherwise.
- * @ingroup PIO_openfile
+ * @ingroup PIO_openfile_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
-                        int mode, int retry)
+int
+PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
+                    int mode, int retry)
 {
     iosystem_desc_t *ios;      /* Pointer to io system information. */
     file_desc_t *file;         /* Pointer to file information. */
@@ -2204,9 +2331,9 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
 
         /* Handle MPI errors. */
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -2303,11 +2430,10 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
                 /* open netcdf file serially on main task */
                 if (ios->io_rank == 0)
                 {
-                    if ((ierr = nc_open(filename, mode, &file->fh)))
-                        return pio_err(ios, file, ierr, __FILE__, __LINE__);
-                    if ((ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, &nvars, &rec_var, &pio_type,
-                                                  &pio_type_size, &mpi_type, &mpi_type_size)))
-                        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+                    ierr = nc_open(filename, mode, &file->fh);
+                    if (ierr == PIO_NOERR)
+                        ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, &nvars, &rec_var, &pio_type,
+                                                 &pio_type_size, &mpi_type, &mpi_type_size);
                 }
                 else
                     file->do_io = 0;
@@ -2318,10 +2444,11 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
     }
 
     /* Broadcast and check the return code. */
-    LOG((2, "Bcasting error code ierr %d ios->ioroot %d ios->my_comm %d",
-         ierr, ios->ioroot, ios->my_comm));
+    if (ios->ioroot == ios->union_rank)
+        LOG((2, "Bcasting error code ierr %d ios->ioroot %d ios->my_comm %d",
+             ierr, ios->ioroot, ios->my_comm));
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     LOG((2, "Bcast openfile_retry error code ierr = %d", ierr));
 
     /* If there was an error, free allocated memory and deal with the error. */
@@ -2333,45 +2460,45 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
 
     /* Broadcast writability to all tasks. */
     if ((mpierr = MPI_Bcast(&file->writable, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     /* Broadcast some values to all tasks from io root. */
     if (ios->async)
     {
         LOG((3, "open bcasting pio_next_ncid %d ios->ioroot %d", pio_next_ncid, ios->ioroot));
         if ((mpierr = MPI_Bcast(&pio_next_ncid, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
-    
+
     if ((mpierr = MPI_Bcast(&nvars, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
 
     /* Non io tasks need to allocate to store info about variables. */
     if (nvars && !rec_var)
     {
         if (!(rec_var = malloc(nvars * sizeof(int))))
-            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);                    
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         if (!(pio_type = malloc(nvars * sizeof(int))))
-            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);                    
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         if (!(pio_type_size = malloc(nvars * sizeof(int))))
-            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);                    
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         if (!(mpi_type = malloc(nvars * sizeof(MPI_Datatype))))
-            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);                    
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         if (!(mpi_type_size = malloc(nvars * sizeof(int))))
-            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);                    
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
     }
     if (nvars)
     {
         if ((mpierr = MPI_Bcast(rec_var, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(pio_type, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(pio_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(mpi_type, nvars*(int)(sizeof(MPI_Datatype)/sizeof(int)), MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
         if ((mpierr = MPI_Bcast(mpi_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm)))
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* Create the ncid that the user will see. This is necessary
@@ -2422,9 +2549,11 @@ int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filena
  * @param name pointer that gets name of type. Ignored if NULL.
  * @param sizep pointer that gets size of type. Ignored if NULL.
  * @returns 0 on success, error code otherwise.
+ * @author Ed Hartnett
  */
-int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
-                          PIO_Offset *sizep)
+int
+pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
+                      PIO_Offset *sizep)
 {
     int typelen;
 
@@ -2468,8 +2597,11 @@ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
  *
  * @param ncid the ncid of the file to enddef or redef
  * @param is_enddef set to non-zero for enddef, 0 for redef.
- * @returns PIO_NOERR on success, error code on failure. */
-int pioc_change_def(int ncid, int is_enddef)
+ * @returns PIO_NOERR on success, error code on failure.
+ * @author Ed Hartnett
+ */
+int
+pioc_change_def(int ncid, int is_enddef)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -2501,9 +2633,9 @@ int pioc_change_def(int ncid, int is_enddef)
         /* Handle MPI errors. */
         LOG((3, "pioc_change_def handling MPI errors"));
         if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
-            check_mpi(file, mpierr2, __FILE__, __LINE__);
+            check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
         if (mpierr)
-            return check_mpi(file, mpierr, __FILE__, __LINE__);
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
 
     /* If this is an IO task, then call the netCDF function. */
@@ -2536,7 +2668,7 @@ int pioc_change_def(int ncid, int is_enddef)
     /* Broadcast and check the return code. */
     LOG((3, "pioc_change_def bcasting return code ierr = %d", ierr));
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-        return check_mpi(file, mpierr, __FILE__, __LINE__);
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     if (ierr)
         return check_netcdf(file, ierr, __FILE__, __LINE__);
     LOG((3, "pioc_change_def succeeded"));
@@ -2549,8 +2681,10 @@ int pioc_change_def(int ncid, int is_enddef)
  *
  * @param iotype the IO type to check
  * @returns 0 if valid, non-zero otherwise.
+ * @author Jim Edwards
  */
-int iotype_is_valid(int iotype)
+int
+iotype_is_valid(int iotype)
 {
     /* Assume it's not valid. */
     int ret = 0;
@@ -2577,6 +2711,7 @@ int iotype_is_valid(int iotype)
 /**
  * Set the rearranger options associated with an iosystem
  *
+ * @param iosysid a defined pio system descriptor.
  * @param comm_type Type of communication (pt2pt/coll) used
  * by the rearranger. See PIO_REARR_COMM_TYPE for more detail.
  * Possible values are :
@@ -2605,13 +2740,14 @@ int iotype_is_valid(int iotype)
  * data, from io to compute processes
  * @param max_pend_req_i2c Maximum pending requests during
  * data rearragment from io processes to compute processes
- * @param iosysidp index of the defined system descriptor
  * @return 0 on success, otherwise a PIO error code.
+ * @author Jayesh Krishna
  */
-int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
-                        bool enable_isend_c2i, int max_pend_req_c2i,
-                        bool enable_hs_i2c, bool enable_isend_i2c,
-                        int max_pend_req_i2c)
+int
+PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
+                    bool enable_isend_c2i, int max_pend_req_c2i,
+                    bool enable_hs_i2c, bool enable_isend_i2c,
+                    int max_pend_req_i2c)
 {
     iosystem_desc_t *ios;
     rearr_opt_t user_rearr_opts = {
@@ -2650,7 +2786,7 @@ int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
  * Note that memory is allocated for my_proc_list. This must be freed
  * by the caller.
  *
- * @param num_io_proc the number of IO processes.
+ * @param num_io_procs the number of IO processes.
  * @param component_count the number of computational components.
  * @param num_procs_per_comp array (length component_count) which
  * contains the number of processes to assign to each computation
@@ -2658,14 +2794,15 @@ int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
  * @param proc_list array (length component count) of arrays (length
  * num_procs_per_comp_array[cmp]) which contain the list of processes
  * for each computation component. May be NULL.
- * @param array (length component count) of arrays (length
+ * @param my_proc_list array (length component count) of arrays (length
  * num_procs_per_comp_array[cmp]) which will get the list of processes
  * for each computation component.
  * @returns 0 for success, error code otherwise
  * @author Ed Hartnett
  */
-int determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp,
-                    int **proc_list, int **my_proc_list)
+int
+determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp,
+                int **proc_list, int **my_proc_list)
 {
     /* If the user did not provide a list of processes for each
      * component, create one. */
diff --git a/src/clib/topology.c b/src/clib/topology.c
index f8e1fc016be7..2fc886d1e73c 100644
--- a/src/clib/topology.c
+++ b/src/clib/topology.c
@@ -122,8 +122,8 @@ void determineiotasks(const MPI_Comm comm, int *numiotasks,int *base, int *strid
 
         Kernel_GetPersonality(&pers, sizeof(pers));
 
-        int numIONodes,numPsets,numNodesInPset,rankInPset;
-        int numiotasks_per_node,remainder,numIONodes_per_pset;
+        int numIONodes, numPsets, numNodesInPset, rankInPset;
+        int numiotasks_per_node, remainder = 0, numIONodes_per_pset;
         int lstride;
 
         /* Number of computational nodes in processor set */
diff --git a/src/clib/uthash.h b/src/clib/uthash.h
new file mode 100644
index 000000000000..59910166f598
--- /dev/null
+++ b/src/clib/uthash.h
@@ -0,0 +1,1230 @@
+/*
+Copyright (c) 2003-2018, Troy D. Hanson     http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTHASH_H
+#define UTHASH_H
+
+#define UTHASH_VERSION 2.1.0
+
+#include    /* memcmp, memset, strlen */
+#include    /* ptrdiff_t */
+#include    /* exit */
+
+/* These macros use decltype or the earlier __typeof GNU extension.
+   As decltype is only available in newer compilers (VS2010 or gcc 4.3+
+   when compiling c++ source) this code uses whatever method is needed
+   or, for VS2008 where neither is available, uses casting workarounds. */
+#if !defined(DECLTYPE) && !defined(NO_DECLTYPE)
+#if defined(_MSC_VER)   /* MS compiler */
+#if _MSC_VER >= 1600 && defined(__cplusplus)  /* VS2010 or newer in C++ mode */
+#define DECLTYPE(x) (decltype(x))
+#else                   /* VS2008 or older (or VS2010 in C mode) */
+#define NO_DECLTYPE
+#endif
+#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__)
+#define NO_DECLTYPE
+#else                   /* GNU, Sun and other compilers */
+#define DECLTYPE(x) (__typeof(x))
+#endif
+#endif
+
+#ifdef NO_DECLTYPE
+#define DECLTYPE(x)
+#define DECLTYPE_ASSIGN(dst,src)                                                 \
+do {                                                                             \
+  char **_da_dst = (char**)(&(dst));                                             \
+  *_da_dst = (char*)(src);                                                       \
+} while (0)
+#else
+#define DECLTYPE_ASSIGN(dst,src)                                                 \
+do {                                                                             \
+  (dst) = DECLTYPE(dst)(src);                                                    \
+} while (0)
+#endif
+
+/* a number of the hash function use uint32_t which isn't defined on Pre VS2010 */
+#if defined(_WIN32)
+#if defined(_MSC_VER) && _MSC_VER >= 1600
+#include 
+#elif defined(__WATCOMC__) || defined(__MINGW32__) || defined(__CYGWIN__)
+#include 
+#else
+typedef unsigned int uint32_t;
+typedef unsigned char uint8_t;
+#endif
+#elif defined(__GNUC__) && !defined(__VXWORKS__)
+#include 
+#else
+typedef unsigned int uint32_t;
+typedef unsigned char uint8_t;
+#endif
+
+#ifndef uthash_malloc
+#define uthash_malloc(sz) malloc(sz)      /* malloc fcn                      */
+#endif
+#ifndef uthash_free
+#define uthash_free(ptr,sz) free(ptr)     /* free fcn                        */
+#endif
+#ifndef uthash_bzero
+#define uthash_bzero(a,n) memset(a,'\0',n)
+#endif
+#ifndef uthash_strlen
+#define uthash_strlen(s) strlen(s)
+#endif
+
+#ifdef uthash_memcmp
+/* This warning will not catch programs that define uthash_memcmp AFTER including uthash.h. */
+#warning "uthash_memcmp is deprecated; please use HASH_KEYCMP instead"
+#else
+#define uthash_memcmp(a,b,n) memcmp(a,b,n)
+#endif
+
+#ifndef HASH_KEYCMP
+#define HASH_KEYCMP(a,b,n) uthash_memcmp(a,b,n)
+#endif
+
+#ifndef uthash_noexpand_fyi
+#define uthash_noexpand_fyi(tbl)          /* can be defined to log noexpand  */
+#endif
+#ifndef uthash_expand_fyi
+#define uthash_expand_fyi(tbl)            /* can be defined to log expands   */
+#endif
+
+#ifndef HASH_NONFATAL_OOM
+#define HASH_NONFATAL_OOM 0
+#endif
+
+#if HASH_NONFATAL_OOM
+/* malloc failures can be recovered from */
+
+#ifndef uthash_nonfatal_oom
+#define uthash_nonfatal_oom(obj) do {} while (0)    /* non-fatal OOM error */
+#endif
+
+#define HASH_RECORD_OOM(oomed) do { (oomed) = 1; } while (0)
+#define IF_HASH_NONFATAL_OOM(x) x
+
+#else
+/* malloc failures result in lost memory, hash tables are unusable */
+
+#ifndef uthash_fatal
+#define uthash_fatal(msg) exit(-1)        /* fatal OOM error */
+#endif
+
+#define HASH_RECORD_OOM(oomed) uthash_fatal("out of memory")
+#define IF_HASH_NONFATAL_OOM(x)
+
+#endif
+
+/* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS 32U     /* initial number of buckets        */
+#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */
+#define HASH_BKT_CAPACITY_THRESH 10U     /* expand when bucket count reaches */
+
+/* calculate the element whose hash handle address is hhp */
+#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho)))
+/* calculate the hash handle from element address elp */
+#define HH_FROM_ELMT(tbl,elp) ((UT_hash_handle *)(((char*)(elp)) + ((tbl)->hho)))
+
+#define HASH_ROLLBACK_BKT(hh, head, itemptrhh)                                   \
+do {                                                                             \
+  struct UT_hash_handle *_hd_hh_item = (itemptrhh);                              \
+  unsigned _hd_bkt;                                                              \
+  HASH_TO_BKT(_hd_hh_item->hashv, (head)->hh.tbl->num_buckets, _hd_bkt);         \
+  (head)->hh.tbl->buckets[_hd_bkt].count++;                                      \
+  _hd_hh_item->hh_next = NULL;                                                   \
+  _hd_hh_item->hh_prev = NULL;                                                   \
+} while (0)
+
+#define HASH_VALUE(keyptr,keylen,hashv)                                          \
+do {                                                                             \
+  HASH_FCN(keyptr, keylen, hashv);                                               \
+} while (0)
+
+#define HASH_FIND_BYHASHVALUE(hh,head,keyptr,keylen,hashval,out)                 \
+do {                                                                             \
+  (out) = NULL;                                                                  \
+  if (head) {                                                                    \
+    unsigned _hf_bkt;                                                            \
+    HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt);                  \
+    if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) {                         \
+      HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+
+#define HASH_FIND(hh,head,keyptr,keylen,out)                                     \
+do {                                                                             \
+  unsigned _hf_hashv;                                                            \
+  HASH_VALUE(keyptr, keylen, _hf_hashv);                                         \
+  HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out);               \
+} while (0)
+
+#ifdef HASH_BLOOM
+#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM)
+#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8UL) + (((HASH_BLOOM_BITLEN%8UL)!=0UL) ? 1UL : 0UL)
+#define HASH_BLOOM_MAKE(tbl,oomed)                                               \
+do {                                                                             \
+  (tbl)->bloom_nbits = HASH_BLOOM;                                               \
+  (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN);                 \
+  if (!(tbl)->bloom_bv) {                                                        \
+    HASH_RECORD_OOM(oomed);                                                      \
+  } else {                                                                       \
+    uthash_bzero((tbl)->bloom_bv, HASH_BLOOM_BYTELEN);                           \
+    (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE;                                     \
+  }                                                                              \
+} while (0)
+
+#define HASH_BLOOM_FREE(tbl)                                                     \
+do {                                                                             \
+  uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN);                              \
+} while (0)
+
+#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U)))
+#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U)))
+
+#define HASH_BLOOM_ADD(tbl,hashv)                                                \
+  HASH_BLOOM_BITSET((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U)))
+
+#define HASH_BLOOM_TEST(tbl,hashv)                                               \
+  HASH_BLOOM_BITTEST((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U)))
+
+#else
+#define HASH_BLOOM_MAKE(tbl,oomed)
+#define HASH_BLOOM_FREE(tbl)
+#define HASH_BLOOM_ADD(tbl,hashv)
+#define HASH_BLOOM_TEST(tbl,hashv) (1)
+#define HASH_BLOOM_BYTELEN 0U
+#endif
+
+#define HASH_MAKE_TABLE(hh,head,oomed)                                           \
+do {                                                                             \
+  (head)->hh.tbl = (UT_hash_table*)uthash_malloc(sizeof(UT_hash_table));         \
+  if (!(head)->hh.tbl) {                                                         \
+    HASH_RECORD_OOM(oomed);                                                      \
+  } else {                                                                       \
+    uthash_bzero((head)->hh.tbl, sizeof(UT_hash_table));                         \
+    (head)->hh.tbl->tail = &((head)->hh);                                        \
+    (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS;                      \
+    (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2;            \
+    (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head);                  \
+    (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc(                    \
+        HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket));               \
+    (head)->hh.tbl->signature = HASH_SIGNATURE;                                  \
+    if (!(head)->hh.tbl->buckets) {                                              \
+      HASH_RECORD_OOM(oomed);                                                    \
+      uthash_free((head)->hh.tbl, sizeof(UT_hash_table));                        \
+    } else {                                                                     \
+      uthash_bzero((head)->hh.tbl->buckets,                                      \
+          HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket));             \
+      HASH_BLOOM_MAKE((head)->hh.tbl, oomed);                                    \
+      IF_HASH_NONFATAL_OOM(                                                      \
+        if (oomed) {                                                             \
+          uthash_free((head)->hh.tbl->buckets,                                   \
+              HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket));           \
+          uthash_free((head)->hh.tbl, sizeof(UT_hash_table));                    \
+        }                                                                        \
+      )                                                                          \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+
+#define HASH_REPLACE_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,replaced,cmpfcn) \
+do {                                                                             \
+  (replaced) = NULL;                                                             \
+  HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \
+  if (replaced) {                                                                \
+    HASH_DELETE(hh, head, replaced);                                             \
+  }                                                                              \
+  HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn); \
+} while (0)
+
+#define HASH_REPLACE_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add,replaced) \
+do {                                                                             \
+  (replaced) = NULL;                                                             \
+  HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \
+  if (replaced) {                                                                \
+    HASH_DELETE(hh, head, replaced);                                             \
+  }                                                                              \
+  HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add); \
+} while (0)
+
+#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced)                   \
+do {                                                                             \
+  unsigned _hr_hashv;                                                            \
+  HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv);                         \
+  HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced); \
+} while (0)
+
+#define HASH_REPLACE_INORDER(hh,head,fieldname,keylen_in,add,replaced,cmpfcn)    \
+do {                                                                             \
+  unsigned _hr_hashv;                                                            \
+  HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv);                         \
+  HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced, cmpfcn); \
+} while (0)
+
+#define HASH_APPEND_LIST(hh, head, add)                                          \
+do {                                                                             \
+  (add)->hh.next = NULL;                                                         \
+  (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail);           \
+  (head)->hh.tbl->tail->next = (add);                                            \
+  (head)->hh.tbl->tail = &((add)->hh);                                           \
+} while (0)
+
+#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn)                                 \
+do {                                                                             \
+  do {                                                                           \
+    if (cmpfcn(DECLTYPE(head)(_hs_iter), add) > 0) {                             \
+      break;                                                                     \
+    }                                                                            \
+  } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next));           \
+} while (0)
+
+#ifdef NO_DECLTYPE
+#undef HASH_AKBI_INNER_LOOP
+#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn)                                 \
+do {                                                                             \
+  char *_hs_saved_head = (char*)(head);                                          \
+  do {                                                                           \
+    DECLTYPE_ASSIGN(head, _hs_iter);                                             \
+    if (cmpfcn(head, add) > 0) {                                                 \
+      DECLTYPE_ASSIGN(head, _hs_saved_head);                                     \
+      break;                                                                     \
+    }                                                                            \
+    DECLTYPE_ASSIGN(head, _hs_saved_head);                                       \
+  } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next));           \
+} while (0)
+#endif
+
+#if HASH_NONFATAL_OOM
+
+#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed)            \
+do {                                                                             \
+  if (!(oomed)) {                                                                \
+    unsigned _ha_bkt;                                                            \
+    (head)->hh.tbl->num_items++;                                                 \
+    HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt);                  \
+    HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed);    \
+    if (oomed) {                                                                 \
+      HASH_ROLLBACK_BKT(hh, head, &(add)->hh);                                   \
+      HASH_DELETE_HH(hh, head, &(add)->hh);                                      \
+      (add)->hh.tbl = NULL;                                                      \
+      uthash_nonfatal_oom(add);                                                  \
+    } else {                                                                     \
+      HASH_BLOOM_ADD((head)->hh.tbl, hashval);                                   \
+      HASH_EMIT_KEY(hh, head, keyptr, keylen_in);                                \
+    }                                                                            \
+  } else {                                                                       \
+    (add)->hh.tbl = NULL;                                                        \
+    uthash_nonfatal_oom(add);                                                    \
+  }                                                                              \
+} while (0)
+
+#else
+
+#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed)            \
+do {                                                                             \
+  unsigned _ha_bkt;                                                              \
+  (head)->hh.tbl->num_items++;                                                   \
+  HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt);                    \
+  HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed);      \
+  HASH_BLOOM_ADD((head)->hh.tbl, hashval);                                       \
+  HASH_EMIT_KEY(hh, head, keyptr, keylen_in);                                    \
+} while (0)
+
+#endif
+
+
+#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh,head,keyptr,keylen_in,hashval,add,cmpfcn) \
+do {                                                                             \
+  IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; )                                     \
+  (add)->hh.hashv = (hashval);                                                   \
+  (add)->hh.key = (char*) (keyptr);                                              \
+  (add)->hh.keylen = (unsigned) (keylen_in);                                     \
+  if (!(head)) {                                                                 \
+    (add)->hh.next = NULL;                                                       \
+    (add)->hh.prev = NULL;                                                       \
+    HASH_MAKE_TABLE(hh, add, _ha_oomed);                                         \
+    IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { )                                    \
+      (head) = (add);                                                            \
+    IF_HASH_NONFATAL_OOM( } )                                                    \
+  } else {                                                                       \
+    void *_hs_iter = (head);                                                     \
+    (add)->hh.tbl = (head)->hh.tbl;                                              \
+    HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn);                                 \
+    if (_hs_iter) {                                                              \
+      (add)->hh.next = _hs_iter;                                                 \
+      if (((add)->hh.prev = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev)) {     \
+        HH_FROM_ELMT((head)->hh.tbl, (add)->hh.prev)->next = (add);              \
+      } else {                                                                   \
+        (head) = (add);                                                          \
+      }                                                                          \
+      HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev = (add);                      \
+    } else {                                                                     \
+      HASH_APPEND_LIST(hh, head, add);                                           \
+    }                                                                            \
+  }                                                                              \
+  HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed);       \
+  HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE_INORDER");                    \
+} while (0)
+
+#define HASH_ADD_KEYPTR_INORDER(hh,head,keyptr,keylen_in,add,cmpfcn)             \
+do {                                                                             \
+  unsigned _hs_hashv;                                                            \
+  HASH_VALUE(keyptr, keylen_in, _hs_hashv);                                      \
+  HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, _hs_hashv, add, cmpfcn); \
+} while (0)
+
+#define HASH_ADD_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,cmpfcn) \
+  HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn)
+
+#define HASH_ADD_INORDER(hh,head,fieldname,keylen_in,add,cmpfcn)                 \
+  HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, cmpfcn)
+
+#define HASH_ADD_KEYPTR_BYHASHVALUE(hh,head,keyptr,keylen_in,hashval,add)        \
+do {                                                                             \
+  IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; )                                     \
+  (add)->hh.hashv = (hashval);                                                   \
+  (add)->hh.key = (char*) (keyptr);                                              \
+  (add)->hh.keylen = (unsigned) (keylen_in);                                     \
+  if (!(head)) {                                                                 \
+    (add)->hh.next = NULL;                                                       \
+    (add)->hh.prev = NULL;                                                       \
+    HASH_MAKE_TABLE(hh, add, _ha_oomed);                                         \
+    IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { )                                    \
+      (head) = (add);                                                            \
+    IF_HASH_NONFATAL_OOM( } )                                                    \
+  } else {                                                                       \
+    (add)->hh.tbl = (head)->hh.tbl;                                              \
+    HASH_APPEND_LIST(hh, head, add);                                             \
+  }                                                                              \
+  HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed);       \
+  HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE");                            \
+} while (0)
+
+#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add)                            \
+do {                                                                             \
+  unsigned _ha_hashv;                                                            \
+  HASH_VALUE(keyptr, keylen_in, _ha_hashv);                                      \
+  HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, add);      \
+} while (0)
+
+#define HASH_ADD_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add)            \
+  HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add)
+
+#define HASH_ADD(hh,head,fieldname,keylen_in,add)                                \
+  HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add)
+
+#define HASH_TO_BKT(hashv,num_bkts,bkt)                                          \
+do {                                                                             \
+  bkt = ((hashv) & ((num_bkts) - 1U));                                           \
+} while (0)
+
+/* delete "delptr" from the hash table.
+ * "the usual" patch-up process for the app-order doubly-linked-list.
+ * The use of _hd_hh_del below deserves special explanation.
+ * These used to be expressed using (delptr) but that led to a bug
+ * if someone used the same symbol for the head and deletee, like
+ *  HASH_DELETE(hh,users,users);
+ * We want that to work, but by changing the head (users) below
+ * we were forfeiting our ability to further refer to the deletee (users)
+ * in the patch-up process. Solution: use scratch space to
+ * copy the deletee pointer, then the latter references are via that
+ * scratch pointer rather than through the repointed (users) symbol.
+ */
+#define HASH_DELETE(hh,head,delptr)                                              \
+    HASH_DELETE_HH(hh, head, &(delptr)->hh)
+
+#define HASH_DELETE_HH(hh,head,delptrhh)                                         \
+do {                                                                             \
+  struct UT_hash_handle *_hd_hh_del = (delptrhh);                                \
+  if ((_hd_hh_del->prev == NULL) && (_hd_hh_del->next == NULL)) {                \
+    HASH_BLOOM_FREE((head)->hh.tbl);                                             \
+    uthash_free((head)->hh.tbl->buckets,                                         \
+                (head)->hh.tbl->num_buckets * sizeof(struct UT_hash_bucket));    \
+    uthash_free((head)->hh.tbl, sizeof(UT_hash_table));                          \
+    (head) = NULL;                                                               \
+  } else {                                                                       \
+    unsigned _hd_bkt;                                                            \
+    if (_hd_hh_del == (head)->hh.tbl->tail) {                                    \
+      (head)->hh.tbl->tail = HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev);     \
+    }                                                                            \
+    if (_hd_hh_del->prev != NULL) {                                              \
+      HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev)->next = _hd_hh_del->next;   \
+    } else {                                                                     \
+      DECLTYPE_ASSIGN(head, _hd_hh_del->next);                                   \
+    }                                                                            \
+    if (_hd_hh_del->next != NULL) {                                              \
+      HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->next)->prev = _hd_hh_del->prev;   \
+    }                                                                            \
+    HASH_TO_BKT(_hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt);        \
+    HASH_DEL_IN_BKT((head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del);               \
+    (head)->hh.tbl->num_items--;                                                 \
+  }                                                                              \
+  HASH_FSCK(hh, head, "HASH_DELETE_HH");                                         \
+} while (0)
+
+/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */
+#define HASH_FIND_STR(head,findstr,out)                                          \
+do {                                                                             \
+    unsigned _uthash_hfstr_keylen = (unsigned)uthash_strlen(findstr);            \
+    HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out);                     \
+} while (0)
+#define HASH_ADD_STR(head,strfield,add)                                          \
+do {                                                                             \
+    unsigned _uthash_hastr_keylen = (unsigned)uthash_strlen((add)->strfield);    \
+    HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add);                  \
+} while (0)
+#define HASH_REPLACE_STR(head,strfield,add,replaced)                             \
+do {                                                                             \
+    unsigned _uthash_hrstr_keylen = (unsigned)uthash_strlen((add)->strfield);    \
+    HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, replaced);    \
+} while (0)
+#define HASH_FIND_INT(head,findint,out)                                          \
+    HASH_FIND(hh,head,findint,sizeof(int),out)
+#define HASH_ADD_INT(head,intfield,add)                                          \
+    HASH_ADD(hh,head,intfield,sizeof(int),add)
+#define HASH_REPLACE_INT(head,intfield,add,replaced)                             \
+    HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced)
+#define HASH_FIND_PTR(head,findptr,out)                                          \
+    HASH_FIND(hh,head,findptr,sizeof(void *),out)
+#define HASH_ADD_PTR(head,ptrfield,add)                                          \
+    HASH_ADD(hh,head,ptrfield,sizeof(void *),add)
+#define HASH_REPLACE_PTR(head,ptrfield,add,replaced)                             \
+    HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced)
+#define HASH_DEL(head,delptr)                                                    \
+    HASH_DELETE(hh,head,delptr)
+
+/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined.
+ * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined.
+ */
+#ifdef HASH_DEBUG
+#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0)
+#define HASH_FSCK(hh,head,where)                                                 \
+do {                                                                             \
+  struct UT_hash_handle *_thh;                                                   \
+  if (head) {                                                                    \
+    unsigned _bkt_i;                                                             \
+    unsigned _count = 0;                                                         \
+    char *_prev;                                                                 \
+    for (_bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; ++_bkt_i) {           \
+      unsigned _bkt_count = 0;                                                   \
+      _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head;                            \
+      _prev = NULL;                                                              \
+      while (_thh) {                                                             \
+        if (_prev != (char*)(_thh->hh_prev)) {                                   \
+          HASH_OOPS("%s: invalid hh_prev %p, actual %p\n",                       \
+              (where), (void*)_thh->hh_prev, (void*)_prev);                      \
+        }                                                                        \
+        _bkt_count++;                                                            \
+        _prev = (char*)(_thh);                                                   \
+        _thh = _thh->hh_next;                                                    \
+      }                                                                          \
+      _count += _bkt_count;                                                      \
+      if ((head)->hh.tbl->buckets[_bkt_i].count !=  _bkt_count) {                \
+        HASH_OOPS("%s: invalid bucket count %u, actual %u\n",                    \
+            (where), (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count);         \
+      }                                                                          \
+    }                                                                            \
+    if (_count != (head)->hh.tbl->num_items) {                                   \
+      HASH_OOPS("%s: invalid hh item count %u, actual %u\n",                     \
+          (where), (head)->hh.tbl->num_items, _count);                           \
+    }                                                                            \
+    _count = 0;                                                                  \
+    _prev = NULL;                                                                \
+    _thh =  &(head)->hh;                                                         \
+    while (_thh) {                                                               \
+      _count++;                                                                  \
+      if (_prev != (char*)_thh->prev) {                                          \
+        HASH_OOPS("%s: invalid prev %p, actual %p\n",                            \
+            (where), (void*)_thh->prev, (void*)_prev);                           \
+      }                                                                          \
+      _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh);                         \
+      _thh = (_thh->next ? HH_FROM_ELMT((head)->hh.tbl, _thh->next) : NULL);     \
+    }                                                                            \
+    if (_count != (head)->hh.tbl->num_items) {                                   \
+      HASH_OOPS("%s: invalid app item count %u, actual %u\n",                    \
+          (where), (head)->hh.tbl->num_items, _count);                           \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+#else
+#define HASH_FSCK(hh,head,where)
+#endif
+
+/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to
+ * the descriptor to which this macro is defined for tuning the hash function.
+ * The app can #include  to get the prototype for write(2). */
+#ifdef HASH_EMIT_KEYS
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)                                   \
+do {                                                                             \
+  unsigned _klen = fieldlen;                                                     \
+  write(HASH_EMIT_KEYS, &_klen, sizeof(_klen));                                  \
+  write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen);                        \
+} while (0)
+#else
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)
+#endif
+
+/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */
+#ifdef HASH_FUNCTION
+#define HASH_FCN HASH_FUNCTION
+#else
+#define HASH_FCN HASH_JEN
+#endif
+
+/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */
+#define HASH_BER(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned _hb_keylen = (unsigned)keylen;                                        \
+  const unsigned char *_hb_key = (const unsigned char*)(key);                    \
+  (hashv) = 0;                                                                   \
+  while (_hb_keylen-- != 0U) {                                                   \
+    (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++;                           \
+  }                                                                              \
+} while (0)
+
+
+/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at
+ * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
+#define HASH_SAX(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned _sx_i;                                                                \
+  const unsigned char *_hs_key = (const unsigned char*)(key);                    \
+  hashv = 0;                                                                     \
+  for (_sx_i=0; _sx_i < keylen; _sx_i++) {                                       \
+    hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i];                       \
+  }                                                                              \
+} while (0)
+/* FNV-1a variation */
+#define HASH_FNV(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned _fn_i;                                                                \
+  const unsigned char *_hf_key = (const unsigned char*)(key);                    \
+  (hashv) = 2166136261U;                                                         \
+  for (_fn_i=0; _fn_i < keylen; _fn_i++) {                                       \
+    hashv = hashv ^ _hf_key[_fn_i];                                              \
+    hashv = hashv * 16777619U;                                                   \
+  }                                                                              \
+} while (0)
+
+#define HASH_OAT(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned _ho_i;                                                                \
+  const unsigned char *_ho_key=(const unsigned char*)(key);                      \
+  hashv = 0;                                                                     \
+  for(_ho_i=0; _ho_i < keylen; _ho_i++) {                                        \
+      hashv += _ho_key[_ho_i];                                                   \
+      hashv += (hashv << 10);                                                    \
+      hashv ^= (hashv >> 6);                                                     \
+  }                                                                              \
+  hashv += (hashv << 3);                                                         \
+  hashv ^= (hashv >> 11);                                                        \
+  hashv += (hashv << 15);                                                        \
+} while (0)
+
+#define HASH_JEN_MIX(a,b,c)                                                      \
+do {                                                                             \
+  a -= b; a -= c; a ^= ( c >> 13 );                                              \
+  b -= c; b -= a; b ^= ( a << 8 );                                               \
+  c -= a; c -= b; c ^= ( b >> 13 );                                              \
+  a -= b; a -= c; a ^= ( c >> 12 );                                              \
+  b -= c; b -= a; b ^= ( a << 16 );                                              \
+  c -= a; c -= b; c ^= ( b >> 5 );                                               \
+  a -= b; a -= c; a ^= ( c >> 3 );                                               \
+  b -= c; b -= a; b ^= ( a << 10 );                                              \
+  c -= a; c -= b; c ^= ( b >> 15 );                                              \
+} while (0)
+
+#define HASH_JEN(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned _hj_i,_hj_j,_hj_k;                                                    \
+  unsigned const char *_hj_key=(unsigned const char*)(key);                      \
+  hashv = 0xfeedbeefu;                                                           \
+  _hj_i = _hj_j = 0x9e3779b9u;                                                   \
+  _hj_k = (unsigned)(keylen);                                                    \
+  while (_hj_k >= 12U) {                                                         \
+    _hj_i +=    (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 )                      \
+        + ( (unsigned)_hj_key[2] << 16 )                                         \
+        + ( (unsigned)_hj_key[3] << 24 ) );                                      \
+    _hj_j +=    (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 )                      \
+        + ( (unsigned)_hj_key[6] << 16 )                                         \
+        + ( (unsigned)_hj_key[7] << 24 ) );                                      \
+    hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 )                         \
+        + ( (unsigned)_hj_key[10] << 16 )                                        \
+        + ( (unsigned)_hj_key[11] << 24 ) );                                     \
+                                                                                 \
+     HASH_JEN_MIX(_hj_i, _hj_j, hashv);                                          \
+                                                                                 \
+     _hj_key += 12;                                                              \
+     _hj_k -= 12U;                                                               \
+  }                                                                              \
+  hashv += (unsigned)(keylen);                                                   \
+  switch ( _hj_k ) {                                                             \
+    case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); /* FALLTHROUGH */         \
+    case 10: hashv += ( (unsigned)_hj_key[9] << 16 );  /* FALLTHROUGH */         \
+    case 9:  hashv += ( (unsigned)_hj_key[8] << 8 );   /* FALLTHROUGH */         \
+    case 8:  _hj_j += ( (unsigned)_hj_key[7] << 24 );  /* FALLTHROUGH */         \
+    case 7:  _hj_j += ( (unsigned)_hj_key[6] << 16 );  /* FALLTHROUGH */         \
+    case 6:  _hj_j += ( (unsigned)_hj_key[5] << 8 );   /* FALLTHROUGH */         \
+    case 5:  _hj_j += _hj_key[4];                      /* FALLTHROUGH */         \
+    case 4:  _hj_i += ( (unsigned)_hj_key[3] << 24 );  /* FALLTHROUGH */         \
+    case 3:  _hj_i += ( (unsigned)_hj_key[2] << 16 );  /* FALLTHROUGH */         \
+    case 2:  _hj_i += ( (unsigned)_hj_key[1] << 8 );   /* FALLTHROUGH */         \
+    case 1:  _hj_i += _hj_key[0];                                                \
+  }                                                                              \
+  HASH_JEN_MIX(_hj_i, _hj_j, hashv);                                             \
+} while (0)
+
+/* The Paul Hsieh hash function */
+#undef get16bits
+#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__)             \
+  || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__)
+#define get16bits(d) (*((const uint16_t *) (d)))
+#endif
+
+#if !defined (get16bits)
+#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8)             \
+                       +(uint32_t)(((const uint8_t *)(d))[0]) )
+#endif
+#define HASH_SFH(key,keylen,hashv)                                               \
+do {                                                                             \
+  unsigned const char *_sfh_key=(unsigned const char*)(key);                     \
+  uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen;                                \
+                                                                                 \
+  unsigned _sfh_rem = _sfh_len & 3U;                                             \
+  _sfh_len >>= 2;                                                                \
+  hashv = 0xcafebabeu;                                                           \
+                                                                                 \
+  /* Main loop */                                                                \
+  for (;_sfh_len > 0U; _sfh_len--) {                                             \
+    hashv    += get16bits (_sfh_key);                                            \
+    _sfh_tmp  = ((uint32_t)(get16bits (_sfh_key+2)) << 11) ^ hashv;              \
+    hashv     = (hashv << 16) ^ _sfh_tmp;                                        \
+    _sfh_key += 2U*sizeof (uint16_t);                                            \
+    hashv    += hashv >> 11;                                                     \
+  }                                                                              \
+                                                                                 \
+  /* Handle end cases */                                                         \
+  switch (_sfh_rem) {                                                            \
+    case 3: hashv += get16bits (_sfh_key);                                       \
+            hashv ^= hashv << 16;                                                \
+            hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)]) << 18;              \
+            hashv += hashv >> 11;                                                \
+            break;                                                               \
+    case 2: hashv += get16bits (_sfh_key);                                       \
+            hashv ^= hashv << 11;                                                \
+            hashv += hashv >> 17;                                                \
+            break;                                                               \
+    case 1: hashv += *_sfh_key;                                                  \
+            hashv ^= hashv << 10;                                                \
+            hashv += hashv >> 1;                                                 \
+  }                                                                              \
+                                                                                 \
+  /* Force "avalanching" of final 127 bits */                                    \
+  hashv ^= hashv << 3;                                                           \
+  hashv += hashv >> 5;                                                           \
+  hashv ^= hashv << 4;                                                           \
+  hashv += hashv >> 17;                                                          \
+  hashv ^= hashv << 25;                                                          \
+  hashv += hashv >> 6;                                                           \
+} while (0)
+
+#ifdef HASH_USING_NO_STRICT_ALIASING
+/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads.
+ * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error.
+ * MurmurHash uses the faster approach only on CPU's where we know it's safe.
+ *
+ * Note the preprocessor built-in defines can be emitted using:
+ *
+ *   gcc -m64 -dM -E - < /dev/null                  (on gcc)
+ *   cc -## a.c (where a.c is a simple test file)   (Sun Studio)
+ */
+#if (defined(__i386__) || defined(__x86_64__)  || defined(_M_IX86))
+#define MUR_GETBLOCK(p,i) p[i]
+#else /* non intel */
+#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 3UL) == 0UL)
+#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 3UL) == 1UL)
+#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 3UL) == 2UL)
+#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 3UL) == 3UL)
+#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL))
+#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__))
+#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24))
+#define MUR_TWO_TWO(p)   ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >>  8))
+#else /* assume little endian non-intel */
+#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24))
+#define MUR_TWO_TWO(p)   ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16))
+#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) <<  8))
+#endif
+#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) :           \
+                            (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \
+                             (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) :  \
+                                                      MUR_ONE_THREE(p))))
+#endif
+#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
+#define MUR_FMIX(_h) \
+do {                 \
+  _h ^= _h >> 16;    \
+  _h *= 0x85ebca6bu; \
+  _h ^= _h >> 13;    \
+  _h *= 0xc2b2ae35u; \
+  _h ^= _h >> 16;    \
+} while (0)
+
+#define HASH_MUR(key,keylen,hashv)                                     \
+do {                                                                   \
+  const uint8_t *_mur_data = (const uint8_t*)(key);                    \
+  const int _mur_nblocks = (int)(keylen) / 4;                          \
+  uint32_t _mur_h1 = 0xf88D5353u;                                      \
+  uint32_t _mur_c1 = 0xcc9e2d51u;                                      \
+  uint32_t _mur_c2 = 0x1b873593u;                                      \
+  uint32_t _mur_k1 = 0;                                                \
+  const uint8_t *_mur_tail;                                            \
+  const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+(_mur_nblocks*4)); \
+  int _mur_i;                                                          \
+  for (_mur_i = -_mur_nblocks; _mur_i != 0; _mur_i++) {                \
+    _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i);                        \
+    _mur_k1 *= _mur_c1;                                                \
+    _mur_k1 = MUR_ROTL32(_mur_k1,15);                                  \
+    _mur_k1 *= _mur_c2;                                                \
+                                                                       \
+    _mur_h1 ^= _mur_k1;                                                \
+    _mur_h1 = MUR_ROTL32(_mur_h1,13);                                  \
+    _mur_h1 = (_mur_h1*5U) + 0xe6546b64u;                              \
+  }                                                                    \
+  _mur_tail = (const uint8_t*)(_mur_data + (_mur_nblocks*4));          \
+  _mur_k1=0;                                                           \
+  switch ((keylen) & 3U) {                                             \
+    case 0: break;                                                     \
+    case 3: _mur_k1 ^= (uint32_t)_mur_tail[2] << 16; /* FALLTHROUGH */ \
+    case 2: _mur_k1 ^= (uint32_t)_mur_tail[1] << 8;  /* FALLTHROUGH */ \
+    case 1: _mur_k1 ^= (uint32_t)_mur_tail[0];                         \
+    _mur_k1 *= _mur_c1;                                                \
+    _mur_k1 = MUR_ROTL32(_mur_k1,15);                                  \
+    _mur_k1 *= _mur_c2;                                                \
+    _mur_h1 ^= _mur_k1;                                                \
+  }                                                                    \
+  _mur_h1 ^= (uint32_t)(keylen);                                       \
+  MUR_FMIX(_mur_h1);                                                   \
+  hashv = _mur_h1;                                                     \
+} while (0)
+#endif  /* HASH_USING_NO_STRICT_ALIASING */
+
+/* iterate over items in a known bucket to find desired item */
+#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,hashval,out)               \
+do {                                                                             \
+  if ((head).hh_head != NULL) {                                                  \
+    DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head));                     \
+  } else {                                                                       \
+    (out) = NULL;                                                                \
+  }                                                                              \
+  while ((out) != NULL) {                                                        \
+    if ((out)->hh.hashv == (hashval) && (out)->hh.keylen == (keylen_in)) {       \
+      if (HASH_KEYCMP((out)->hh.key, keyptr, keylen_in) == 0) {              \
+        break;                                                                   \
+      }                                                                          \
+    }                                                                            \
+    if ((out)->hh.hh_next != NULL) {                                             \
+      DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next));                \
+    } else {                                                                     \
+      (out) = NULL;                                                              \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+
+/* add an item to a bucket  */
+#define HASH_ADD_TO_BKT(head,hh,addhh,oomed)                                     \
+do {                                                                             \
+  UT_hash_bucket *_ha_head = &(head);                                            \
+  _ha_head->count++;                                                             \
+  (addhh)->hh_next = _ha_head->hh_head;                                          \
+  (addhh)->hh_prev = NULL;                                                       \
+  if (_ha_head->hh_head != NULL) {                                               \
+    _ha_head->hh_head->hh_prev = (addhh);                                        \
+  }                                                                              \
+  _ha_head->hh_head = (addhh);                                                   \
+  if ((_ha_head->count >= ((_ha_head->expand_mult + 1U) * HASH_BKT_CAPACITY_THRESH)) \
+      && !(addhh)->tbl->noexpand) {                                              \
+    HASH_EXPAND_BUCKETS(addhh,(addhh)->tbl, oomed);                              \
+    IF_HASH_NONFATAL_OOM(                                                        \
+      if (oomed) {                                                               \
+        HASH_DEL_IN_BKT(head,addhh);                                             \
+      }                                                                          \
+    )                                                                            \
+  }                                                                              \
+} while (0)
+
+/* remove an item from a given bucket */
+#define HASH_DEL_IN_BKT(head,delhh)                                              \
+do {                                                                             \
+  UT_hash_bucket *_hd_head = &(head);                                            \
+  _hd_head->count--;                                                             \
+  if (_hd_head->hh_head == (delhh)) {                                            \
+    _hd_head->hh_head = (delhh)->hh_next;                                        \
+  }                                                                              \
+  if ((delhh)->hh_prev) {                                                        \
+    (delhh)->hh_prev->hh_next = (delhh)->hh_next;                                \
+  }                                                                              \
+  if ((delhh)->hh_next) {                                                        \
+    (delhh)->hh_next->hh_prev = (delhh)->hh_prev;                                \
+  }                                                                              \
+} while (0)
+
+/* Bucket expansion has the effect of doubling the number of buckets
+ * and redistributing the items into the new buckets. Ideally the
+ * items will distribute more or less evenly into the new buckets
+ * (the extent to which this is true is a measure of the quality of
+ * the hash function as it applies to the key domain).
+ *
+ * With the items distributed into more buckets, the chain length
+ * (item count) in each bucket is reduced. Thus by expanding buckets
+ * the hash keeps a bound on the chain length. This bounded chain
+ * length is the essence of how a hash provides constant time lookup.
+ *
+ * The calculation of tbl->ideal_chain_maxlen below deserves some
+ * explanation. First, keep in mind that we're calculating the ideal
+ * maximum chain length based on the *new* (doubled) bucket count.
+ * In fractions this is just n/b (n=number of items,b=new num buckets).
+ * Since the ideal chain length is an integer, we want to calculate
+ * ceil(n/b). We don't depend on floating point arithmetic in this
+ * hash, so to calculate ceil(n/b) with integers we could write
+ *
+ *      ceil(n/b) = (n/b) + ((n%b)?1:0)
+ *
+ * and in fact a previous version of this hash did just that.
+ * But now we have improved things a bit by recognizing that b is
+ * always a power of two. We keep its base 2 log handy (call it lb),
+ * so now we can write this with a bit shift and logical AND:
+ *
+ *      ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0)
+ *
+ */
+#define HASH_EXPAND_BUCKETS(hh,tbl,oomed)                                        \
+do {                                                                             \
+  unsigned _he_bkt;                                                              \
+  unsigned _he_bkt_i;                                                            \
+  struct UT_hash_handle *_he_thh, *_he_hh_nxt;                                   \
+  UT_hash_bucket *_he_new_buckets, *_he_newbkt;                                  \
+  _he_new_buckets = (UT_hash_bucket*)uthash_malloc(                              \
+           2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket));            \
+  if (!_he_new_buckets) {                                                        \
+    HASH_RECORD_OOM(oomed);                                                      \
+  } else {                                                                       \
+    uthash_bzero(_he_new_buckets,                                                \
+        2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket));               \
+    (tbl)->ideal_chain_maxlen =                                                  \
+       ((tbl)->num_items >> ((tbl)->log2_num_buckets+1U)) +                      \
+       ((((tbl)->num_items & (((tbl)->num_buckets*2U)-1U)) != 0U) ? 1U : 0U);    \
+    (tbl)->nonideal_items = 0;                                                   \
+    for (_he_bkt_i = 0; _he_bkt_i < (tbl)->num_buckets; _he_bkt_i++) {           \
+      _he_thh = (tbl)->buckets[ _he_bkt_i ].hh_head;                             \
+      while (_he_thh != NULL) {                                                  \
+        _he_hh_nxt = _he_thh->hh_next;                                           \
+        HASH_TO_BKT(_he_thh->hashv, (tbl)->num_buckets * 2U, _he_bkt);           \
+        _he_newbkt = &(_he_new_buckets[_he_bkt]);                                \
+        if (++(_he_newbkt->count) > (tbl)->ideal_chain_maxlen) {                 \
+          (tbl)->nonideal_items++;                                               \
+          if (_he_newbkt->count > _he_newbkt->expand_mult * (tbl)->ideal_chain_maxlen) { \
+            _he_newbkt->expand_mult++;                                           \
+          }                                                                      \
+        }                                                                        \
+        _he_thh->hh_prev = NULL;                                                 \
+        _he_thh->hh_next = _he_newbkt->hh_head;                                  \
+        if (_he_newbkt->hh_head != NULL) {                                       \
+          _he_newbkt->hh_head->hh_prev = _he_thh;                                \
+        }                                                                        \
+        _he_newbkt->hh_head = _he_thh;                                           \
+        _he_thh = _he_hh_nxt;                                                    \
+      }                                                                          \
+    }                                                                            \
+    uthash_free((tbl)->buckets, (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \
+    (tbl)->num_buckets *= 2U;                                                    \
+    (tbl)->log2_num_buckets++;                                                   \
+    (tbl)->buckets = _he_new_buckets;                                            \
+    (tbl)->ineff_expands = ((tbl)->nonideal_items > ((tbl)->num_items >> 1)) ?   \
+        ((tbl)->ineff_expands+1U) : 0U;                                          \
+    if ((tbl)->ineff_expands > 1U) {                                             \
+      (tbl)->noexpand = 1;                                                       \
+      uthash_noexpand_fyi(tbl);                                                  \
+    }                                                                            \
+    uthash_expand_fyi(tbl);                                                      \
+  }                                                                              \
+} while (0)
+
+
+/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */
+/* Note that HASH_SORT assumes the hash handle name to be hh.
+ * HASH_SRT was added to allow the hash handle name to be passed in. */
+#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn)
+#define HASH_SRT(hh,head,cmpfcn)                                                 \
+do {                                                                             \
+  unsigned _hs_i;                                                                \
+  unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize;               \
+  struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail;            \
+  if (head != NULL) {                                                            \
+    _hs_insize = 1;                                                              \
+    _hs_looping = 1;                                                             \
+    _hs_list = &((head)->hh);                                                    \
+    while (_hs_looping != 0U) {                                                  \
+      _hs_p = _hs_list;                                                          \
+      _hs_list = NULL;                                                           \
+      _hs_tail = NULL;                                                           \
+      _hs_nmerges = 0;                                                           \
+      while (_hs_p != NULL) {                                                    \
+        _hs_nmerges++;                                                           \
+        _hs_q = _hs_p;                                                           \
+        _hs_psize = 0;                                                           \
+        for (_hs_i = 0; _hs_i < _hs_insize; ++_hs_i) {                           \
+          _hs_psize++;                                                           \
+          _hs_q = ((_hs_q->next != NULL) ?                                       \
+            HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL);                   \
+          if (_hs_q == NULL) {                                                   \
+            break;                                                               \
+          }                                                                      \
+        }                                                                        \
+        _hs_qsize = _hs_insize;                                                  \
+        while ((_hs_psize != 0U) || ((_hs_qsize != 0U) && (_hs_q != NULL))) {    \
+          if (_hs_psize == 0U) {                                                 \
+            _hs_e = _hs_q;                                                       \
+            _hs_q = ((_hs_q->next != NULL) ?                                     \
+              HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL);                 \
+            _hs_qsize--;                                                         \
+          } else if ((_hs_qsize == 0U) || (_hs_q == NULL)) {                     \
+            _hs_e = _hs_p;                                                       \
+            if (_hs_p != NULL) {                                                 \
+              _hs_p = ((_hs_p->next != NULL) ?                                   \
+                HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL);               \
+            }                                                                    \
+            _hs_psize--;                                                         \
+          } else if ((cmpfcn(                                                    \
+                DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_p)),             \
+                DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_q))              \
+                )) <= 0) {                                                       \
+            _hs_e = _hs_p;                                                       \
+            if (_hs_p != NULL) {                                                 \
+              _hs_p = ((_hs_p->next != NULL) ?                                   \
+                HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL);               \
+            }                                                                    \
+            _hs_psize--;                                                         \
+          } else {                                                               \
+            _hs_e = _hs_q;                                                       \
+            _hs_q = ((_hs_q->next != NULL) ?                                     \
+              HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL);                 \
+            _hs_qsize--;                                                         \
+          }                                                                      \
+          if ( _hs_tail != NULL ) {                                              \
+            _hs_tail->next = ((_hs_e != NULL) ?                                  \
+              ELMT_FROM_HH((head)->hh.tbl, _hs_e) : NULL);                       \
+          } else {                                                               \
+            _hs_list = _hs_e;                                                    \
+          }                                                                      \
+          if (_hs_e != NULL) {                                                   \
+            _hs_e->prev = ((_hs_tail != NULL) ?                                  \
+              ELMT_FROM_HH((head)->hh.tbl, _hs_tail) : NULL);                    \
+          }                                                                      \
+          _hs_tail = _hs_e;                                                      \
+        }                                                                        \
+        _hs_p = _hs_q;                                                           \
+      }                                                                          \
+      if (_hs_tail != NULL) {                                                    \
+        _hs_tail->next = NULL;                                                   \
+      }                                                                          \
+      if (_hs_nmerges <= 1U) {                                                   \
+        _hs_looping = 0;                                                         \
+        (head)->hh.tbl->tail = _hs_tail;                                         \
+        DECLTYPE_ASSIGN(head, ELMT_FROM_HH((head)->hh.tbl, _hs_list));           \
+      }                                                                          \
+      _hs_insize *= 2U;                                                          \
+    }                                                                            \
+    HASH_FSCK(hh, head, "HASH_SRT");                                             \
+  }                                                                              \
+} while (0)
+
+/* This function selects items from one hash into another hash.
+ * The end result is that the selected items have dual presence
+ * in both hashes. There is no copy of the items made; rather
+ * they are added into the new hash through a secondary hash
+ * hash handle that must be present in the structure. */
+#define HASH_SELECT(hh_dst, dst, hh_src, src, cond)                              \
+do {                                                                             \
+  unsigned _src_bkt, _dst_bkt;                                                   \
+  void *_last_elt = NULL, *_elt;                                                 \
+  UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL;                         \
+  ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst));                 \
+  if ((src) != NULL) {                                                           \
+    for (_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) {    \
+      for (_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head;               \
+        _src_hh != NULL;                                                         \
+        _src_hh = _src_hh->hh_next) {                                            \
+        _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh);                         \
+        if (cond(_elt)) {                                                        \
+          IF_HASH_NONFATAL_OOM( int _hs_oomed = 0; )                             \
+          _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho);                 \
+          _dst_hh->key = _src_hh->key;                                           \
+          _dst_hh->keylen = _src_hh->keylen;                                     \
+          _dst_hh->hashv = _src_hh->hashv;                                       \
+          _dst_hh->prev = _last_elt;                                             \
+          _dst_hh->next = NULL;                                                  \
+          if (_last_elt_hh != NULL) {                                            \
+            _last_elt_hh->next = _elt;                                           \
+          }                                                                      \
+          if ((dst) == NULL) {                                                   \
+            DECLTYPE_ASSIGN(dst, _elt);                                          \
+            HASH_MAKE_TABLE(hh_dst, dst, _hs_oomed);                             \
+            IF_HASH_NONFATAL_OOM(                                                \
+              if (_hs_oomed) {                                                   \
+                uthash_nonfatal_oom(_elt);                                       \
+                (dst) = NULL;                                                    \
+                continue;                                                        \
+              }                                                                  \
+            )                                                                    \
+          } else {                                                               \
+            _dst_hh->tbl = (dst)->hh_dst.tbl;                                    \
+          }                                                                      \
+          HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt);      \
+          HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt], hh_dst, _dst_hh, _hs_oomed); \
+          (dst)->hh_dst.tbl->num_items++;                                        \
+          IF_HASH_NONFATAL_OOM(                                                  \
+            if (_hs_oomed) {                                                     \
+              HASH_ROLLBACK_BKT(hh_dst, dst, _dst_hh);                           \
+              HASH_DELETE_HH(hh_dst, dst, _dst_hh);                              \
+              _dst_hh->tbl = NULL;                                               \
+              uthash_nonfatal_oom(_elt);                                         \
+              continue;                                                          \
+            }                                                                    \
+          )                                                                      \
+          HASH_BLOOM_ADD(_dst_hh->tbl, _dst_hh->hashv);                          \
+          _last_elt = _elt;                                                      \
+          _last_elt_hh = _dst_hh;                                                \
+        }                                                                        \
+      }                                                                          \
+    }                                                                            \
+  }                                                                              \
+  HASH_FSCK(hh_dst, dst, "HASH_SELECT");                                         \
+} while (0)
+
+#define HASH_CLEAR(hh,head)                                                      \
+do {                                                                             \
+  if ((head) != NULL) {                                                          \
+    HASH_BLOOM_FREE((head)->hh.tbl);                                             \
+    uthash_free((head)->hh.tbl->buckets,                                         \
+                (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket));      \
+    uthash_free((head)->hh.tbl, sizeof(UT_hash_table));                          \
+    (head) = NULL;                                                               \
+  }                                                                              \
+} while (0)
+
+#define HASH_OVERHEAD(hh,head)                                                   \
+ (((head) != NULL) ? (                                                           \
+ (size_t)(((head)->hh.tbl->num_items   * sizeof(UT_hash_handle))   +             \
+          ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket))   +             \
+           sizeof(UT_hash_table)                                   +             \
+           (HASH_BLOOM_BYTELEN))) : 0U)
+
+#ifdef NO_DECLTYPE
+#define HASH_ITER(hh,head,el,tmp)                                                \
+for(((el)=(head)), ((*(char**)(&(tmp)))=(char*)((head!=NULL)?(head)->hh.next:NULL)); \
+  (el) != NULL; ((el)=(tmp)), ((*(char**)(&(tmp)))=(char*)((tmp!=NULL)?(tmp)->hh.next:NULL)))
+#else
+#define HASH_ITER(hh,head,el,tmp)                                                \
+for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL));      \
+  (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL)))
+#endif
+
+/* obtain a count of items in the hash */
+#define HASH_COUNT(head) HASH_CNT(hh,head)
+#define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U)
+
+/** Hash bucket. */
+typedef struct UT_hash_bucket {
+   struct UT_hash_handle *hh_head;
+   unsigned count;
+
+   /* expand_mult is normally set to 0. In this situation, the max chain length
+    * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If
+    * the bucket's chain exceeds this length, bucket expansion is triggered).
+    * However, setting expand_mult to a non-zero value delays bucket expansion
+    * (that would be triggered by additions to this particular bucket)
+    * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH.
+    * (The multiplier is simply expand_mult+1). The whole idea of this
+    * multiplier is to reduce bucket expansions, since they are expensive, in
+    * situations where we know that a particular bucket tends to be overused.
+    * It is better to let its chain length grow to a longer yet-still-bounded
+    * value, than to do an O(n) bucket expansion too often.
+    */
+   unsigned expand_mult;
+
+} UT_hash_bucket;
+
+/* random signature used only to find hash tables in external analysis */
+#define HASH_SIGNATURE 0xa0111fe1u
+#define HASH_BLOOM_SIGNATURE 0xb12220f2u
+
+/** Hash table. */
+typedef struct UT_hash_table {
+   UT_hash_bucket *buckets;
+   unsigned num_buckets, log2_num_buckets;
+   unsigned num_items;
+   struct UT_hash_handle *tail; /* tail hh in app order, for fast append    */
+   ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */
+
+   /* in an ideal situation (all buckets used equally), no bucket would have
+    * more than ceil(#items/#buckets) items. that's the ideal chain length. */
+   unsigned ideal_chain_maxlen;
+
+   /* nonideal_items is the number of items in the hash whose chain position
+    * exceeds the ideal chain maxlen. these items pay the penalty for an uneven
+    * hash distribution; reaching them in a chain traversal takes >ideal steps */
+   unsigned nonideal_items;
+
+   /* ineffective expands occur when a bucket doubling was performed, but
+    * afterward, more than half the items in the hash had nonideal chain
+    * positions. If this happens on two consecutive expansions we inhibit any
+    * further expansion, as it's not helping; this happens when the hash
+    * function isn't a good fit for the key domain. When expansion is inhibited
+    * the hash will still work, albeit no longer in constant time. */
+   unsigned ineff_expands, noexpand;
+
+   uint32_t signature; /* used only to find hash tables in external analysis */
+#ifdef HASH_BLOOM
+   uint32_t bloom_sig; /* used only to test bloom exists in external analysis */
+   uint8_t *bloom_bv;
+   uint8_t bloom_nbits;
+#endif
+
+} UT_hash_table;
+
+/** Hash handle. */
+typedef struct UT_hash_handle {
+   struct UT_hash_table *tbl;
+   void *prev;                       /* prev element in app order      */
+   void *next;                       /* next element in app order      */
+   struct UT_hash_handle *hh_prev;   /* previous hh in bucket order    */
+   struct UT_hash_handle *hh_next;   /* next hh in bucket order        */
+   void *key;                        /* ptr to enclosing struct's key  */
+   unsigned keylen;                  /* enclosing struct's key len     */
+   unsigned hashv;                   /* result of hash-fcn(key)        */
+} UT_hash_handle;
+
+#endif /* UTHASH_H */
diff --git a/src/flib/CMakeLists.txt b/src/flib/CMakeLists.txt
index 3d2c17112520..42b70edb7899 100644
--- a/src/flib/CMakeLists.txt
+++ b/src/flib/CMakeLists.txt
@@ -52,9 +52,16 @@ if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
   target_compile_options (piof
     PRIVATE -ffree-line-length-none)
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
-  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all " )
   #    target_compile_options (piof
   #        PRIVATE -mismatch_all)
+elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+  set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ef")
+elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
+  set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal")
+endif()
+if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+     set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g")
 endif()
 
 # Look for c_sizeof capability
@@ -154,7 +161,7 @@ if (MPIF_H_PATH)
   check_macro (MPI_HAS_MPIIO
     NAME TryMPIIO.f90
     HINTS ${CMAKE_MODULE_PATH}
-    DEFINITIONS -I{MPIF_H_PATH}
+    DEFINITIONS -I${MPIF_H_PATH}
     COMMENT "whether MPIIO is supported")
   if (${MPI_HAS_MPIIO})
     message (STATUS "MPIIO verified and enabled.")
diff --git a/src/flib/Makefile.am b/src/flib/Makefile.am
new file mode 100644
index 000000000000..8260910e3dbf
--- /dev/null
+++ b/src/flib/Makefile.am
@@ -0,0 +1,158 @@
+## This is the automake file to build the PIO Fortran library.
+# Ed Hartnett 3/19/19
+
+# Turn off parallel builds in this directory.
+.NOTPARALLEL:
+
+# The library we are building.
+lib_LTLIBRARIES = libpiof.la
+
+AM_CPPFLAGS = -D_NETCDF -D_NETCDF4 -D_PETCDF
+
+# These linker flags specify libtool version info.
+# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
+# for information regarding incrementing `-version-info`.
+libpiof_la_LDFLAGS = -version-info 2:0:1
+
+# The library soure files.
+libpiof_la_LIBADD = libpio_nf.la libpio_kinds.la libpio_support.la	\
+libpiodarray.la libpionfatt.la libpionfget.la libpionfput.la		\
+libpiolib_mod.la libpio.la
+
+libpiof_la_SOURCES = pio_types.F90
+
+# Build these uninstalled convenience libraries.
+noinst_LTLIBRARIES = libpio_kinds.la libpio_types.la		\
+libpio_support.la libpio_nf.la libpiodarray.la libpionfatt.la	\
+libpionfget.la libpionfput.la libpiolib_mod.la libpio.la
+
+# The convenience libraries depends on their source.
+libpio_kinds_la_SOURCES = pio_kinds.F90
+libpio_types_la_SOURCES = pio_types.F90
+libpio_support_la_SOURCES = pio_support.F90
+libpio_nf_la_SOURCES = pio_nf.F90
+libpiodarray_la_SOURCES = piodarray.F90
+libpionfatt_la_SOURCES = pionfatt_mod.F90
+libpionfget_la_SOURCES = pionfget_mod.F90
+libpionfput_la_SOURCES = pionfput_mod.F90
+libpiolib_mod_la_SOURCES = piolib_mod.F90
+libpio_la_SOURCES = pio.F90
+
+# These F90 files are generated from .F90.in files, using the script
+# genf90.pl.
+piodarray.F90: piodarray.F90.in
+	${top_srcdir}/scripts/genf90.pl $< > $@
+pionfatt_mod.F90: pionfatt_mod.F90.in
+	${top_srcdir}/scripts/genf90.pl $< > $@
+pionfget_mod.F90: pionfget_mod.F90.in
+	${top_srcdir}/scripts/genf90.pl $< > $@
+pionfput_mod.F90: pionfput_mod.F90.in
+	${top_srcdir}/scripts/genf90.pl $< > $@
+
+# Each mod file depends on the .o file.
+pio_kinds.mod: pio_kinds.$(OBJEXT)
+pio_types.mod: pio_types.$(OBJEXT)
+pio_support.mod: pio_support.$(OBJEXT)
+pio_nf.mod: pio_nf.$(OBJEXT)
+piodarray.mod: piodarray.F90 piodarray.$(OBJEXT)
+pionfatt_mod.mod: pionfatt_mod.F90 pionfatt_mod.$(OBJEXT)
+pionfget_mod.mod: pionfget_mod.F90 pionfget_mod.$(OBJEXT)
+pionfput_mod.mod: pionfput_mod.F90 pionfput_mod.$(OBJEXT)
+piolib_mod.mod: piolib_mod.$(OBJEXT)
+pio.mod: pio.$(OBJEXT)
+
+# Some mod files depend on other mod files.
+pio.$(OBJEXT): pio_kinds.mod piolib_mod.mod pio_types.mod piodarray.mod \
+pio_nf.mod pionfatt_mod.mod pionfget_mod.mod pionfput_mod.mod pio_support.mod
+
+# Mod files are built and then installed as headers.
+MODFILES = pio_kinds.mod pio_types.mod pio_support.mod pio_nf.mod	\
+piodarray.mod pionfatt_mod.mod pionfget_mod.mod pionfput_mod.mod	\
+piolib_mod.mod pio.mod
+BUILT_SOURCES = $(MODFILES)
+include_HEADERS = $(MODFILES)
+
+# Doxygen does not cope well with pre-processor use in Fortran. So
+# create .f90 files from .F90 files by running the C
+# pre-processor. These will only be used by doxygen when --enable-docs
+# is used at configure.
+if BUILD_DOCS
+BUILT_SOURCES += piodarray.f90 piolib_mod.f90 pionfatt_mod.f90 pionfget_mod.f90 pionfput_mod.f90 pionfatt_mod_2.f90 pionfget_mod_2.f90
+piodarray.f90: piodarray.F90
+	$(CC) -E $< > $@
+piolib_mod.f90: piolib_mod.F90
+	$(CC) -E $< > $@
+pionfatt_mod.f90: pionfatt_mod.F90
+	$(CC) -E $< > $@
+pionfget_mod.f90: pionfget_mod.F90
+	$(CC) -E $< > $@
+pionfput_mod.f90: pionfput_mod.F90
+	$(CC) -E $< > $@
+
+# Unfortunately the genf90.pl script which generates these fortran
+# files has no way of handling doxygen documentation. So use sed to
+# insert some documentation lines to suppress warnings.
+pionfatt_mod_2.f90: pionfatt_mod.f90
+	sed -e '/^  integer function put_att_desc_real.*/i !> put real att' \
+            -e '/^  integer function put_att_desc_double.*/i !> put double att' \
+            -e '/^  integer function put_att_desc_int.*/i !> put int att' \
+            -e '/^  integer function put_att_vid_text.*/i !> put text att' \
+            -e '/^  integer function put_att_vid_real.*/i !> put real att' \
+            -e '/^  integer function put_att_vid_double.*/i !> put double att' \
+            -e '/^  integer function put_att_vid_int.*/i !> put int att' \
+            -e '/^  integer function put_att_id_int.*/i !> put int att' \
+            -e '/^  integer function put_att_id_real.*/i !> put att' \
+            -e '/^  integer function put_att_id_double.*/i !> put att' \
+            -e '/^  integer function put_att_id_text.*/i !> put att' \
+            -e '/^  integer function put_att_1d_id_text.*/i !> put att' \
+            -e '/^  integer function put_att_1d_id_text_internal.*/i !> put att' \
+            -e '/^  integer function get_att_id_text.*/i !> get att' \
+            -e '/^  integer function put_att_1d_id_internal_real.*/i !> att' \
+            -e '/^  integer function put_att_1d_id_internal_double.*/i !> att' \
+            -e '/^  integer function put_att_1d_id_internal_int.*/i !> att' \
+            -e '/^  integer function put_att_1d_vid_text.*/i !> att' \
+            -e '/^  integer function put_att_1d_vid_real.*/i !> att' \
+            -e '/^  integer function put_att_1d_vid_double.*/i !> att' \
+            -e '/^  integer function put_att_1d_vid_int.*/i !> att' \
+            -e '/^  integer function get_att_desc_real.*/i !> att' \
+            -e '/^  integer function get_att_desc_double.*/i !> att' \
+            -e '/^  integer function get_att_desc_int.*/i !> att' \
+            -e '/^  integer function get_att_desc_1d_real.*/i !> att' \
+            -e '/^  integer function get_att_desc_1d_double.*/i !> att' \
+            -e '/^  integer function get_att_id_real.*/i !> att' \
+            -e '/^  integer function get_att_id_double.*/i !> att' \
+            -e '/^  integer function get_att_text.*/i !> att' \
+            -e '/^  integer function get_att_real.*/i !> att' \
+            -e '/^  integer function get_att_double.*/i !> att' \
+            -e '/^  integer function get_att_int.*/i !> att' \
+            -e '/^  integer function get_att_1d_id_real.*/i !> att' \
+            -e '/^  integer function get_att_1d_id_int.*/i !> att' \
+            -e '/^  integer function get_att_1d_id_double.*/i !> att' \
+            -e '/^     integer(C_INT) function PIOc_get_att_double.*/i !> att' \
+            -e '/^  integer function pioc_get_att_float.*/i !> att' \
+            -e '/^  integer function pioc_get_att_int.*/i !> att' \
+            -e '/^  integer function pioc_get_att_text.*/i !> att' \
+            -e '/^  integer function pioc_put_att_double.*/i !> att' \
+            -e '/^  integer function pioc_put_att_float.*/i !> att' \
+            -e '/^  integer function pioc_put_att_int.*/i !> att' \
+            -e '/^  integer function pioc_put_att_text.*/i !> att' \
+            $< > $@
+pionfget_mod_2.f90: pionfget_mod.f90
+	sed -e '/^  integer function get_var1_id_real.*/i !> var' \
+            -e '/^  integer function get_var1_id_double.*/i !> var' \
+            -e '/^  integer function get_var1_id_text.*/i !> var' \
+            -e '/^  integer function get_var_0d_real.*/i !> var' \
+            -e '/^  integer function get_var_0d_double.*/i !> var' \
+            -e '/^  integer function get_var_1d_text.*/i !> var' \
+            -e '/^  integer function get_vara_real_internal.*/i !> var' \
+            -e '/^  integer function get_vara_double_internal.*/i !> var' \
+            -e '/^  integer function get_vara_text_internal.*/i !> var' \
+            -e '/^  integer function get_var_1d_int.*/i !> var' \
+            $< > $@
+endif
+
+CLEANFILES = *.mod piodarray.F90 pionfatt_mod.F90 pionfget_mod.F90	\
+pionfput_mod.F90 *.f90
+
+EXTRA_DIST = CMakeLists.txt piodarray.F90.in pionfatt_mod.F90.in	\
+pionfget_mod.F90.in pionfput_mod.F90.in
diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index cf8c96105b01..b4939e29060c 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -1,9 +1,13 @@
 !>
 !! @file 
-!! @brief User interface Module for PIO, this is the only file a user program should 'use'
-!! 
+!! User interface Module for PIO, this is the only file a user program should 'use'.
+!! @author Jim Edwards
 !<
 
+!>
+!! @defgroup PIO_set_blocksize Box Rearranger Settings
+!! Set the box rearranger blocksize in Fortran.
+
 module pio
 ! Package all exposed variables and functions under one roof
 
@@ -77,15 +81,11 @@ module pio
   implicit none
   public
 contains
-!>
-!! @public
-!! @defgroup PIO_set_blocksize
-!<
-!>
-!! @public
-!! @ingroup PIO_set_blocksize
-!! @brief Set the target blocksize for the box rearranger
-!<
+  !>
+  !! @ingroup PIO_set_blocksize
+  !! @brief Set the target blocksize for the box rearranger
+  !! @author Jim Edwards
+  !<
   subroutine pio_set_blocksize(blocksize)
     integer :: blocksize
     integer :: ierr
@@ -100,10 +100,10 @@ end function PIOc_set_blocksize
   end subroutine pio_set_blocksize
 
 
-!>
-!! @public
-!! @brief Logical function returns true if the task is an IO task.
-!<
+  !>
+  !! Logical function returns true if the task is an IO task.
+  !! @author Jim Edwards
+  !<
   function pio_iam_iotask(iosystem) result(task)
     use iso_c_binding
     type(iosystem_desc_t), intent(in) :: iosystem
@@ -123,10 +123,10 @@ end function PIOc_iam_iotask
     task = ctask
   end function pio_iam_iotask
   
-!>
-!! @public
-!! @brief Integer function returns rank of IO task.
-!<
+  !>
+  !! Integer function returns rank of IO task.
+  !! @author Jim Edwards
+  !<
   function pio_iotask_rank(iosystem) result(rank)
     type(iosystem_desc_t), intent(in) :: iosystem
     integer :: rank, ierr
@@ -142,10 +142,10 @@ end function PIOc_iotask_rank
     ierr = PIOc_iotask_rank(iosystem%iosysid, rank)
   end function pio_iotask_rank
 
-!>
-!! @public
-!! @brief Sets active to true if IO system is active.
-!<
+  !>
+  !! Sets active to true if IO system is active.
+  !! @author Jim Edwards
+  !<
   subroutine pio_iosystem_is_active(iosystem, active)
     use iso_c_binding
     type(iosystem_desc_t), intent(in) :: iosystem
@@ -165,6 +165,5 @@ end function PIOc_iosystem_is_active
     active = lactive
   end subroutine pio_iosystem_is_active
 
-
 end module pio
 
diff --git a/src/flib/pio_kinds.F90 b/src/flib/pio_kinds.F90
index 98006f4eada2..8e4148c23f29 100644
--- a/src/flib/pio_kinds.F90
+++ b/src/flib/pio_kinds.F90
@@ -1,22 +1,11 @@
 !>
-!! @file pio_kinds.F90
-!! @brief basic data types 
+!! @file
+!! This module defines default numerical data types for all common data
+!! types like integer, character, logical, real4 and real8.
 !!
 !<
  module pio_kinds
 
-!BOP
-! !MODULE: pio_kinds
-!
-! !DESCRIPTION:
-!  This module defines default numerical data types for all common data
-!  types like integer, character, logical, real4 and real8.
-!
-! !REVISION HISTORY:
-!  CVS:$Id: pio_kinds.F90,v 1.1.1.1 2006/07/31 16:15:30 dennis Exp $
-!  CVS:$Name:  $
-
-! !USES:
 !  uses mpi if available
 #ifndef NO_MPIMOD
    use mpi, only : MPI_OFFSET_KIND ! _EXTERNAL
@@ -30,25 +19,18 @@ module pio_kinds
 ! !DEFINED PARAMETERS:
 
    integer, parameter, public ::          &
-      char_len  = 360                    ,&
-      log_kind  = kind(.true.)           ,&
-      int_kind  = kind(1)                ,&
-      i4        = selected_int_kind(6)   ,&
-      i8        = selected_int_kind(13)  ,&
-      r4        = selected_real_kind(6)  ,&
-      r8        = selected_real_kind(13)
+      char_len  = 360                    ,& !< char len
+      log_kind  = kind(.true.)           ,& !< logical kind
+      int_kind  = kind(1)                ,& !< int kind
+      i4        = selected_int_kind(6)   ,& !< i4 kind
+      i8        = selected_int_kind(13)  ,& !< i8 kind
+      r4        = selected_real_kind(6)  ,& !< r4 kind
+      r8        = selected_real_kind(13)    !< r8 kind
 !
 !  MPI defines MPI_OFFSET_KIND as the byte size of the 
 !  type, which is not nessasarily the type kind
 !
-   
+!> Byte size of the MPI_OFFSET type.
    integer, parameter, public :: PIO_OFFSET_KIND=MPI_OFFSET_KIND
 
-!EOP
-!BOC
-!EOC
-!***********************************************************************
-
  end module pio_kinds
-
-!|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 1b19beec4b98..709cd9a97f55 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -1,3 +1,39 @@
+!>
+!! @file
+!! Code to implement the classic netCDF Fortran API in PIO.
+!! @author Jim Edwards
+!<
+
+!> @defgroup PIO_inquire_dimension Learn About Dimension
+!! Learn dimension name, ID, or length in Fortran.
+!!
+!! @defgroup PIO_inquire Learn About a File
+!! Learn the number of variables, dimensions, global attributes, and
+!! the unlimited dimension ID in Fortran.
+!!
+!! @defgroup PIO_enddef Define Mode
+!! End or re-enter define mode in Fortran.
+!!
+!! @defgroup PIO_set_log_level Debug Logging
+!! Set debugging log level in Fortran.
+!!
+!! @defgroup PIO_strerror Error Messages
+!! Get the error message from an error in Fortran.
+!!
+!! @defgroup PIO_def_dim Define a Dimension
+!! Define a new dimension, with name and length in Fortran.
+!!
+!! @defgroup PIO_inquire_variable Learn About a Variable
+!! Learn variable name, ID, type, dimensions, compression, chunking in
+!! Fortran.
+!!
+!! @defgroup PIO_inq_att Learn About an Attribute
+!! Learn attribute name, number, type, size in Fortran.
+!!
+!! @defgroup PIO_def_var Define a Variable
+!! Define a new variable in Fortran.
+!<
+
 module pio_nf
 #ifdef TIMING
   use perf_mod           , only : t_startf, t_stopf      ! _EXTERNAL
@@ -39,7 +75,7 @@ module pio_nf
        pio_redef                                            , &
        pio_set_log_level                                    , &
        pio_strerror
-!       pio_copy_att    to be done
+  !       pio_copy_att    to be done
 
   interface pio_def_var
      module procedure &
@@ -47,78 +83,78 @@ module pio_nf
           def_var_md_desc                                   , &
           def_var_0d_id                                     , &
           def_var_md_id
-  end interface
+  end interface pio_def_var
   interface pio_def_var_deflate
      module procedure &
           def_var_deflate_desc                              , &
           def_var_deflate_id
-  end interface
+  end interface pio_def_var_deflate
   interface pio_def_var_chunking
      module procedure &
           def_var_chunking
-  end interface
+  end interface pio_def_var_chunking
   interface pio_inq_attname
      module procedure &
           inq_attname_desc                                  , &
           inq_attname_vid                                   ,    &
           inq_attname_id
-  end interface
+  end interface pio_inq_attname
   interface pio_inq_att
      module procedure &
           inq_att_desc                                      , &
           inq_att_vid                                       ,    &
           inq_att_id
-  end interface
+  end interface pio_inq_att
   interface pio_inq_attlen
      module procedure &
           inq_attlen_desc                                   , &
           inq_attlen_vid                                    ,    &
           inq_attlen_id
-  end interface
+  end interface pio_inq_attlen
   interface pio_inq_varid
      module procedure &
           inq_varid_desc                                    , &
           inq_varid_vid                                     ,    &
           inq_varid_id
-  end interface
+  end interface pio_inq_varid
 
   interface pio_inq_varname
      module procedure &
           inq_varname_desc                                  , &
           inq_varname_vid                                   ,    &
           inq_varname_id
-  end interface
+  end interface pio_inq_varname
 
   interface pio_inq_vartype
      module procedure &
           inq_vartype_desc                                  , &
           inq_vartype_vid                                   ,    &
           inq_vartype_id
-  end interface
+  end interface pio_inq_vartype
   interface pio_inq_varndims
      module procedure &
           inq_varndims_desc                                 , &
           inq_varndims_vid                                  ,    &
           inq_varndims_id
-  end interface
+  end interface pio_inq_varndims
   interface pio_inq_vardimid
      module procedure &
           inq_vardimid_desc                                 , &
           inq_vardimid_vid                                  ,    &
           inq_vardimid_id
-  end interface
+  end interface pio_inq_vardimid
   interface pio_inq_varnatts
      module procedure &
           inq_varnatts_desc                                 , &
           inq_varnatts_vid                                  ,    &
           inq_varnatts_id
-  end interface
+  end interface pio_inq_varnatts
   interface pio_inq_var_deflate
      module procedure &
           inq_var_deflate_desc                                 , &
           inq_var_deflate_vid                                  , &
           inq_var_deflate_id
-  end interface
+  end interface pio_inq_var_deflate
   interface pio_inquire_dimension
      module procedure &
           inquire_dimension_desc                            , &
@@ -130,7 +166,7 @@ module pio_nf
           inquire_variable_desc                             , &
           inquire_variable_vid                              , &
           inquire_variable_id
-  end interface
+  end interface pio_inquire_variable
 
   interface pio_def_dim
      module procedure &
@@ -138,45 +174,45 @@ module pio_nf
           def_dim_id                                        , &
           def_dim_int_desc                                  , &
           def_dim_int_id
-  end interface
+  end interface pio_def_dim
   interface pio_inq_dimlen
      module procedure &
           inq_dimlen_desc                                   , &
           inq_dimlen_id                                     , &
           inq_dimlen_desc_long                              , &
           inq_dimlen_id_long
-  end interface
+  end interface pio_inq_dimlen
   interface pio_inq_ndims
      module procedure &
           inq_ndims_desc                                    , &
           inq_ndims_id
-  end interface
+  end interface pio_inq_ndims
   interface pio_inq_dimid
      module procedure &
           inq_dimid_desc                                    , &
           inq_dimid_id
-  end interface
+  end interface pio_inq_dimid
   interface pio_inq_dimname
      module procedure &
           inq_dimname_desc                                  , &
           inq_dimname_id
-  end interface
+  end interface pio_inq_dimname
 
   interface pio_inq_nvars
      module procedure &
           inq_nvars_desc                                    , &
           inq_nvars_id
-  end interface
+  end interface pio_inq_nvars
   interface pio_inq_natts
      module procedure &
           inq_natts_desc                                    , &
           inq_natts_id
-  end interface
+  end interface pio_inq_natts
   interface pio_inq_unlimdim
      module procedure &
           inq_unlimdim_desc                                 , &
           inq_unlimdim_id
-  end interface
+  end interface pio_inq_unlimdim
 
   interface pio_enddef
      module procedure &
@@ -188,7 +224,7 @@ module pio_nf
      module procedure &
           redef_desc                                        , &
           redef_id
-  end interface
+  end interface pio_redef
 
   interface pio_set_log_level
      module procedure &
@@ -204,58 +240,58 @@ module pio_nf
      module procedure &
           inquire_desc                                      , &
           inquire_id
-  end interface
+  end interface pio_inquire
 
   interface pio_set_chunk_cache
      module procedure &
           set_chunk_cache
-  end interface
+  end interface pio_set_chunk_cache
 
   interface pio_get_chunk_cache
      module procedure &
           get_chunk_cache
-  end interface
+  end interface pio_get_chunk_cache
 
   interface pio_set_var_chunk_cache
      module procedure &
           set_var_chunk_cache_desc                          , &
           set_var_chunk_cache_id
-  end interface
+  end interface pio_set_var_chunk_cache
 
   interface pio_get_var_chunk_cache
      module procedure &
           get_var_chunk_cache_desc                          , &
           get_var_chunk_cache_id
-  end interface
+  end interface pio_get_var_chunk_cache
 
 contains
 
-!>
-!! @defgroup PIO_inq_dimid PIO_inq_dimid
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_dimid
-!! @brief Returns the netcdf dimension id for the name.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param name : The name of the netcdf dimension.
-!! @param dimid : The netcdf dimension id.
-!! @retval ierr @copydoc error_return
-!!
-!! Note that we do not want internal error checking for this function.
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Returns the netcdf dimension id for the name.
+  !!
+  !! @note We do not want internal error checking for this function.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param name The name of the netcdf dimension.
+  !! @param dimid The netcdf dimension id.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_dimid_desc(File                      ,name,dimid) result(ierr)
     type (File_desc_t)                                      , intent(in) :: File
     character(len=*)                                        , intent(in)   :: name
     integer                                                 , intent(out)           :: dimid        !dimension ID
     ierr = inq_dimid_id(file%fh                             ,name,dimid)
   end function inq_dimid_desc
-!>
-!! @public
-!! @ingroup PIO_inq_dimid
-!! @brief Returns the netcdf dimension id for the name.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Returns the netcdf dimension id for the name.
+  !! @author Jim Edwards
+  !<
   integer function inq_dimid_id(ncid                        ,name,dimid) result(ierr)
     integer                                                 , intent(in) :: ncid
     character(len=*)                                        , intent(in)   :: name
@@ -273,20 +309,19 @@ end function PIOc_inq_dimid
     dimid=dimid+1
   end function inq_dimid_id
 
-!>
-!! @defgroup PIO_inquire_dimension PIO_inquire_dimension
-!<
-!>
-!! @public
-!! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file
-!! @details
-!! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
-!! @param dimid : The netcdf dimension ID.
-!! @param name : The name of the dimension.
-!! @param len : The length of the dimesions name.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about a particular dimension in netcdf file.
+  !!
+  !! @param file A netcdf file descriptor returned by \ref
+  !! PIO_openfile or \ref PIO_createfile.
+  !! @param dimid The netcdf dimension ID.
+  !! @param name The name of the dimension.
+  !! @param len The length of the dimesions name.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inquire_dimension_desc(file              , dimid, name, len) result(ierr)
     type(file_desc_T)                                       ,             intent(in)  :: file
     integer                                                 ,                       intent( in) :: dimid
@@ -294,11 +329,20 @@ integer function inquire_dimension_desc(file              , dimid, name, len) re
     integer                                                 ,             optional, intent(out) :: len
     ierr = Inquire_dimension_id(file%fh                     , dimid, name, len)
   end function inquire_dimension_desc
-!>
-!! @public
-!! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about a particular dimension in netcdf file.
+  !!
+  !! @param ncid A netcdf file ID returned by \ref
+  !! PIO_openfile or \ref PIO_createfile.
+  !! @param dimid The netcdf dimension ID.
+  !! @param name The name of the dimension.
+  !! @param len The length of the dimesions name.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inquire_dimension_id(ncid                , dimid, name, len) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 ,                       intent( in) :: dimid
@@ -313,42 +357,46 @@ integer function inquire_dimension_id(ncid                , dimid, name, len) re
 
   end function inquire_dimension_id
 
-
-!>
-!! @defgroup PIO_inq_dimlen PIO_inq_dimlen
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param dimid : The netcdf dimension ID.
-!! @param len : The length of the dimesion.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the length of a particular dimension in
+  !! netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param dimid The netcdf dimension ID.
+  !! @param len The length of the dimesion.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_dimlen_desc(File                     , dimid, len) result(ierr)
     type(file_desc_t)                                       , intent(in) :: File
     integer                                                 , intent(in) :: dimid
     integer                                                 , intent(out) :: len
     ierr = inq_dimlen_id(file%fh                            ,dimid,len)
   end function inq_dimlen_desc
-!>
-!! @public
-!! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the length of a particular dimension in
+  !! netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_dimlen_desc_long(File                , dimid, len) result(ierr)
     type(file_desc_t)                                       , intent(in) :: File
     integer                                                 , intent(in) :: dimid
     integer(PIO_OFFSET_KIND)                                , intent(out) :: len
     ierr = inq_dimlen_id_long(file%fh                       ,dimid,len)
   end function inq_dimlen_desc_long
-!>
-!! @public
-!! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the length of a particular dimension in
+  !! netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_dimlen_id(ncid                       , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: dimid
@@ -357,11 +405,14 @@ integer function inq_dimlen_id(ncid                       , dimid, len) result(i
     ierr = inq_dimlen_id_long(ncid                          ,dimid,llen)
     len = int(llen)
   end function inq_dimlen_id
-!>
-!! @public
-!! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the length of a particular dimension in
+  !! netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_dimlen_id_long(ncid                  , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: dimid
@@ -379,31 +430,30 @@ end function PIOc_inq_dimlen
     ierr = PIOc_inq_dimlen(ncid                             ,dimid-1,len)
   end function inq_dimlen_id_long
 
-
-!>
-!! @defgroup PIO_inq_dimname PIO_inq_dimname
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param dimid : The netcdf dimension ID.
-!! @param len : The length of the dimesion.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the name of of a dimension.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param dimid The netcdf dimension ID.
+  !! @param name The name of the dimesion.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_dimname_desc(File                    , dimid, name) result(ierr)
     type(file_desc_t)                                       , intent(in) :: File
     integer                                                 , intent(in) :: dimid
     character(len=*)                                        , intent(out) :: name
     ierr = inq_dimname_id(file%fh                           ,dimid,name)
   end function inq_dimname_desc
-!>
-!! @public
-!! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_dimension
+  !! Get information about the name of of a dimension.
+  !! @author Jim Edwards
+  !<
   integer function inq_dimname_id(ncid                      , dimid, name) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: dimid
@@ -424,29 +474,30 @@ end function PIOc_inq_dimname
 
   end function inq_dimname_id
 
-
-!>
-!! @defgroup PIO_inq_ndims PIO_inq_ndims
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_ndims
-!! @brief  Get information about the number of dimensions of a file or group.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param ndims : The number of dimensions in the file.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of dimensions of a file or
+  !! group.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param ndims The number of dimensions in the file.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_ndims_desc(File                      , ndims) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     integer                                                 , intent(out) :: ndims
     ierr = inq_ndims_id(file%fh                             , ndims)
   end function inq_ndims_desc
-!>
-!! @public
-!! @ingroup PIO_inq_ndims
-!! @brief  Get information about the number of dimensions of a file or group.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of dimensions of a file or
+  !! group.
+  !! @author Jim Edwards
+  !<
   integer function inq_ndims_id(ncid                        , ndims) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(out) :: ndims
@@ -461,28 +512,28 @@ end function PIOc_inq_ndims
     ierr = PIOc_inq_ndims(ncid                              ,ndims)
   end function inq_ndims_id
 
-!>
-!! @defgroup PIO_inq_nvars PIO_inq_nvars
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_nvars
-!! @brief  Get information about the number of variables in a file or group.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param nvars : The number of variables in the file.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of variables in a file or group.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param nvars The number of variables in the file.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_nvars_desc(File                      , nvars) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     integer                                                 , intent(out) :: nvars
     ierr = inq_nvars_id(file%fh                             , nvars)
   end function inq_nvars_desc
-!>
-!! @public
-!! @ingroup PIO_inq_nvars
-!! @brief  Get information about the number of variables in a file or group.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of variables in a file or group.
+  !! @author Jim Edwards
+  !<
   integer function inq_nvars_id(ncid                        , nvars) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(out) :: nvars
@@ -497,28 +548,30 @@ end function PIOc_inq_nvars
     ierr = PIOc_inq_nvars(ncid                              ,nvars)
   end function inq_nvars_id
 
-!>
-!! @defgroup PIO_inq_natts PIO_inq_natts
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_natts
-!! @brief  Get information about the number of global attributes in a file or group.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param natts : The number of attributes in the file.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of global attributes in a file
+  !! or group.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param natts The number of attributes in the file.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_natts_desc(File                      , natts) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     integer                                                 , intent(out) :: natts
     ierr = inq_natts_id(file%fh                             , natts)
   end function inq_natts_desc
-!>
-!! @public
-!! @ingroup PIO_inq_natts
-!! @brief  Get information about the number of global attributes in a file or group.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the number of global attributes in a file
+  !! or group.
+  !! @author Jim Edwards
+  !<
   integer function inq_natts_id(ncid                        , natts) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(out) :: natts
@@ -533,28 +586,29 @@ end function PIOc_inq_natts
     ierr = PIOc_inq_natts(ncid                              ,natts)
   end function inq_natts_id
 
-!>
-!! @defgroup PIO_inq_unlimdim PIO_inq_unlimdim
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_unlimdm
-!! @brief  Get information about the unlimited dimension in a file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param unlimdim : Pointer to the unlimted dimension. If no unlimited dimension, this will be -1.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the unlimited dimension in a file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param unlimdim Pointer to the unlimted dimension. If no
+  !! unlimited dimension, this will be -1.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_unlimdim_desc(File                   , unlimdim) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     integer                                                 , intent(out) :: unlimdim
     ierr = inq_unlimdim_id(file%fh                          , unlimdim)
   end function inq_unlimdim_desc
-!>
-!! @public
-!! @ingroup PIO_inq_unlimdm
-!! @brief  Get information about the unlimited dimension in a file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Get information about the unlimited dimension in a file.
+  !! @author Jim Edwards
+  !<
   integer function inq_unlimdim_id(ncid                     ,unlimdim) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(out) :: unlimdim
@@ -570,21 +624,21 @@ end function PIOc_inq_unlimdim
     if(unlimdim>=0) unlimdim=unlimdim+1
   end function inq_unlimdim_id
 
-!>
-!! @defgroup PIO_inquire PIO_inquire
-!<
-!>
-!! @public
-!! @ingroup PIO_inquire
-!! @brief Gets metadata information for netcdf file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param nDimensions :  Number of dimensions defined for the netcdf file
-!! @param nVariables : Number of variables defined for the netcdf file
-!! @param nAttributes : Number of attributes defined for the netcdf file
-!! @param unlimitedDimID : the Unlimited dimension ID
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param nDimensions Number of dimensions defined for the netcdf
+  !! file
+  !! @param nVariables Number of variables defined for the netcdf file
+  !! @param nAttributes Number of attributes defined for the netcdf
+  !! file
+  !! @param unlimitedDimID the Unlimited dimension ID
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inquire_desc(File                        ,nDimensions,nVariables,nAttributes,unlimitedDimID) result(ierr)
     type (File_desc_t)                                      , intent(in) :: File
 
@@ -596,11 +650,13 @@ integer function inquire_desc(File                        ,nDimensions,nVariable
 
     ierr = inquire_id(file%fh                               ,ndimensions,nvariables,nattributes,unlimitedDimID)
   end function inquire_desc
-!>
-!! @public
-!! @ingroup PIO_inquire
-!! @brief Gets metadata information for netcdf file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inquire_id(ncid                          ,nDimensions,nVariables,nAttributes,unlimitedDimID) result(ierr)
     integer                                                 ,intent(in) :: ncid
     integer                                                 , optional, intent(out) :: &
@@ -615,26 +671,26 @@ integer function inquire_id(ncid                          ,nDimensions,nVariable
     if(present(unlimitedDimID)) ierr = inq_unlimdim_id(ncid ,unlimitedDimID)
   end function inquire_id
 
-!>
-!! @defgroup PIO_enddef PIO_enddef
-!<
-!>
-!! @public
-!! @ingroup PIO_enddef
-!! @brief Exits netcdf define mode.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_enddef
+  !! Exits netcdf define mode.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function enddef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = enddef_id(file%fh)
   end function enddef_desc
-!>
-!! @public
-!! @ingroup PIO_enddef
-!! @brief Wrapper for the C function \ref PIOc_enddef .
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_enddef
+  !! Wrapper for the C function \ref PIOc_enddef .
+  !! @author Jim Edwards
+  !<
   integer function enddef_id(ncid) result(ierr)
     integer                                                 ,intent(in) :: ncid
     interface
@@ -646,33 +702,31 @@ end function PIOc_enddef
     end interface
     ierr = PIOc_enddef(ncid)
   end function enddef_id
-!>
-!! @defgroup PIO_redef PIO_redef
-!<
-!>
-!! @public
-!! @ingroup PIO_redef
-!! @brief Exits netcdf define mode.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @retval ierr @copydoc error_return
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_enddef
+  !! Exits netcdf define mode.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function redef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = redef_id(file%fh)
   end function redef_desc
 
-!>
-!! @defgroup PIO_set_log_level
-!<
-!>
-!! @ingroup PIO_set_log_level
-!! Sets the logging level. Only takes effect if PIO was built with
-!! PIO_ENABLE_LOGGING=On
-!!
-!! @param log_level the logging level.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_set_log_level
+  !! Sets the logging level. Only takes effect if PIO was built with
+  !! PIO_ENABLE_LOGGING=On
+  !!
+  !! @param log_level the logging level.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function set_log_level(log_level) result(ierr)
     integer, intent(in) :: log_level
     interface
@@ -686,14 +740,14 @@ end function PIOc_set_log_level
   end function set_log_level
 
   !>
-  !! @defgroup PIO_strerror
-  !<
-  !>
+  !! @public
   !! @ingroup PIO_strerror
   !! Returns a descriptive string for an error code.
   !!
-  !! @param errcode the error code
-  !! @retval a description of the error
+  !! @param errcode the error code.
+  !! @param errmsg the error message.
+  !! @retval 0 for success, error code otherwise.
+  !! @author Jim Edwards
   !<
   integer function strerror(errcode, errmsg) result(ierr)
     integer, intent(in) :: errcode
@@ -712,11 +766,12 @@ end function PIOc_strerror
 
   end function strerror
 
-!>
-!! @public
-!! @ingroup PIO_redef
-!! @brief Wrapper for the C function \ref PIOc_redef .
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_enddef
+  !! Wrapper for the C function \ref PIOc_redef .
+  !! @author Jim Edwards
+  !<
   integer function redef_id(ncid) result(ierr)
     integer, intent(in) :: ncid
     interface
@@ -729,20 +784,16 @@ end function PIOc_redef
     ierr = PIOc_redef(ncid)
   end function redef_id
 
-!>
-!! @defgroup PIO_def_dim PIO_def_dim
-!! @brief A set of functions to define dimensions and their attributes in NetCDF files.
-!<
-!>
-!! @public
-!! @ingroup PIO_def_dim
-!! @brief Defines the netcdf dimension.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param name : The name of the dimension to define
-!! @param len :  The size of the dimension
-!! @param dimid : The dimension identifier
-!<
+  !>
+  !! @ingroup PIO_def_dim
+  !! Defines the netcdf dimension.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param name The name of the dimension to define
+  !! @param len  The size of the dimension
+  !! @param dimid The dimension identifier
+  !! @author Jim Edwards
+  !<
   integer function def_dim_int_desc(File                    ,name,len,dimid) result(ierr)
 
     type (File_desc_t)                                      , intent(in)  :: File
@@ -752,11 +803,12 @@ integer function def_dim_int_desc(File                    ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_desc
-!>
-!! @public
-!! @ingroup PIO_def_dim
-!! @brief  Defines the netcdf dimension.
-!<
+
+  !>
+  !! @ingroup PIO_def_dim
+  !! Defines the netcdf dimension.
+  !! @author Jim Edwards
+  !<
   integer function def_dim_int_id(ncid                      ,name,len,dimid) result(ierr)
     integer                                                 , intent(in) :: ncid
     character(len=*)                                        , intent(in)    :: name
@@ -765,11 +817,12 @@ integer function def_dim_int_id(ncid                      ,name,len,dimid) resul
 
     ierr = def_dim_id(ncid                                  ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_id
-!>
-!! @public
-!! @ingroup PIO_def_dim
-!! @brief  Defines the netcdf dimension.
-!<
+
+  !>
+  !! @ingroup PIO_def_dim
+  !! Defines the netcdf dimension.
+  !! @author Jim Edwards
+  !<
   integer function def_dim_desc(File                        ,name,len,dimid) result(ierr)
 
     type (File_desc_t)                                      , intent(in)  :: File
@@ -779,11 +832,12 @@ integer function def_dim_desc(File                        ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,len,dimid)
   end function def_dim_desc
-!>
-!! @public
-!! @ingroup PIO_def_dim
-!! @brief  Defines the netcdf dimension.
-!<
+
+  !>
+  !! @ingroup PIO_def_dim
+  !! Defines the netcdf dimension.
+  !! @author Jim Edwards
+  !<
   integer function def_dim_id(ncid                          ,name,len,dimid) result(ierr)
     integer                                                 , intent(in)         :: ncid
     character(len=*)                                        , intent(in)    :: name
@@ -804,24 +858,24 @@ end function PIOc_def_dim
     dimid=dimid+1
   end function def_dim_id
 
-
-!>
-!! @defgroup PIO_inquire_variable PIO_inquire_variable
-!<
-!>
-!! @public
-!! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes
-!! @details
-!! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
-!! @param vardesc @copydoc var_desc_t
-!! @param name : The name of the variable
-!! @param xtype : The type of the variable
-!! @param ndims : The number of dimensions for the variable.
-!! @param dimids : The dimension identifier returned by \ref PIO_def_dim
-!! @param natts : Number of attributes associated with the variable
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Inquires if a NetCDF variable is present and returns its
+  !! attributes.
+  !!
+  !! @param file A netcdf file descriptor returned by \ref
+  !! PIO_openfile or \ref PIO_createfile.
+  !! @param vardesc @copydoc var_desc_t
+  !! @param name The name of the variable
+  !! @param xtype The type of the variable
+  !! @param ndims The number of dimensions for the variable.
+  !! @param dimids The dimension identifier returned by \ref
+  !! PIO_def_dim
+  !! @param natts Number of attributes associated with the variable
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inquire_variable_desc(file               , vardesc, name, xtype, ndims, dimids, natts) result(ierr)
     type(file_desc_t)                                       ,               intent(in) :: file
     type(var_desc_t)                                        ,                intent( in) :: vardesc
@@ -832,11 +886,14 @@ integer function inquire_variable_desc(file               , vardesc, name, xtype
 
     ierr = pio_inquire_variable(file%fh                     ,vardesc%varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_desc
-!>
-!! @public
-!! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Inquires if a NetCDF variable is present and returns its
+  !! attributes.
+  !! @author Jim Edwards
+  !<
   integer function inquire_variable_vid(file                , varid, name, xtype, ndims, dimids, natts) result(ierr)
     type(file_desc_t)                                       ,               intent(in) :: file
     integer                                                 ,                intent( in) :: varid
@@ -847,11 +904,14 @@ integer function inquire_variable_vid(file                , varid, name, xtype,
 
     ierr = pio_inquire_variable(file%fh                     ,varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_vid
-!>
-!! @public
-!! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Inquires if a NetCDF variable is present and returns its
+  !! attributes.
+  !! @author Jim Edwards
+  !<
   integer function inquire_variable_id(ncid                 , varid, name, xtype, ndims, dimids, natts) result(ierr)
     integer                                                 ,                intent( in) :: ncid
     integer                                                 ,                intent( in) :: varid
@@ -867,19 +927,18 @@ integer function inquire_variable_id(ncid                 , varid, name, xtype,
     if(present(xtype)) ierr = pio_inq_vartype(ncid          , varid, xtype)
   end function inquire_variable_id
 
-!>
-!! @defgroup PIO_inq_vardimid PIO_inq_vardimid
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_vardimid
-!! @brief returns the dimids of the variable as an interger array
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param dimids : The dimension identifier returned by \ref PIO_def_dim
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the dimids of the variable as an interger array.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param dimids The dimension identifier returned by \ref
+  !! PIO_def_dim
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_vardimid_desc(File                   ,vardesc,dimids) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -889,11 +948,13 @@ integer function inq_vardimid_desc(File                   ,vardesc,dimids) resul
 
     ierr = pio_inq_vardimid(File%fh                         , vardesc%varid, dimids)
   end function inq_vardimid_desc
-!>
-!! @public
-!! @ingroup PIO_inq_vardimid
-!! @brief returns the dimids of the variable as an interger array
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the dimids of the variable as an interger array.
+  !! @author Jim Edwards
+  !<
   integer function inq_vardimid_vid(File                    ,varid,dimids) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -903,11 +964,13 @@ integer function inq_vardimid_vid(File                    ,varid,dimids) result(
 
     ierr = pio_inq_vardimid(File%fh                         , varid, dimids)
   end function inq_vardimid_vid
-!>
-!! @public
-!! @ingroup PIO_inq_vardimid
-!! @brief returns the dimids of the variable as an interger array
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the dimids of the variable as an interger array.
+  !! @author Jim Edwards
+  !<
   integer function inq_vardimid_id(ncid                     ,varid,dimids) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -935,19 +998,17 @@ end function PIOc_inq_vardimid
 
   end function inq_vardimid_id
 
-!>
-!! @defgroup PIO_inq_varndims PIO_inq_varndims
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_varndims
-!! @brief Gets the number of dimension associated with a netcdf variable
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param ndims : The number of dimensions for the variable
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets the number of dimension associated with a netcdf variable.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param ndims The number of dimensions for the variable
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_varndims_desc(File                   ,vardesc,ndims) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -956,11 +1017,13 @@ integer function inq_varndims_desc(File                   ,vardesc,ndims) result
 
     ierr = pio_inq_varndims(File%fh                         , vardesc%varid, ndims)
   end function inq_varndims_desc
-!>
-!! @public
-!! @ingroup PIO_inq_varndims
-!! @brief Gets the number of dimension associated with a netcdf variable
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets the number of dimension associated with a netcdf variable.
+  !! @author Jim Edwards
+  !<
   integer function inq_varndims_vid(File                    ,varid,ndims) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -969,11 +1032,13 @@ integer function inq_varndims_vid(File                    ,varid,ndims) result(i
 
     ierr = pio_inq_varndims(File%fh                         , varid, ndims)
   end function inq_varndims_vid
-!>
-!! @public
-!! @ingroup PIO_inq_varndims
-!! @brief Gets the number of dimension associated with a netcdf variable
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets the number of dimension associated with a netcdf variable.
+  !! @author Jim Edwards
+  !<
   integer function inq_varndims_id(ncid                     ,varid,ndims) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -990,19 +1055,17 @@ end function PIOc_inq_varndims
     ierr = PIOc_inq_varndims(ncid                           ,varid-1,ndims)
   end function inq_varndims_id
 
-!>
-!! @defgroup PIO_inq_vartype PIO_inq_vartype
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_vartype
-!! @brief Gets metadata information for netcdf file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param type : The type of variable
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param type The type of variable
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_vartype_desc(File                    ,vardesc,type) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1011,11 +1074,13 @@ integer function inq_vartype_desc(File                    ,vardesc,type) result(
 
     ierr = pio_inq_vartype(File%fh                          , vardesc%varid, type)
   end function inq_vartype_desc
-!>
-!! @public
-!! @ingroup PIO_inq_vartype
-!! @brief Gets metadata information for netcdf file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_vartype_vid(File                     ,varid,type) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1024,11 +1089,13 @@ integer function inq_vartype_vid(File                     ,varid,type) result(ie
 
     ierr = pio_inq_vartype(File%fh                          , varid, type)
   end function inq_vartype_vid
-!>
-!! @public
-!! @ingroup PIO_inq_vartype
-!! @brief Gets metadata information for netcdf file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_vartype_id(ncid                      ,varid,type) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -1047,19 +1114,17 @@ end function PIOc_inq_vartype
     ierr = PIOc_inq_vartype(ncid                            ,varid-1,type)
   end function inq_vartype_id
 
-!>
-!!  @defgroup PIO_inq_varnatts PIO_inq_varnatts
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_varnatts
-!! @brief Gets metadata information for netcdf file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param type : The type of variable
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param natts The number of atts
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_varnatts_desc(File                   ,vardesc,natts) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1068,11 +1133,13 @@ integer function inq_varnatts_desc(File                   ,vardesc,natts) result
 
     ierr = pio_inq_varnatts(File%fh                         , vardesc%varid,natts)
   end function inq_varnatts_desc
-!>
-!! @public
-!! @ingroup PIO_inq_varnatts
-!! @brief Gets metadata information for netcdf file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_varnatts_vid(File                    ,varid,natts) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1081,11 +1148,13 @@ integer function inq_varnatts_vid(File                    ,varid,natts) result(i
 
     ierr = pio_inq_varnatts(File%fh                         , varid, natts)
   end function inq_varnatts_vid
-!>
-!! @public
-!! @ingroup PIO_inq_varnatts
-!! @brief Gets metadata information for netcdf file.
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_varnatts_id(ncid                     ,varid,natts) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -1104,19 +1173,19 @@ end function PIOc_inq_varnatts
     ierr = PIOc_inq_varnatts(ncid                           ,varid-1,natts)
   end function inq_varnatts_id
 
-!>
-!!  @defgroup PIO_inq_var_deflate PIO_inq_var_deflate
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_var_deflate
-!! @brief Gets metadata information for netcdf file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param type : The type of variable
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param shuffle Value of shuffle
+  !! @param deflate Status of deflate
+  !! @param deflate_level Level of deflate
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_var_deflate_desc(File, vardesc, shuffle, deflate, &
        deflate_level) result(ierr)
 
@@ -1129,11 +1198,12 @@ integer function inq_var_deflate_desc(File, vardesc, shuffle, deflate, &
     ierr = pio_inq_var_deflate(File%fh, vardesc%varid, shuffle, deflate, deflate_level)
   end function inq_var_deflate_desc
 
-!>
-!! @public
-!! @ingroup PIO_inq_var_deflate
-!! @brief Gets metadata information for netcdf file.
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_level) result(ierr)
 
     type (File_desc_t), intent(in) :: File
@@ -1145,11 +1215,12 @@ integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_leve
     ierr = pio_inq_var_deflate(File%fh, varid, shuffle, deflate, deflate_level)
   end function inq_var_deflate_vid
 
-!>
-!! @public
-!! @ingroup PIO_inq_var_deflate
-!! @brief Gets metadata information for netcdf file.
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Jim Edwards
+  !<
   integer function inq_var_deflate_id(ncid, varid, shuffle, deflate, &
        deflate_level) result(ierr)
     integer, intent(in) :: ncid
@@ -1173,19 +1244,17 @@ end function PIOc_inq_var_deflate
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
 
-!>
-!! @defgroup PIO_inq_varname
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_varname
-!! @brief Get the name associated with a variable
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param name : The name of the netcdf variable.
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Get the name associated with a variable.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param name The name of the netcdf variable.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_varname_desc(File                    ,vardesc,name) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1195,11 +1264,13 @@ integer function inq_varname_desc(File                    ,vardesc,name) result(
     ierr = pio_inq_varname(file%fh                          ,vardesc%varid,name)
 
   end function inq_varname_desc
-!>
-!! @public
-!! @ingroup PIO_inq_varname
-!! @brief Get the name associated with a variable
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Get the name associated with a variable.
+  !! @author Jim Edwards
+  !<
   integer function inq_varname_vid(File                     ,varid,name) result(ierr)
 
     type (File_desc_t)                                      , intent(in)   :: File
@@ -1209,11 +1280,13 @@ integer function inq_varname_vid(File                     ,varid,name) result(ie
     ierr = pio_inq_varname(file%fh                          ,varid,name)
 
   end function inq_varname_vid
-!>
-!! @public
-!! @ingroup PIO_inq_varname
-!! @brief Get the name associated with a variable
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Get the name associated with a variable.
+  !! @author Jim Edwards
+  !<
   integer function inq_varname_id(ncid                      ,varid,name) result(ierr)
     integer                                                 ,intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -1233,20 +1306,17 @@ end function PIOc_inq_varname
 
   end function inq_varname_id
 
-
-!>
-!! @defgroup PIO_inq_varid
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param name   : Name of the returned attribute
-!! @param vardesc @copydoc var_desc_t
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the ID of a netcdf variable given its name.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param name   Name of the returned attribute
+  !! @param vardesc @copydoc var_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_varid_desc(File,name,vardesc) result(ierr)
 
     type (File_desc_t), intent(in)   :: File
@@ -1255,11 +1325,13 @@ integer function inq_varid_desc(File,name,vardesc) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, vardesc%varid)
   end function inq_varid_desc
-!>
-!! @public
-!! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the ID of a netcdf variable given its name.
+  !! @author Jim Edwards
+  !<
   integer function inq_varid_vid(File,name,varid) result(ierr)
 
     type (File_desc_t), intent(in)   :: File
@@ -1268,11 +1340,13 @@ integer function inq_varid_vid(File,name,varid) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, varid)
   end function inq_varid_vid
-!>
-!! @public
-!! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Returns the ID of a netcdf variable given its name.
+  !! @author Jim Edwards
+  !<
   integer function inq_varid_id(ncid,name,varid) result(ierr)
 
     integer, intent(in)   :: ncid
@@ -1294,20 +1368,18 @@ end function PIOc_inq_varid
     varid = varid+1
   end function inq_varid_id
 
-!>
-!! @defgroup PIO_inq_attlen
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param name : name of attribute
-!! @param len : Length of attribute
-!! @retval ierr @copydoc error_return
-!>
+  !>
+  !! @public
+  !! @ingroup PIO_inq_att
+  !! Gets the attribute length.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param name name of attribute
+  !! @param len Length of attribute
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !>
   integer function inq_attlen_desc(File,vardesc,name,len) result(ierr)
 
     type (File_desc_t), intent(inout) :: File
@@ -1318,11 +1390,12 @@ integer function inq_attlen_desc(File,vardesc,name,len) result(ierr)
     ierr = pio_inq_attlen(file%fh, vardesc%varid, name, len)
 
   end function inq_attlen_desc
-!>
-!! @public
-!! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets the attribute length.
+  !! @author Jim Edwards
+  !<
   integer function inq_attlen_vid(File,varid,name,len) result(ierr)
 
     type (File_desc_t), intent(inout) :: File
@@ -1333,11 +1406,11 @@ integer function inq_attlen_vid(File,varid,name,len) result(ierr)
     ierr = pio_inq_attlen(file%fh, varid, name, len)
 
   end function inq_attlen_vid
-!>
-!! @public
-!! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets the attribute length.
+  !<
   integer function inq_attlen_id(ncid,varid,name,len) result(ierr)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -1357,22 +1430,18 @@ end function PIOc_inq_attlen
     ierr = PIOc_inq_attlen(ncid,varid-1,trim(name)//C_NULL_CHAR,len)
   end function inq_attlen_id
 
-
-!>
-!! @defgroup PIO_inq_att PIO_inq_att
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_att
-!! @brief  Gets information about attributes
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param name : Name of the attribute
-!! @param xtype : The type of attribute
-!! @param len : The length of the attribute
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets information about attributes.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param name Name of the attribute
+  !! @param xtype The type of attribute
+  !! @param len The length of the attribute
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
 
     type (File_desc_t), intent(inout) :: File
@@ -1384,11 +1453,12 @@ integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
     ierr = pio_inq_att(file%fh, vardesc%varid, name, xtype, len)
 
   end function inq_att_desc
-!>
-!! @public
-!! @ingroup PIO_inq_att
-!! @brief  Gets information about attributes
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets information about attributes.
+  !! @author Jim Edwards
+  !<
   integer function inq_att_vid(File,varid,name,xtype,len) result(ierr)
 
     type (File_desc_t), intent(in) :: File
@@ -1400,11 +1470,12 @@ integer function inq_att_vid(File,varid,name,xtype,len) result(ierr)
     ierr = pio_inq_att(file%fh, varid, name, xtype, len)
 
   end function inq_att_vid
-!>
-!! @public
-!! @ingroup PIO_inq_att
-!! @brief  Gets information about attributes
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets information about attributes.
+  !! @author Jim Edwards
+  !<
   integer function inq_att_id(ncid,varid,name,xtype,len) result(ierr)
 
     integer, intent(in) :: ncid
@@ -1433,14 +1504,12 @@ end function PIOc_inq_att
     if(present(xtype)) xtype = ixtype
 
   end function inq_att_id
-!>
-!! @defgroup PIO_inq_attname
-!<
-!>
-!! @public
-!! @ingroup PIO_inq_attname
-!! @brief  Gets the name of an attribute
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets the name of an attribute.
+  !! @author Jim Edwards
+  !<
   integer function inq_attname_desc(File,vdesc,attnum,name) result(ierr)
     type (File_desc_t), intent(inout) :: File
     type (var_desc_t), intent(in)           :: vdesc
@@ -1450,11 +1519,12 @@ integer function inq_attname_desc(File,vdesc,attnum,name) result(ierr)
     ierr = inq_attname_id(file%fh,vdesc%varid,attnum,name)
 
   end function inq_attname_desc
-!>
-!! @public
-!! @ingroup PIO_inq_attname
-!! @brief   Gets the name of an attribute
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets the name of an attribute.
+  !! @author Jim Edwards
+  !<
   integer function inq_attname_vid(File,varid,attnum,name) result(ierr)
     type (File_desc_t), intent(inout) :: File
     integer, intent(in)           :: varid
@@ -1464,11 +1534,12 @@ integer function inq_attname_vid(File,varid,attnum,name) result(ierr)
     ierr = inq_attname_id(file%fh,varid,attnum,name)
 
   end function inq_attname_vid
-!>
-!! @public
-!! @ingroup PIO_inq_attname
-!! @brief   Gets the name of an attribute
-!<
+
+  !>
+  !! @ingroup PIO_inq_att
+  !! Gets the name of an attribute.
+  !! @author Jim Edwards
+  !<
   integer function inq_attname_id(ncid,varid,attnum,name) result(ierr)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -1491,22 +1562,17 @@ end function PIOc_inq_attname
 
   end function inq_attname_id
 
-
-!>
-!! @defgroup PIO_def_var PIO_def_var
-!<
-
-!>
-!! @public
-!! @ingroup PIO_def_var
-!! @brief Defines a netcdf variable
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param name : The name of the variable to define
-!! @param type : The type of variable
-!! @param vardesc @copydoc var_desc_t
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @ingroup PIO_def_var
+  !! Defines a netcdf variable.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param name The name of the variable to define
+  !! @param type The type of variable
+  !! @param vardesc @copydoc var_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function def_var_0d_desc(File,name,type,vardesc) result(ierr)
 
     type (File_desc_t), intent(in)  :: File
@@ -1518,11 +1584,12 @@ integer function def_var_0d_desc(File,name,type,vardesc) result(ierr)
     ierr = def_var_md_id(File%fh,name,type,dimids,vardesc%varid)
 
   end function def_var_0d_desc
-!>
-!! @public
-!! @ingroup PIO_def_var
-!! @brief Defines a netcdf variable
-!<
+
+  !>
+  !! @ingroup PIO_def_var
+  !! Defines a netcdf variable.
+  !! @author Jim Edwards
+  !<
   integer function def_var_0d_id(ncid,name,type,varid) result(ierr)
 
     integer,intent(in) :: ncid
@@ -1535,18 +1602,18 @@ integer function def_var_0d_id(ncid,name,type,varid) result(ierr)
 
   end function def_var_0d_id
 
-!>
-!! @public
-!! @ingroup PIO_def_var
-!! @brief Defines the a netcdf variable
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param name : The name of the variable to define
-!! @param type : The type of variable
-!! @param dimids : The dimension identifier returned by \ref PIO_def_dim
-!! @param vardesc @copydoc var_desc_t
-!! @retval ierr @copydoc error_return
-!<
+  !>
+  !! @ingroup PIO_def_var
+  !! Defines the a netcdf variable.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param name The name of the variable to define
+  !! @param type The type of variable
+  !! @param dimids The dimension identifier returned by \ref PIO_def_dim
+  !! @param vardesc @copydoc var_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function def_var_md_desc(File,name,type,dimids,vardesc) result(ierr)
     type (File_desc_t), intent(in)  :: File
     character(len=*), intent(in)    :: name
@@ -1556,11 +1623,12 @@ integer function def_var_md_desc(File,name,type,dimids,vardesc) result(ierr)
 
     ierr = def_var_md_id(file%fh,name,type,dimids,vardesc%varid)
   end function def_var_md_desc
-!>
-!! @public
-!! @ingroup PIO_def_var
-!! @brief Defines a netcdf variable
-!<
+
+  !>
+  !! @ingroup PIO_def_var
+  !! Defines a netcdf variable.
+  !! @author Jim Edwards
+  !<
   integer function def_var_md_id(ncid,name,type,dimids,varid) result(ierr)
     integer,intent(in) :: ncid
     character(len=*), intent(in)    :: name
@@ -1590,11 +1658,11 @@ end function PIOc_def_var
     varid = varid+1
   end function def_var_md_id
 
-!>
-!! @public
-!! @ingroup PIO_def_var_deflate
-!! @brief Changes compression settings for a netCDF-4/HDF5 variable.
-!<
+  !>
+  !! @ingroup PIO_def_var_deflate
+  !! Changes compression settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
   integer function def_var_deflate_id(file, varid, shuffle, deflate, deflate_level) &
        result(ierr)
     type (File_desc_t), intent(in)  :: file
@@ -1618,11 +1686,11 @@ end function PIOc_def_var_deflate
     ierr = PIOc_def_var_deflate(file%fh, varid-1, shuffle, deflate, deflate_level)
   end function def_var_deflate_id
 
-!>
-!! @public
-!! @ingroup PIO_def_var_deflate
-!! @brief Changes compression settings for a netCDF-4/HDF5 variable.
-!<
+  !>
+  !! @ingroup PIO_def_var_deflate
+  !! Changes compression settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
   integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_level) &
        result(ierr)
     type (File_desc_t), intent(in)  :: file
@@ -1634,11 +1702,11 @@ integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_l
     ierr = def_var_deflate_id(file, vardesc%varid, shuffle, deflate, deflate_level)
   end function def_var_deflate_desc
 
-!>
-!! @public
-!! @ingroup PIO_def_var_chunking
-!! @brief Changes chunking settings for a netCDF-4/HDF5 variable.
-!<
+  !>
+  !! @ingroup PIO_def_var_chunking
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
   integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
@@ -1665,11 +1733,11 @@ end function PIOc_def_var_chunking
     ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
   end function def_var_chunking
 
-!>
-!! @public
-!! @ingroup PIO_set_chunk_cache
-!! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
-!<
+  !>
+  !! @ingroup PIO_set_chunk_cache
+  !! Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
+  !! @author Ed Hartnett
+  !<
   integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
        chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
@@ -1695,11 +1763,11 @@ end function PIOc_set_chunk_cache
          chunk_cache_preemption)
   end function set_chunk_cache
 
-!>
-!! @public
-!! @ingroup PIO_get_chunk_cache
-!! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
-!<
+  !>
+  !! @ingroup PIO_get_chunk_cache
+  !! Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files).
+  !! @author Ed Hartnett
+  !<
   integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
        chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
@@ -1725,11 +1793,11 @@ end function PIOc_get_chunk_cache
          chunk_cache_preemption)
   end function get_chunk_cache
 
-!>
-!! @public
-!! @ingroup PIO_set_var_chunk_cache
-!! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
-!<
+  !>
+  !! @ingroup PIO_set_var_chunk_cache
+  !! Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
+  !! @author Ed Hartnett
+  !<
   integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, &
        chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
@@ -1756,10 +1824,10 @@ end function PIOc_set_var_chunk_cache
   end function set_var_chunk_cache_id
 
   !>
-!! @public
-!! @ingroup PIO_set_var_chunk_cache
-!! @brief Changes chunk cacne for a variable.
-!<
+  !! @ingroup PIO_set_var_chunk_cache
+  !! Changes chunk cacne for a variable.
+  !! @author Ed Hartnett
+  !<
   integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
        chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
@@ -1772,11 +1840,11 @@ integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_desc
 
-!>
-!! @public
-!! @ingroup PIO_get_var_chunk_cache
-!! @brief Get the chunk cache settings for a variable.
-!<
+  !>
+  !! @ingroup PIO_get_var_chunk_cache
+  !! Get the chunk cache settings for a variable.
+  !! @author Ed Hartnett
+  !<
   integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
        chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
@@ -1789,11 +1857,11 @@ integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_desc
 
-!>
-!! @public
-!! @ingroup PIO_get_var_chunk_cache
-!! @brief Get the chunk cache settings for a variable.
-!<
+  !>
+  !! @ingroup PIO_get_var_chunk_cache
+  !! Get the chunk cache settings for a variable.
+  !! @author Ed Hartnett
+  !<
   integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, &
        chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
diff --git a/src/flib/pio_support.F90 b/src/flib/pio_support.F90
index fe0fde0b59e0..bb0381488957 100644
--- a/src/flib/pio_support.F90
+++ b/src/flib/pio_support.F90
@@ -1,7 +1,6 @@
 !>
-!! @file pio_support.F90
-!! @brief internal code for compiler workarounds, aborts and debug functions
-!!
+!! @file
+!! Internal code for compiler workarounds, aborts and debug functions.
 !<
 module pio_support
   use pio_kinds
@@ -20,18 +19,16 @@ module pio_support
   public :: pio_writedof
   public :: replace_c_null
 
-  logical, public :: Debug=.FALSE.
-  logical, public :: DebugIO=.FALSE.
-  logical, public :: DebugAsync=.FALSE.
+  logical, public :: Debug=.FALSE.            !< debug mode
+  logical, public :: DebugIO=.FALSE.          !< IO debug mode
+  logical, public :: DebugAsync=.FALSE.       !< async debug mode
   integer,private,parameter :: versno = 1001
 
   character(len=*), parameter :: modName='pio_support'
 
 contains
-!> 
-!! @public
-!! @brief Remove null termination (C-style) from strings for Fortran.
-!<
+  !> Remove null termination (C-style) from strings for Fortran.
+  !<
   subroutine replace_c_null(istr, ilen)
     use iso_c_binding, only : C_NULL_CHAR
     character(len=*),intent(inout) :: istr
@@ -48,32 +45,17 @@ subroutine replace_c_null(istr, ilen)
     istr(i:slen)=''
   end subroutine replace_c_null
 
-!>
-!! @public
-!! @brief Abort the model for abnormal termination.
-!! @param file : File where piodie is called from.
-!! @param line : Line number where it is called.
-!! @param msg,msg2,msg3,ival1,ival2,ival3,mpirank : Optional argument for error messages.
-!<
+  !>
+  !! Abort the model for abnormal termination.
+  !!
+  !! @param file File where piodie is called from.
+  !! @param line Line number where it is called.
+  !! @param msg,msg2,msg3,ival1,ival2,ival3,mpirank : Optional
+  !! argument for error messages.
+  !! @author Jim Edwards
+  !<
   subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank)
-    !-----------------------------------------------------------------------
-    ! Purpose:
-    !
-    ! Abort the model for abnormal termination
-    !
-    ! Author: Jim Edwards
-    !
-    ! Change History
-    ! 20070608 R. Loy  added optional args
-    !-----------------------------------------------------------------------
-    ! $Id$
-    !-----------------------------------------------------------------------
-    !-----------------------------------------------------------------------
     implicit none
-    !-----------------------------------------------------------------------
-    !
-    ! Arguments
-    !
     character(len=*), intent(in) :: file
     integer,intent(in) :: line
     character(len=*), intent(in), optional :: msg,msg2,msg3
@@ -81,7 +63,7 @@ subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank)
 
     character(len=*), parameter :: subName=modName//'::pio_die'
     integer :: ierr, myrank=-1
-    
+
     if(present(mpirank)) myrank=mpirank
 
     if (present(ival3)) then
@@ -104,8 +86,8 @@ subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank)
 
 
 #if defined(CPRXLF) && !defined(BGQ)
-  close(5)    ! needed to prevent batch jobs from hanging in xl__trbk
-  call xl__trbk()
+    close(5)    ! needed to prevent batch jobs from hanging in xl__trbk
+    call xl__trbk()
 #endif
 
     ! passing an argument of 1 to mpi_abort will lead to a STOPALL output 
@@ -118,69 +100,51 @@ subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank)
     call abort
 #endif
 
-
   end subroutine piodie
 
-!=============================================
-!  CheckMPIreturn:
-!
-!      Check and prints an error message
-!  if an error occured in a MPI subroutine.
-!=============================================
-!>
-!! @public
-!! @brief Check and prints an error message if an error occured in an MPI 
-!! subroutine.
-!! @param locmesg : Message to output
-!! @param errcode : MPI error code
-!! @param file : The file where the error message originated.
-!! @param line : The line number where the error message originated.
-!<
+  !>
+  !! Check and prints an error message if an error occured in an MPI
+  !! subroutine.
+  !!
+  !! @param locmesg Message to output
+  !! @param errcode MPI error code
+  !! @param file The file where the error message originated.
+  !! @param line The line number where the error message originated.
+  !! @author Jim Edwards
+  !<
   subroutine CheckMPIreturn(locmesg, errcode, file, line)
 
-     character(len=*), intent(in) :: locmesg
-     integer(i4), intent(in) :: errcode
-     character(len=*),optional :: file
-     integer, intent(in),optional :: line
-     character(len=MPI_MAX_ERROR_STRING) :: errorstring
-
-     integer(i4) :: errorlen
-
-     integer(i4) :: ierr
-     if (errcode .ne. MPI_SUCCESS) then
-        call MPI_Error_String(errcode,errorstring,errorlen,ierr)
-        write(*,*) TRIM(ADJUSTL(locmesg))//errorstring(1:errorlen)
-        if(present(file).and.present(line)) then
-           call piodie(file,line)
-        endif
-     end if
+    character(len=*), intent(in) :: locmesg
+    integer(i4), intent(in) :: errcode
+    character(len=*),optional :: file
+    integer, intent(in),optional :: line
+    character(len=MPI_MAX_ERROR_STRING) :: errorstring
+
+    integer(i4) :: errorlen
+
+    integer(i4) :: ierr
+    if (errcode .ne. MPI_SUCCESS) then
+       call MPI_Error_String(errcode,errorstring,errorlen,ierr)
+       write(*,*) TRIM(ADJUSTL(locmesg))//errorstring(1:errorlen)
+       if(present(file).and.present(line)) then
+          call piodie(file,line)
+       endif
+    end if
   end subroutine CheckMPIreturn
 
-!>
-!! @public
-!! @brief Fortran interface to write a mapping file
-!! @param file : The file where the decomp map will be written.
-!! @param gdims : The dimensions of the data array in memory.
-!! @param DOF : The multidimensional array of indexes that describes how 
-!! data in memory are written to a file.
-!! @param comm : The MPI comm index.
-!! @param punit : Optional argument that is no longer used.
-!<
+  !>
+  !! Fortran interface to write a mapping file.
+  !!
+  !! @param file : The file where the decomp map will be written.
+  !! @param gdims : The dimensions of the data array in memory.
+  !! @param DOF : The multidimensional array of indexes that describes how
+  !! data in memory are written to a file.
+  !! @param comm : The MPI comm index.
+  !! @param punit : Optional argument that is no longer used.
+  !! @author T Craig
+  !<
   subroutine pio_writedof (file, gdims, DOF, comm, punit)
-    !-----------------------------------------------------------------------
-    ! Purpose:
-    !
-    ! Write a DOF to standard format
-    !
-    ! Author: T Craig
-    !
-    !-----------------------------------------------------------------------
-    !-----------------------------------------------------------------------
     implicit none
-    !-----------------------------------------------------------------------
-    !
-    ! Arguments
-    !
     character(len=*),intent(in) :: file
     integer, intent(in) :: gdims(:)
     integer(PIO_OFFSET_KIND)  ,intent(in) :: dof(:)
@@ -188,7 +152,7 @@ subroutine pio_writedof (file, gdims, DOF, comm, punit)
     integer,optional,intent(in) :: punit
     integer :: err
     integer :: ndims
-    
+
 
     interface
        integer(c_int) function PIOc_writemap_from_f90(file, ndims, gdims, maplen, map, f90_comm) &
@@ -207,35 +171,21 @@ end function PIOc_writemap_from_f90
 
   end subroutine pio_writedof
 
-!>
-!! @public
-!! @brief Fortran interface to read a mapping file
-!! @param file : The file from where the decomp map is read.
-!! @param ndims : The number of dimensions of the data.
-!! @param gdims : The actual dimensions of the data (pointer to an integer array of length ndims).
-!! @param DOF : Pointer to an integer array where the Decomp map will be stored.
-!! @param comm : MPI comm index
-!! @param punit : Optional argument that is no longer used.
-!<
+  !>
+  !! Fortran interface to read a mapping file.
+  !!
+  !! @param file The file from where the decomp map is read.
+  !! @param ndims The number of dimensions of the data.
+  !! @param gdims The actual dimensions of the data (pointer to an
+  !! integer array of length ndims).
+  !! @param DOF Pointer to an integer array where the Decomp map will
+  !! be stored.
+  !! @param comm MPI comm index
+  !! @param punit Optional argument that is no longer used.
+  !! @author T Craig
+  !<
   subroutine pio_readdof (file, ndims, gdims, DOF, comm, punit)
-    !-----------------------------------------------------------------------
-    ! Purpose:
-    !
-    ! Read a DOF to standard format
-    !
-    ! Author: T Craig
-    !
-    ! Change History
-    ! 
-    !-----------------------------------------------------------------------
-    ! $Id$
-    !-----------------------------------------------------------------------
-    !-----------------------------------------------------------------------
     implicit none
-    !-----------------------------------------------------------------------
-    !
-    ! Arguments
-    !
     character(len=*),intent(in) :: file
     integer(PIO_OFFSET_KIND),pointer:: dof(:)
     integer         ,intent(in) :: comm
@@ -262,9 +212,7 @@ end function PIOc_readmap_from_f90
 
     call c_f_pointer(tgdims, gdims, (/ndims/))
     call c_f_pointer(tmap, DOF, (/maplen/))
-!    DOF = DOF+1
+    !    DOF = DOF+1
   end subroutine pio_readdof
 
-
-
 end module pio_support
diff --git a/src/flib/pio_types.F90 b/src/flib/pio_types.F90
index 1e8fb9532e49..aeba6349b7ac 100644
--- a/src/flib/pio_types.F90
+++ b/src/flib/pio_types.F90
@@ -1,305 +1,213 @@
 !>
 !! @file
-!! @brief Derived datatypes and constants for PIO Fortran API
-!!
-!<
-module pio_types
-    use pio_kinds
-    use iso_c_binding
-    implicit none
-    private
-    !-------------------------------------------
-    !  data structure to describe decomposition
-    !-------------------------------------------
-    type, public :: DecompMap_t
-#ifdef SEQUENCE
-        sequence
-#endif
-        integer(i4) :: start
-        integer(i4) :: length
-    end type
-
-    !------------------------------------
-    !  a file descriptor data structure
-    !------------------------------------
-!>
-!! @public
-!! @struct iosystem_desc_t
-!! @brief A defined PIO system descriptor created by @ref PIO_init (see pio_types)
+!! Derived datatypes and constants for PIO Fortran API.
+!! @author Jim Edwards
 !<
-    type, public :: IOSystem_desc_t
-        integer(kind=c_int) :: iosysid = -1
-     end type IOSystem_desc_t
-
 !>
 !! @private
-!! @struct io_data_list
-!! @brief Linked list of buffers for pnetcdf non-blocking interface
-!>
-!    type, public :: io_data_list
-!       integer :: request
-!       real(r4), pointer :: data_real(:) => null()
-!       integer(i4), pointer :: data_int(:) => null()
-!       real(r8), pointer :: data_double(:) => null()
-!       type(io_data_list), pointer :: next => null()
-!    end type io_data_list
-
-
-!>
-!! @public
-!! @struct file_desc_t
-!! @brief File descriptor returned by \ref PIO_openfile or \ref PIO_createfile (see pio_types)
+!! @defgroup iodesc_generate Creating Decompositions
+!! Create a decomposition of data from a variable to multiple
+!! computation tasks.
 !!
-!>
-    type, public :: File_desc_t
-       integer(kind=c_int) :: fh
-       type(iosystem_desc_t), pointer :: iosystem => null()
-    end type File_desc_t
-
-
-!>
-!! @private
-!! @defgroup iodesc_generate io descriptors, generating
-!! @brief The io descriptor structure in defined in this subroutine
-!! and subsequently used in @ref PIO_read_darray, @ref PIO_write_darray,
-!! @ref PIO_put_var, @ref PIO_get_var calls (see pio_types).
-!<
-
-!>
 !! @public
-!! @struct io_desc_t
-!! @brief  An io descriptor handle that is generated in @ref PIO_initdecomp
-!! (see pio_types)
-!<
-   type, public :: io_desc_t
-#ifdef SEQUENCE
-        sequence
-#endif
-        integer(i4)         :: ioid
-    end type
-
-!>
-!! @public
-!! @struct var_desc_t
-!! @brief A variable descriptor returned from @ref PIO_def_var (see pio_types)
-!<
-    type, public :: Var_desc_t
-#ifdef SEQUENCE
-       sequence
-#endif
-       integer(i4) :: varID
-       integer(i4) :: ncid
-    end type Var_desc_t
-
-!>
 !! @defgroup PIO_iotype PIO_iotype
-!! @public
-!! @brief An integer parameter which controls the iotype
-!! @details
+!! An integer parameter which controls the iotype.
 !!   - PIO_iotype_pnetcdf : parallel read/write of pNetCDF files (netcdf3)
 !!   - PIO_iotype_netcdf : serial read/write of NetCDF files using 'base_node' (netcdf3)
 !!   - PIO_iotype_netcdf4c : parallel read/serial write of NetCDF4 (HDF5) files with data compression
 !!   - PIO_iotype_netcdf4p : parallel read/write of NETCDF4 (HDF5) files
-!>
-    integer(i4), public, parameter ::  &
-        PIO_iotype_pnetcdf = 1, &   ! parallel read/write of pNetCDF files
-        PIO_iotype_netcdf  = 2, &   ! serial read/write of NetCDF file using 'base_node'
-        PIO_iotype_netcdf4c = 3, &  ! netcdf4 (hdf5 format) file opened for compression (serial write access only)
-        PIO_iotype_netcdf4p = 4    ! netcdf4 (hdf5 format) file opened in parallel (all netcdf4 files for read will be opened this way)
-
-
-! These are for backward compatability and should not be used or expanded upon
-    integer(i4), public, parameter ::                       &
-        iotype_pnetcdf = PIO_iotype_pnetcdf,                &
-        iotype_netcdf  = PIO_iotype_netcdf
-
-
-!>
-!! @defgroup PIO_rearr_method PIO_rearr_method
-!! @public
-!! @brief The three choices to control rearrangement are:
-!! @details
+!!
+!! @defgroup PIO_rearr_method Rearranger Methods
+!! Rearranger methods.
 !!  - PIO_rearr_none : Do not use any form of rearrangement
 !!  - PIO_rearr_box : Use a PIO internal box rearrangement
-!! -  PIO_rearr_subset : Use a PIO internal subsetting rearrangement
-!>
-
-    integer(i4), public, parameter :: PIO_rearr_box =  1
-    integer(i4), public, parameter :: PIO_rearr_subset =  2
-
-!>
-!! @public
-!! @defgroup PIO_error_method error_methods
-!! @details
-!! The three types of error handling methods are:
+!!  - PIO_rearr_subset : Use a PIO internal subsetting rearrangement
+!!
+!! @defgroup PIO_error_method Error Handling Methods
+!! The error handling setting controls what happens if errors are
+!! encountered by PIO. The three types of error handling methods are:
 !!  - PIO_INTERNAL_ERROR  : abort on error from any task
 !!  - PIO_BCAST_ERROR     : broadcast an error from io_rank 0 to all tasks in comm
 !!  - PIO_RETURN_ERROR    : do nothing - allow the user to handle it
-!<
-  integer(i4), public, parameter :: PIO_INTERNAL_ERROR = -51
-  integer(i4), public, parameter :: PIO_BCAST_ERROR = -52
-  integer(i4), public, parameter :: PIO_RETURN_ERROR = -53
-
-!>
-!! @public
-!! @defgroup PIO_error_method error_methods
-!! @details
-!! Use this instead of ios to set error handling for the library.
-!<
-  integer(i4), public, parameter :: PIO_DEFAULT = -1
-
-!>
-!! @public
-!! @defgroup error_return error return codes
-!! @brief : The error return code; ierr != PIO_noerr indicates
-!! an error. (see @ref PIO_seterrorhandling )
 !>
-
-!>
-!! @struct use_PIO_kinds
-!! @brief The type of variable(s) associated with this iodesc.
-!! @copydoc PIO_kinds
-!<
-
-!>
-!! @public
-!! @defgroup PIO_kinds PIO_kinds
-!! @brief The base types supported by PIO are:
-!! @details
+!! @defgroup error_return Error Return Codes
+!! The error return code (see @ref PIO_seterrorhandling).
+!!
+!! @defgroup PIO_kinds PIO Fortran Type Kinds
+!! PIO supports different kinds of Fortran types.
 !!  - PIO_double : 8-byte reals or double precision
 !!  - PIO_real : 4-byte reals
 !!  - PIO_int :  4-byte integers
 !!  - PIO_char : character
-!<
-#ifdef _PNETCDF
-#include    /* _EXTERNAL */
-   integer, public, parameter :: PIO_global = nf_global
-   integer, public, parameter :: PIO_unlimited = nf_unlimited
-   integer, public, parameter :: PIO_double = nf_double
-   integer, public, parameter :: PIO_real   = nf_real
-   integer, public, parameter :: PIO_int    = nf_int
-   integer, public, parameter :: PIO_char   = nf_char
-   integer, public, parameter :: PIO_noerr  = nf_noerr
-   integer, public, parameter :: PIO_WRITE  = nf_write
-   integer, public, parameter :: PIO_nowrite  = nf_nowrite
-   integer, public, parameter :: PIO_CLOBBER = nf_clobber
-   integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber
-   integer, public, parameter :: PIO_NOFILL = nf_nofill
-   integer, public, parameter :: PIO_MAX_NAME = nf_max_name
-   integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims)
-   integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset
-   integer, public, parameter :: PIO_64BIT_DATA = nf_64bit_data
-   integer, public, parameter :: PIO_FILL_INT = nf_fill_int;
-   real, public, parameter :: PIO_FILL_FLOAT = nf_fill_float;
-   double precision, public, parameter :: PIO_FILL_DOUBLE = nf_fill_double;
 
-#else
-#ifdef _NETCDF
-#include    /* _EXTERNAL */
-   integer, public, parameter :: PIO_global = nf_global
-   integer, public, parameter :: PIO_unlimited = nf_unlimited
-   integer, public, parameter :: PIO_double = nf_double
-   integer, public, parameter :: PIO_real   = nf_real
-   integer, public, parameter :: PIO_int    = nf_int
-   integer, public, parameter :: PIO_char   = nf_char
-   integer, public, parameter :: PIO_noerr  = nf_noerr
-   integer, public, parameter :: PIO_WRITE  = nf_write
-   integer, public, parameter :: PIO_nowrite = nf_nowrite
-   integer, public, parameter :: PIO_CLOBBER = nf_clobber
-   integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber
-   integer, public, parameter :: PIO_NOFILL = nf_nofill
-   integer, public, parameter :: PIO_MAX_NAME = nf_max_name
-   integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims)
-   integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset
-   integer, public, parameter :: PIO_64BIT_DATA = 0
-   integer, public, parameter :: PIO_FILL_INT = nf_fill_int;
-   real, public, parameter :: PIO_FILL_FLOAT = nf_fill_float;
-   double precision, public, parameter :: PIO_FILL_DOUBLE = nf_fill_double;
-#else
-   integer, public, parameter :: PIO_global = 0
-   integer, public, parameter :: PIO_double = 6
-   integer, public, parameter :: PIO_real   = 5
-   integer, public, parameter :: PIO_int    = 4
-   integer, public, parameter :: PIO_char   = 2
-   integer, public, parameter :: PIO_noerr  = 0
-   integer, public, parameter :: PIO_MAX_NAME = 25
-   integer, public, parameter :: PIO_MAX_VAR_DIMS = 6
-   integer, public, parameter :: PIO_CLOBBER = 10
-   integer, public, parameter :: PIO_NOCLOBBER = 11
-   integer, public, parameter :: PIO_WRITE = 20
-   integer, public, parameter :: PIO_NOWRITE = 21
-   integer, public, parameter :: PIO_64BIT_OFFSET = 0
-   integer, public, parameter :: PIO_64BIT_DATA = 0
-   integer, public, parameter :: PIO_FILL_INT = -2147483647;
-   real, public, parameter :: PIO_FILL_FLOAT =  9.9692099683868690e+36;
-   double precision, public, parameter :: PIO_FILL_DOUBLE = 9.9692099683868690e+36;
+module pio_types
+  use pio_kinds
+  use iso_c_binding
+  implicit none
+  private
+  type, public :: DecompMap_t !< data structure to describe decomposition.
+#ifdef SEQUENCE
+     sequence
+#endif
+     integer(i4) :: start  !< start
+     integer(i4) :: length !< length
+  end type DecompMap_t
+
+  !>
+  !! @struct iosystem_desc_t
+  !! A defined PIO system descriptor created by @ref PIO_init (see
+  !! pio_types).
+  type, public :: IOSystem_desc_t
+     integer(kind=c_int) :: iosysid = -1 !< iosysid
+  end type IOSystem_desc_t
+
+  !>
+  !! @public
+  !! @struct file_desc_t
+  !! File descriptor returned by \ref PIO_openfile or \ref
+  !! PIO_createfile (see pio_types).
+  type, public :: File_desc_t
+     integer(kind=c_int) :: fh !< file handle
+     type(iosystem_desc_t), pointer :: iosystem => null() !< iosystem
+  end type File_desc_t
+
+  !>
+  !! @public
+  !! @struct io_desc_t
+  !! An decomposition handle that is generated in @ref PIO_initdecomp.
+  !! (see pio_types)
+  type, public :: io_desc_t
+#ifdef SEQUENCE
+     sequence
 #endif
+     integer(i4)         :: ioid !< decomposition id
+  end type io_desc_t
+
+  !>
+  !! @public
+  !! @struct var_desc_t
+  !! A variable descriptor returned from @ref PIO_def_var (see
+  !! pio_types).
+  type, public :: Var_desc_t
+#ifdef SEQUENCE
+     sequence
 #endif
-   integer, public, parameter :: PIO_num_OST =  16
+     integer(i4) :: varID !< variable id
+     integer(i4) :: ncid  !< file id
+  end type Var_desc_t
 
-!>
-!! @defgroup PIO_rearr_comm_t PIO_rearr_comm_t
-!! @public
-!! @brief The two choices for rearranger communication
-!! @details
-!!  - PIO_rearr_comm_p2p : Point to point
-!!  - PIO_rearr_comm_coll : Collective
-!>
-    enum, bind(c)
-      enumerator :: PIO_rearr_comm_p2p = 0
-      enumerator :: PIO_rearr_comm_coll
-    end enum
+  integer(i4), public, parameter ::  &
+       PIO_iotype_pnetcdf = 1, &   !< parallel read/write of pNetCDF files
+       PIO_iotype_netcdf  = 2, &   !< serial read/write of NetCDF file using 'base_node'
+       PIO_iotype_netcdf4c = 3, &  !< netcdf4 (hdf5 format) file opened for compression (serial write access only)
+       PIO_iotype_netcdf4p = 4     !< netcdf4 (hdf5 format) file opened in parallel
 
-!>
-!! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
-!! @public
-!! @brief The four choices for rearranger communication direction
-!! @details
-!!  - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
-!!  - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
-!!  - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
-!!  - PIO_rearr_comm_fc_2d_disable: Disable flow control
-!>
-    enum, bind(c)
-      enumerator :: PIO_rearr_comm_fc_2d_enable = 0
-      enumerator :: PIO_rearr_comm_fc_1d_comp2io
-      enumerator :: PIO_rearr_comm_fc_1d_io2comp
-      enumerator :: PIO_rearr_comm_fc_2d_disable
-    end enum
 
-!>
-!! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
-!! @brief Type that defines the PIO rearranger options
-!! @details
-!!  - enable_hs : Enable handshake (true/false)
-!!  - enable_isend : Enable Isends (true/false)
-!!  - max_pend_req : Maximum pending requests (To indicated unlimited
-!!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
-!>
-   type, bind(c), public :: PIO_rearr_comm_fc_opt_t
-      logical(c_bool) :: enable_hs            ! Enable handshake?
-      logical(c_bool) :: enable_isend         ! Enable isends?
-      integer(c_int) :: max_pend_req         ! Maximum pending requests
-    end type PIO_rearr_comm_fc_opt_t
+  ! These are for backward compatability and should not be used or expanded upon
+  integer(i4), public, parameter ::                       &
+       iotype_pnetcdf = PIO_iotype_pnetcdf,                & !< pnetcdf iotype
+       iotype_netcdf  = PIO_iotype_netcdf                    !< netcdf iotype
 
-    integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
-!>
-!! @defgroup PIO_rearr_options PIO_rearr_options
-!! @brief Type that defines the PIO rearranger options
-!! @details
-!!  - comm_type : @copydoc PIO_rearr_comm_t
-!!  - fcd : @copydoc PIO_rearr_comm_dir
-!!  - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
-!>
-    type, bind(c), public :: PIO_rearr_opt_t
-      integer(c_int)                         :: comm_type
-      integer(c_int)                         :: fcd       ! Flow control direction
-      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
-      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
-    end type PIO_rearr_opt_t
 
-    public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
-              PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
-              PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
+  integer(i4), public, parameter :: PIO_rearr_box =  1    !< box rearranger
+  integer(i4), public, parameter :: PIO_rearr_subset =  2 !< subset rearranger
+
+  integer(i4), public, parameter :: PIO_INTERNAL_ERROR = -51 !< abort on error from any task
+  integer(i4), public, parameter :: PIO_BCAST_ERROR = -52    !< broadcast an error
+  integer(i4), public, parameter :: PIO_RETURN_ERROR = -53   !< do nothing
+
+  integer(i4), public, parameter :: PIO_DEFAULT = -1 !< default error handler
+
+  !>
+  !! @struct use_PIO_kinds
+  !! The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+
+#ifdef _PNETCDF
+#include 
+  integer, public, parameter :: PIO_64BIT_DATA = nf_64bit_data            !< CDF5 format
+#else
+#include 
+  integer, public, parameter :: PIO_64BIT_DATA = 0            !< CDF5 format
+#endif
+  integer, public, parameter :: PIO_num_OST =  16 !< num ost
+  integer, public, parameter :: PIO_global = nf_global       !< global atts
+  integer, public, parameter :: PIO_unlimited = nf_unlimited !< unlimited dimension
+  integer, public, parameter :: PIO_double = nf_double       !< double type
+  integer, public, parameter :: PIO_real   = nf_real         !< real type
+  integer, public, parameter :: PIO_int    = nf_int          !< int type
+  integer, public, parameter :: PIO_char   = nf_char         !< char type
+  integer, public, parameter :: PIO_noerr  = nf_noerr        !< no error
+  integer, public, parameter :: PIO_WRITE  = nf_write        !< read-write
+  integer, public, parameter :: PIO_nowrite  = nf_nowrite    !< read-only
+  integer, public, parameter :: PIO_CLOBBER = nf_clobber     !< clobber existing file
+  integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber !< do not clobber existing file
+  integer, public, parameter :: PIO_NOFILL = nf_nofill       !< do not use fill values
+  integer, public, parameter :: PIO_MAX_NAME = nf_max_name   !< max name len
+  integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims) !< max dims for a var
+  integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset        !< 64bit offset format
+  integer, public, parameter :: PIO_FILL_INT = nf_fill_int;               !< int fill value
+  real, public, parameter :: PIO_FILL_FLOAT = nf_fill_float;              !< float fill value
+  double precision, public, parameter :: PIO_FILL_DOUBLE = nf_fill_double; !< double fill value
+
+  enum, bind(c)
+     enumerator :: PIO_rearr_comm_p2p = 0
+     enumerator :: PIO_rearr_comm_coll
+  end enum
+
+  !>
+  !! @defgroup PIO_rearr_comm_t Rearranger Communication
+  !! @public
+  !! There are two choices for rearranger communication.
+  !!  - PIO_rearr_comm_p2p : Point to point
+  !!  - PIO_rearr_comm_coll : Collective
+  !>
+  !>
+  !! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
+  !! @public
+  !! There are four choices for rearranger communication direction.
+  !!  - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
+  !!  - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
+  !!  - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
+  !!  - PIO_rearr_comm_fc_2d_disable: Disable flow control
+  !!
+  !! @defgroup PIO_rearr_comm_fc_options Rearranger Flow Control Options
+  !! Type that defines the PIO rearranger options.
+  !!  - enable_hs : Enable handshake (true/false)
+  !!  - enable_isend : Enable Isends (true/false)
+  !!  - max_pend_req : Maximum pending requests (To indicated unlimited
+  !!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+  !!
+  !! @defgroup PIO_rearr_options Rearranger Options
+  !! Type that defines the PIO rearranger options.
+  !!
+  !!  - comm_type : @copydoc PIO_rearr_comm_t
+  !!  - fcd : @copydoc PIO_rearr_comm_dir
+  !!  - comm_fc_opts_comp2io : @copydoc PIO_rearr_comm_fc_options
+  !!  - comm_fc_opts_io2comp : @copydoc PIO_rearr_comm_fc_options
+  enum, bind(c)
+     enumerator :: PIO_rearr_comm_fc_2d_enable = 0 !< COMM procs to IO procs and vice versa.
+     enumerator :: PIO_rearr_comm_fc_1d_comp2io !< COMM procs to IO procs only.
+     enumerator :: PIO_rearr_comm_fc_1d_io2comp !< IO procs to COMM procs only.
+     enumerator :: PIO_rearr_comm_fc_2d_disable !< Disable flow control.
+  end enum
+
+  type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+     logical(c_bool) :: enable_hs            !< Enable handshake.
+     logical(c_bool) :: enable_isend         !< Enable isends.
+     integer(c_int) :: max_pend_req          !< Maximum pending requests (PIO_REARR_COMM_UNLIMITED_PEND_REQ for unlimited).
+  end type PIO_rearr_comm_fc_opt_t
+
+  integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1 !< unlimited requests
+  type, bind(c), public :: PIO_rearr_opt_t
+     integer(c_int)                         :: comm_type !< Rearranger communication.
+     integer(c_int)                         :: fcd       !< Communication direction.
+     type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io !< The comp2io options.
+     type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp !< The io2comp options.
+  end type PIO_rearr_opt_t
+
+  public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
+       PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
+       PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
 
 end module pio_types
diff --git a/src/flib/piodarray.F90.in b/src/flib/piodarray.F90.in
index 7c80e89df8ad..3fe4e8904c10 100644
--- a/src/flib/piodarray.F90.in
+++ b/src/flib/piodarray.F90.in
@@ -1,7 +1,15 @@
 #define __PIO_FILE__ 'piodarray'
 !>
 !! @file
-!! @brief Read and write routines for decomposed data.
+!! Read and write routines for decomposed data.
+!>
+!! @defgroup PIO_write_darray Write from Distributed Arrays
+!! The overloaded PIO_write_darray writes a distributed array to
+!! disk in Fortran.
+!!
+!! @defgroup PIO_read_darray Read to Distributed Arrays
+!! The overloaded PIO_read_darray function reads a distributed array
+!! from disk in Fortran.
 !<
 module piodarray
   use pio_types, only : file_desc_t, io_desc_t, var_desc_t
@@ -16,11 +24,6 @@ module piodarray
   private
   public :: pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
 
-
-!>
-!! @defgroup PIO_write_darray PIO_write_darray
-!! @brief The overloaded PIO_write_darray writes a distributed array to disk.
-!<
   interface PIO_write_darray
 ! TYPE real,int,double
 ! DIMS 1,2,3,4,5,6,7
@@ -29,11 +32,6 @@ module piodarray
      module procedure write_darray_multi_1d_{TYPE}
   end interface
 
-
-!>
-!! @defgroup PIO_read_darray PIO_read_darray
-!! @brief The overloaded PIO_read_darray function reads a distributed array from disk.
-!<
   interface PIO_read_darray
 ! TYPE real,int,double
 ! DIMS 1,2,3,4,5,6,7
@@ -43,10 +41,6 @@ module piodarray
 
   character(len=*), parameter, private  :: modName='piodarray'
 
-#ifdef MEMCHK
-integer :: msize, rss, mshare, mtext, mstack, lastrss=0
-#endif
-
 interface
    integer(C_INT) function PIOc_write_darray(ncid, vid, ioid, arraylen, array, fillvalue) &
         bind(C,name="PIOc_write_darray")
@@ -88,7 +82,7 @@ end interface
 
 
 contains
-
+  !> Set buffer size limit.
   subroutine pio_set_buffer_size_limit(limit)
     integer(PIO_OFFSET_KIND), intent(in) :: limit
     integer(PIO_OFFSET_KIND) :: oldval
@@ -107,16 +101,11 @@ contains
   end subroutine pio_set_buffer_size_limit
 
 ! TYPE real,int,double
+  !> 1D write_darray for type {TYPE}. Writes a 2-d slab of TYPE to a
+  !! netcdf file.
+  !<
   subroutine write_darray_1d_cinterface_{TYPE} (File,varDesc,ioDesc, arraylen, array, iostat, fillval)
     use iso_c_binding
-    ! !DESCRIPTION:
-    !  Writes a 2-d slab of TYPE to a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
-
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
 
@@ -151,16 +140,9 @@ contains
   end subroutine write_darray_1d_cinterface_{TYPE}
 
 ! TYPE real,int,double
+  !> 1D write_darray_multi for type {TYPE}. Writes a 2-d slab of TYPE to a netcdf file.
   subroutine write_darray_multi_1d_cinterface_{TYPE} (File,varDesc,ioDesc,nvars,arraylen, array, iostat, fillval)
     use iso_c_binding
-    ! !DESCRIPTION:
-    !  Writes a 2-d slab of TYPE to a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
-
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
 
@@ -198,27 +180,19 @@ contains
   end subroutine write_darray_multi_1d_cinterface_{TYPE}
 
 ! TYPE real,int,double
-!>
-!! @public
-!! @ingroup PIO_write_darray
-!! @brief Writes a 1D array of type {TYPE}.
-!! @details
-!! @param File    \ref file_desc_t
-!! @param varDesc \ref var_desc_t
-!! @param ioDesc  \ref io_desc_t
-!! @param array  : The data to be written
-!! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
-!! @param fillval : An optional fill value to fill holes in the data written
-!<
+  !>
+  !! @ingroup PIO_write_darray
+  !! Writes a 1D array of type {TYPE}. Writes a block of TYPE to a netcdf file.
+  !!
+  !! @param File    \ref file_desc_t
+  !! @param varDesc \ref var_desc_t
+  !! @param ioDesc  \ref io_desc_t
+  !! @param array  : The data to be written
+  !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
+  !! @param fillval : An optional fill value to fill holes in the data written
+  !! @author Jim Edwards
+  !<
   subroutine write_darray_multi_1d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
-    ! !DESCRIPTION:
-    !  Writes a block of TYPE to a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
-
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
 
@@ -244,15 +218,8 @@ contains
   end subroutine write_darray_multi_1d_{TYPE}
 
 ! TYPE real,int,double
+  !> Writes a block of TYPE to a netcdf file.
   subroutine write_darray_1d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
-    ! !DESCRIPTION:
-    !  Writes a block of TYPE to a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
-
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
 
@@ -276,21 +243,19 @@ contains
 
 ! TYPE real,int,double
 ! DIMS 2,3,4,5,6,7
-!>
-!! @public
-!! @ingroup PIO_write_darray
-!! @brief Writes a {DIMS}D array of type {TYPE}.
-!! @details
-!! @param File @ref file_desc_t
-!! @param varDesc @ref var_desc_t
-!! @param ioDesc  @ref io_desc_t
-!! @param array  : The data to be written
-!! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
-!! @param fillval : An optional fill value to fill holes in the data written
-!<
+  !>
+  !! @ingroup PIO_write_darray
+  !! Writes a {DIMS}D array of type {TYPE}.
+  !!
+  !! @param File @ref file_desc_t
+  !! @param varDesc @ref var_desc_t
+  !! @param ioDesc  @ref io_desc_t
+  !! @param array  : The data to be written
+  !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
+  !! @param fillval : An optional fill value to fill holes in the data written
+  !! @author Jim Edwards
+  !<
   subroutine write_darray_{DIMS}d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
-    ! !INPUT PARAMETERS:
-
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
 
@@ -331,27 +296,20 @@ contains
 
 ! TYPE real,int,double
 ! DIMS 1,2,3,4,5,6,7
-!>
-!! @public
-!! @ingroup PIO_read_darray
-!! @brief Read distributed array of type {TYPE} from a netCDF variable of {DIMS} dimension(s).
-!! @details
-!! @param File @ref file_desc_t
-!! @param varDesc @ref var_desc_t
-!! @param ioDesc @ref io_desc_t
-!! @param array  : The read data
-!! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
-!<
+  !>
+  !! @ingroup PIO_read_darray
+  !! Read distributed array of type {TYPE} from a netCDF variable of {DIMS} dimension(s).
+  !!
+  !! @param File @ref file_desc_t
+  !! @param varDesc @ref var_desc_t
+  !! @param ioDesc @ref io_desc_t
+  !! @param array  : The read data
+  !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
+  !! @author Jim Edwards
+  !<
   subroutine read_darray_{DIMS}d_{TYPE} (File,varDesc, ioDesc, array, iostat)
     use iso_c_binding
 !    use ifcore, only: tracebackqq
-    ! !DESCRIPTION:
-    !  Reads a slab of TYPE from a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
 
     type (File_desc_t), intent(inout) :: &
          File                   ! file information
@@ -368,22 +326,14 @@ contains
     integer(C_SIZE_T) :: tlen
 
     tlen = size(array)
-
     call read_darray_internal_{TYPE} (File%fh, vardesc%varid, iodesc%ioid, tlen, array, iostat)
 
   end subroutine read_darray_{DIMS}d_{TYPE}
 
 ! TYPE real,int,double
-
+  !> Internal read_darray for type {TYPE}
   subroutine read_darray_internal_{TYPE} (ncid, varid, ioid, alen, array, iostat)
     use iso_c_binding
-    ! !DESCRIPTION:
-    !  Reads a slab of TYPE from a netcdf file.
-    !
-    ! !REVISION HISTORY:
-    !  same as module
-
-    ! !INPUT PARAMETERS:
     integer, intent(in) :: ncid, varid, ioid
     integer(C_SIZE_T), intent(in) :: alen
 
@@ -399,4 +349,3 @@ contains
   end subroutine read_darray_internal_{TYPE}
 
 end module piodarray
-
diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90
index e6bb1a07d0c5..1f6900d3b084 100644
--- a/src/flib/piolib_mod.F90
+++ b/src/flib/piolib_mod.F90
@@ -1,25 +1,70 @@
 #define __PIO_FILE__ "piolib_mod.f90"
-#define debug_rearr 0
-
 !>
 !! @file
-!! @brief Initialization Routines for PIO
+!! Initialization Routines for PIO.
 !!
 !<
+
+!>
+!! @defgroup PIO_openfile Open a File
+!! Open an existing netCDF file with PIO in Fortran.
+!!
+!! @defgroup PIO_syncfile Sync File
+!! Sync a file to disk, flushing all buffers in Fortran.
+!!
+!! @defgroup PIO_createfile Create a File
+!! Create a new netCDF file in Fortran.
+!!
+!! @defgroup PIO_setframe Set Record Number
+!! Set the record number for distributed array reads/writes in
+!! Fortran.
+!!
+!! @defgroup PIO_closefile Close a File
+!! Close a netCDF file in Fortran.
+!!
+!! @defgroup PIO_freedecomp Free a Decomposition
+!! Free a decomposition, releasing all resources in Fortran.
+!!
+!! @defgroup PIO_init Initialize an IOSystem
+!! Create a new IO System, designating numbers of I/O and computation
+!! tasks in Fortran.
+!!
+!! @defgroup PIO_finalize Free an IOSystem
+!! Free an IO System, releasing all resources in Fortran.
+!!
+!! @defgroup PIO_initdecomp Define a Decomposition
+!! Define a new decomposition of variables to distributed arrays in
+!! Fortran.
+!!
+!! @defgroup PIO_getnumiotasks Get Number IO Tasks
+!! Get the number of IO tasks in Fortran.
+!!
+!! @defgroup PIO_setdebuglevel Internal Debug Settings for Fortran
+!! Set the debug level in Fortran.
+!!
+!! @defgroup PIO_seterrorhandling Error Handling for Fortran
+!! Set the behavior if an error is encountered in Fortran.
+!!
+!! @defgroup PIO_get_local_array_size Get Local Array Size
+!! Get the local size of the distributed array in a decomposition in
+!! Fortran.
+!!
+!! @defgroup PIO_set_hint Set MPI Hint
+!! Set the MPI hint in Fortran.
+
 module piolib_mod
   use iso_c_binding
   !--------------
   use pio_kinds
   !--------------
   use pio_types, only : file_desc_t, iosystem_desc_t, var_desc_t, io_desc_t, &
-        pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
-        pio_noerr, pio_rearr_subset, pio_rearr_opt_t
+       pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
+       pio_noerr, pio_rearr_subset, pio_rearr_opt_t
   !--------------
   use pio_support, only : piodie, debug, debugio, debugasync, checkmpireturn
   use pio_nf, only : pio_set_log_level
   !
 
-
 #ifdef TIMING
   use perf_mod, only : t_startf, t_stopf     ! _EXTERNAL
 #endif
@@ -54,99 +99,85 @@ module piolib_mod
        PIO_iotype_available, &
        PIO_set_rearr_opts
 
-#ifdef MEMCHK
-!> this is an internal variable for memory leak debugging
-!! it is used when macro memchk is defined and it causes each task to print the
-!! memory resident set size anytime it changes within pio.
-!<
-  integer :: lastrss=0
-#endif
-
-  !eop
-  !boc
   !-----------------------------------------------------------------------
   !
   !  module variables
   !
   !-----------------------------------------------------------------------
-!>
-!! @defgroup PIO_openfile PIO_openfile
-!<
+  !>
+  !! Open an existing netCDF file.
+  !<
   interface PIO_openfile
      module procedure PIO_openfile
-  end interface
+  end interface PIO_openfile
 
-!>
-!! @defgroup PIO_syncfile PIO_syncfile
-!<
+  !>
+  !! Sync the file to disk, flushing all buffers.
+  !<
   interface PIO_syncfile
      module procedure syncfile
-  end interface
+  end interface PIO_syncfile
 
-!>
-!! @defgroup PIO_createfile PIO_createfile
-!<
+  !>
+  !! Create a new netCDF file with PIO.
+  !<
   interface PIO_createfile
      module procedure createfile
-  end interface
+  end interface PIO_createfile
 
-!>
-!! @defgroup PIO_setframe PIO_setframe
-!! @brief sets the unlimited dimension for netcdf file
-!<
+  !>
+  !! Sets the record number for a future read/write of distributed
+  !! arrays (see @ref PIO_write_darray, @ref PIO_read_darray).
+  !<
   interface PIO_setframe
      module procedure setframe
-  end interface
+  end interface PIO_setframe
 
-!>
-!! @defgroup PIO_advanceframe PIO_advanceframe
-!<
+  !>
+  !! Increment the record number for a future read/write of distributed
+  !! arrays (see @ref PIO_write_darray, @ref PIO_read_darray).
+  !<
   interface PIO_advanceframe
      module procedure advanceframe
-  end interface
+  end interface PIO_advanceframe
 
-!>
-!! @defgroup PIO_closefile PIO_closefile
-!<
+  !>
+  !! Close an open file.
+  !<
   interface PIO_closefile
      module procedure closefile
-  end interface
-
+  end interface PIO_closefile
 
-!>
-!! @defgroup PIO_freedecomp PIO_freedecomp
-!! free memory associated with a io descriptor
-!<
+  !>
+  !! Free memory associated with a decomposition.
+  !<
   interface PIO_freedecomp
      module procedure freedecomp_ios
      module procedure freedecomp_file
-  end interface
+  end interface PIO_freedecomp
 
-!>
-!! @defgroup PIO_init PIO_init
-!! initializes the pio subsystem
-!<
+  !>
+  !! Initializes the PIO subsystem, creating a new IOSystem.
+  !<
   interface PIO_init
      module procedure init_intracom
      module procedure init_intercom
 
-  end interface
+  end interface PIO_init
 
-!>
-!! @defgroup PIO_finalize PIO_finalize
-!! Shuts down and cleans up any memory associated with the pio library.
-!<
+  !>
+  !! Shuts down an IOSystem and associated resources.
+  !<
   interface PIO_finalize
      module procedure finalize
-  end interface
-
-!>
-!! @defgroup PIO_initdecomp PIO_initdecomp
-!! @brief PIO_initdecomp is an overload interface the models decomposition to pio.
-!! @details initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4,
-!! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards
-!! compatibility.
-!<
+  end interface PIO_finalize
+
+  !>
+  !! PIO_initdecomp is an overload interface the models decomposition to pio.
+  !! initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4,
+  !! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards
+  !! compatibility.
+  !<
   interface PIO_initdecomp
      module procedure PIO_initdecomp_dof_i4  ! previous name: initdecomop_1dof_nf_box
      module procedure PIO_initdecomp_dof_i8  ! previous name: initdecomop_1dof_nf_box
@@ -159,56 +190,47 @@ module piolib_mod
      module procedure initdecomp_2dof_bin_i4
      module procedure initdecomp_2dof_bin_i8
      module procedure PIO_initdecomp_bc
-!     module procedure PIO_initdecomp_dof_dof
-  end interface
-
-!>
+     !     module procedure PIO_initdecomp_dof_dof
+  end interface PIO_initdecomp
 
-!>
-!! @defgroup PIO_getnumiotasks PIO_getnumiotasks
-!!  returns the actual number of IO-tasks used.  PIO
-!!  will reset the total number of IO-tasks if certain
-!!  conditions are meet
-!<
+  !>
+  !! Return the actual number of IO-tasks used. PIO will reset the
+  !! total number of IO-tasks if certain conditions are meet.
+  !<
   interface PIO_get_numiotasks
      module procedure getnumiotasks
-  end interface
+  end interface PIO_get_numiotasks
   interface PIO_getnumiotasks
      module procedure getnumiotasks
-  end interface
+  end interface PIO_getnumiotasks
 
-!>
-!!  @defgroup PIO_setdebuglevel PIO_setdebuglevel
-!!  sets the level of debug information that pio will generate.
-!<
+  !>
+  !! Set the level of debug information that PIO will generate.
+  !<
   interface PIO_setdebuglevel
      module procedure setdebuglevel
-  end interface
-
-!>
-!!  @defgroup PIO_seterrorhandling PIO_seterrorhandling
-!!  sets the form of error handling for pio.
-!!
-!! By default pio handles errors internally by printing a string
-!! describing the error and calling mpi_abort.  Application
-!! developers can change this behavior for calls to the underlying netcdf
-!! libraries with a call to PIO_seterrorhandling. For example if a
-!! developer wanted to see if an input netcdf format file contained the variable
-!! 'u' they might write the following
-!! @verbinclude errorhandle
-!<
+  end interface PIO_setdebuglevel
+
+  !>
+  !! Set the form of error handling for PIO.
+  !!
+  !! By default pio handles errors internally by printing a string
+  !! describing the error and calling mpi_abort. Application
+  !! developers can change this behavior for calls to the underlying
+  !! netcdf libraries with a call to PIO_seterrorhandling. For example
+  !! if a developer wanted to see if an input netcdf format file
+  !! contained the variable 'u' they might write the following
+  !! @verbinclude errorhandle
+  !<
   interface PIO_seterrorhandling
      module procedure seterrorhandlingfile
      module procedure seterrorhandlingiosystem
      module procedure seterrorhandlingiosysid
-  end interface
-
-!>
-!! @defgroup PIO_get_local_array_size PIO_get_local_array_size
-!<
+  end interface PIO_seterrorhandling
 
-  !eoc
-  !***********************************************************************
+  !>
+  !! Get the local size of a distributed array.
+  !<
 
 contains
 
@@ -224,13 +246,12 @@ module piolib_mod
 #define fptr(arg) arg
 !!$#endif
 
-!>
-!! @public
-!! @ingroup PIO_file_is_open
-!! @brief This logical function indicates if a file is open.
-!! @details
-!! @param File @copydoc file_desc_t
-!<
+  !>
+  !! @ingroup PIO_file_is_open
+  !! This logical function indicates if a file is open.
+  !! @param File @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
   logical function PIO_FILE_IS_OPEN(File)
     type(file_desc_t), intent(in) :: file
     interface
@@ -243,21 +264,22 @@ end function PIOc_File_is_Open
     end interface
     PIO_FILE_IS_OPEN = .false.
     if(associated(file%iosystem)) then
-      if(PIOc_File_is_Open(file%fh)==1) then
-        PIO_FILE_IS_OPEN = .true.
-      endif
+       if(PIOc_File_is_Open(file%fh)==1) then
+          PIO_FILE_IS_OPEN = .true.
+       endif
     endif
 
   end function PIO_FILE_IS_OPEN
 
-!>
-!! @public
-!! @ingroup PIO_get_local_array_size
-!! @brief This function returns the expected local size of an array associated with iodesc
-!! @details
-!! @param iodesc
-!! @copydoc io_desc_t
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_get_local_array_size
+  !! Return the expected local size of an array associated with a
+  !! decomposition.
+  !! @param iodesc the decomposition.
+  !! @copydoc io_desc_t
+  !! @author Jim Edwards
+  !<
   integer function PIO_get_local_array_size(iodesc)
     type(io_desc_t), intent(in) :: iodesc
     interface
@@ -271,14 +293,16 @@ end function PIOc_get_local_array_size
     PIO_get_local_array_size = PIOc_get_local_array_size(iodesc%ioid)
   end function PIO_get_local_array_size
 
-!>
-!! @public
-!! @ingroup PIO_advanceframe
-!! @brief advances the record dimension of a variable in a netcdf format file
-!!  or the block address in a binary file
-!! @details
-!! @param[in,out] vardesc @copybrief var_desc_t
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_setframe
+  !! Advance the record dimension of a variable in a netcdf format
+  !! file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copybrief var_desc_t
+  !! @author Jim Edwards
+  !<
   subroutine advanceframe(file, vardesc)
     type(file_desc_t), intent(in) :: file
     type(var_desc_t), intent(inout) :: vardesc
@@ -295,15 +319,17 @@ end function PIOc_advanceframe
     ierr = PIOc_advanceframe(file%fh, vardesc%varid-1)
   end subroutine advanceframe
 
-!>
-!! @public
-!! @ingroup PIO_setframe
-!! @brief sets the record dimension of a variable in a netcdf format file
-!! or the block address in a binary file
-!! @details
-!! @param vardesc @copydoc var_desc_t
-!! @param frame   : frame number to set
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_setframe
+  !! Set the record dimension of a variable in a netcdf format file
+  !! or the block address in a binary file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param frame record number
+  !! @author Jim Edwards
+  !<
   subroutine setframe(file, vardesc,frame)
     type(file_desc_t) :: file
     type(var_desc_t), intent(inout) :: vardesc
@@ -323,13 +349,15 @@ end function PIOc_setframe
     ierr = PIOc_setframe(file%fh, vardesc%varid-1, iframe)
   end subroutine setframe
 
-!>
-!! @public
-!! @ingroup PIO_setdebuglevel
-!! @brief sets the level of debug information output to stdout by pio
-!! @details
-!! @param level : default value is 0, allowed values 0-6
-!<
+  !>
+  !! @public
+  !! @public
+  !! @ingroup PIO_setdebuglevel
+  !! Set the level of debug information output to stdout by PIO.
+  !!
+  !! @param level default value is 0, allowed values 0-6
+  !! @author Jim Edwards
+  !<
   subroutine setdebuglevel(level)
     integer(i4), intent(in) :: level
     integer :: ierr
@@ -364,20 +392,22 @@ subroutine setdebuglevel(level)
     end if
     ierr = PIO_set_log_level(level)
     if(ierr /= PIO_NOERR) then
-      ! This is not a fatal error
-      print *, __PIO_FILE__, __LINE__, "Setting log level failed, ierr =",ierr
+       ! This is not a fatal error
+       print *, __PIO_FILE__, __LINE__, "Setting log level failed, ierr =",ierr
     end if
   end subroutine setdebuglevel
 
-!>
-!! @ingroup PIO_seterrorhandling
-!! @public
-!! @brief set the pio error handling method for a file
-!!
-!! @param file @copydoc file_desc_t
-!! @param method :
-!! @copydoc PIO_error_method
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a file.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @param method error handling method
+  !! @param oldmethod old error handling method
+  !! @copydoc PIO_error_method
+  !! @author Jim Edwards
+  !<
   subroutine seterrorhandlingfile(file, method, oldmethod)
     type(file_desc_t), intent(inout) :: file
     integer, intent(in) :: method
@@ -385,14 +415,16 @@ subroutine seterrorhandlingfile(file, method, oldmethod)
     call seterrorhandlingiosysid(file%iosystem%iosysid, method, oldmethod)
   end subroutine seterrorhandlingfile
 
-!>
-!! @ingroup PIO_seterrorhandling
-!! @public
-!! @brief set the pio error handling method for a pio system
-!! @param iosystem : a defined pio system descriptor, see PIO_types
-!! @param method :
-!! @copydoc PIO_error_method
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a pio system.
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param method
+  !! @copydoc PIO_error_method
+  !! @param oldmethod old error handling method
+  !! @author Jim Edwards
+  !<
   subroutine seterrorhandlingiosystem(iosystem, method, oldmethod)
     type(iosystem_desc_t), intent(inout) :: iosystem
     integer, intent(in) :: method
@@ -400,14 +432,18 @@ subroutine seterrorhandlingiosystem(iosystem, method, oldmethod)
     call seterrorhandlingiosysid(iosystem%iosysid, method, oldmethod)
   end subroutine seterrorhandlingiosystem
 
-!>
-!! @ingroup PIO_seterrorhandling
-!! @public
-!! @brief set the pio error handling method for a pio system or globally
-!! @param iosysid : a pio system ID (pass PIO_DEFAULT to change the global default error handling)
-!! @param method :
-!! @copydoc PIO_error_method
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a pio system or globally.
+  !!
+  !! @param iosysid a pio system ID (pass PIO_DEFAULT to change the
+  !! global default error handling)
+  !! @param method
+  !! @copydoc PIO_error_method
+  !! @param oldmethod old error handling method
+  !! @author Jim Edwards
+  !<
   subroutine seterrorhandlingiosysid(iosysid, method, oldmethod)
     integer, intent(in) :: iosysid
     integer, intent(in) :: method
@@ -428,25 +464,30 @@ end function PIOc_Set_IOSystem_Error_Handling
 
   end subroutine seterrorhandlingiosysid
 
-
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief Implements the @ref decomp_bc for PIO_initdecomp
-!! @details  This provides the ability to describe a computational
-!! decomposition in PIO that has a block-cyclic form.  That is
-!! something that can be described using start and count arrays.
-!! Optional parameters for this subroutine allows for the specification
-!! of io decomposition using iostart and iocount arrays.  If iostart
-!! and iocount arrays are not specified by the user, and rearrangement
-!! is turned on then PIO will calculate a suitable IO decomposition
-!! @param iosystem @copydoc iosystem_desc_t
-!! @param basepiotype @copydoc use_PIO_kinds
-!! @param dims An array of the global length of each dimesion of the variable(s)
-!! @param compstart The start index into the block-cyclic computational decomposition
-!! @param compcount The count for the block-cyclic computational decomposition
-!! @param iodesc @copydoc iodesc_generate
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! Implements the block-cyclic decomposition for PIO_initdecomp.
+  !! This provides the ability to describe a computational
+  !! decomposition in PIO that has a block-cyclic form. That is
+  !! something that can be described using start and count arrays.
+  !! Optional parameters for this subroutine allows for the
+  !! specification of io decomposition using iostart and iocount
+  !! arrays. If iostart and iocount arrays are not specified by the
+  !! user, and rearrangement is turned on then PIO will calculate a
+  !! suitable IO decomposition
+  !!
+  !! @param iosystem @copydoc iosystem_desc_t
+  !! @param basepiotype @copydoc use_PIO_kinds
+  !! @param dims An array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param compstart The start index into the block-cyclic
+  !! computational decomposition
+  !! @param compcount The count for the block-cyclic computational
+  !! decomposition
+  !! @param iodesc @copydoc iodesc_generate
+  !! @author Jim Edwards
+  !<
   subroutine PIO_initdecomp_bc(iosystem,basepiotype,dims,compstart,compcount,iodesc)
     type (iosystem_desc_t), intent(inout) :: iosystem
     integer(i4), intent(in)               :: basepiotype
@@ -492,26 +533,23 @@ end function PIOc_InitDecomp_bc
 
   end subroutine PIO_initdecomp_bc
 
-
-
-
-
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief A deprecated interface to the PIO_initdecomp method.
-!! @details
-!! @deprecated
-!! @param iosystem : a defined pio system descriptor, see PIO_types
-!! @param basepiotype : the type of variable(s) associated with this iodesc.
-!! @copydoc PIO_kinds
-!! @param dims : an array of the global length of each dimesion of the variable(s)
-!! @param lenblocks :
-!! @param compdof : mapping of the storage order of the variable to its memory order
-!! @param iodofr :
-!! @param iodofw :
-!! @param iodesc @copydoc iodesc_generate
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_2dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -527,10 +565,27 @@ subroutine initdecomp_2dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,io
     call initdecomp_2dof_bin_i8(iosystem,basepiotype,dims,lenblocks,int(compdof,PIO_OFFSET_KIND),int(iodofr,PIO_OFFSET_KIND), &
          int(iodofw,PIO_OFFSET_KIND),iodesc)
 
-
   end subroutine initdecomp_2dof_bin_i4
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_2dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,iodesc)
-!    use calcdisplace_mod, only : calcdisplace
+    !    use calcdisplace_mod, only : calcdisplace
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
     integer(i4), intent(in)           :: dims(:)
@@ -540,27 +595,26 @@ subroutine initdecomp_2dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,io
     integer (PIO_OFFSET_KIND), intent(in)          :: iodofw(:)     !> global degrees of freedom for io decomposition
     type (io_desc_t), intent(inout)     :: iodesc
 
-
-
-
   end subroutine initdecomp_2dof_bin_i8
 
-
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief A deprecated interface to the PIO_initdecomp method.
-!! @details
-!! @deprecated
-!! @param iosystem : a defined pio system descriptor, see PIO_types
-!! @param basepiotype : the type of variable(s) associated with this iodesc.
-!! @copydoc PIO_kinds
-!! @param dims : an array of the global length of each dimesion of the variable(s)
-!! @param lenblocks :
-!! @param compdof : mapping of the storage order of the variable to its memory order
-!! @param iodofr :
-!! @param iodesc @copydoc iodesc_generate
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_1dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -578,6 +632,24 @@ subroutine initdecomp_1dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,io
     call initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,start, count, iodesc)
   end subroutine initdecomp_1dof_bin_i8
 
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_1dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -596,24 +668,28 @@ subroutine initdecomp_1dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,io
          int(compdof,PIO_OFFSET_KIND),int(iodofr,PIO_OFFSET_KIND),start, count, iodesc)
   end subroutine initdecomp_1dof_bin_i4
 
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief A deprecated interface to the PIO_initdecomp method.
-!! @details
-!! @deprecated
-!! @param iosystem : a defined pio system descriptor, see PIO_types
-!! @param basepiotype : the type of variable(s) associated with this iodesc.
-!! @copydoc PIO_kinds
-!! @param dims : an array of the global length of each dimesion of the variable(s)
-!! @param lenblocks :
-!! @param compdof : mapping of the storage order of the variable to its memory order
-!! @param iodofr :
-!! @param iodofw :
-!! @param start : used with count to give a block description of the shape of the data
-!! @param count :
-!! @param iodesc @copydoc iodesc_generate
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims: an array of the global length of each dimesion of
+  !! the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param start used with count to give a block description of the
+  !! shape of the data
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_2dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,start, count, iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -634,6 +710,28 @@ subroutine initdecomp_2dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iod
 
   end subroutine initdecomp_2dof_nf_i4
 
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims: an array of the global length of each dimesion of
+  !! the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param start used with count to give a block description of the
+  !! shape of the data
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_2dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,start, count, iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -656,23 +754,24 @@ subroutine initdecomp_2dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iod
 
   end subroutine initdecomp_2dof_nf_i8
 
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief A deprecated interface to the PIO_initdecomp method.
-!! @details
-!! @deprecated
-!! @param iosystem : a defined PIO system descriptor, see pio_types
-!! @param basepiotype : The type of variable(s) associated with this iodesc.
-!! @copydoc PIO_kinds
-!! @param dims : an array of the global length of each dimesion of the variable(s)
-!! @param lenblocks :
-!! @param compdof : mapping of the storage order of the variable to its memory order
-!! @param iodof :
-!! @param start :
-!! @param count :
-!! @param iodesc @copydoc iodesc_generate
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined PIO system descriptor, see pio_types
+  !! @param basepiotype The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodof
+  !! @param start
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_1dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodof,start, count, iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -689,6 +788,24 @@ subroutine initdecomp_1dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iod
 
   end subroutine initdecomp_1dof_nf_i4
 
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined PIO system descriptor, see pio_types
+  !! @param basepiotype The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodof
+  !! @param start
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
   subroutine initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodof,start, count, iodesc)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -700,9 +817,6 @@ subroutine initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iod
     integer :: piotype
     integer(PIO_OFFSET_KIND), intent(in) :: start(:), count(:)
 
-
-
-
     if(any(iodof/=compdof)) then
        call piodie( __PIO_FILE__,__LINE__, &
             'Not sure what to do here')
@@ -710,31 +824,40 @@ subroutine initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iod
        call PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc,PIO_REARR_SUBSET, start,count)
     endif
 
-
   end subroutine initdecomp_1dof_nf_i8
 
-!>
-!! @public
-!! @ingroup PIO_initdecomp
-!! @brief Implements the @ref decomp_dof for PIO_initdecomp (previous name: \b initdecomp_1dof_nf_box)
-!! @details  This provides the ability to describe a computational
-!! decomposition in PIO using degrees of freedom method. This is
-!! a decomposition that can not be easily described using a start
-!! and count method (see @ref decomp_dof).
-!! Optional parameters for this subroutine allows for the specififcation of
-!! io decomposition using iostart and iocount arrays.  If iostart
-!! and iocount arrays are not specified by the user, and rearrangement
-!! is turned on then PIO will calculate an suitable IO decomposition.
-!! Note that this subroutine was previously called \em initdecomp_1dof_nf_box
-!! @param iosystem @copydoc iosystem_desc_t
-!! @param basepiotype @copydoc use_PIO_kinds
-!! @param dims An array of the global length of each dimesion of the variable(s)
-!! @param compdof Mapping of the storage order for the computational decomposition to its memory order
-!! @param iodesc @copydoc iodesc_generate
-!! @param iostart   The start index for the block-cyclic io decomposition
-!! @param iocount   The count for the block-cyclic io decomposition
-!<
-  subroutine PIO_initdecomp_dof_i4(iosystem,basepiotype,dims,compdof, iodesc, rearr, iostart, iocount)
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! Implements the degrees of freedom decomposition for
+  !! PIO_initdecomp(). This provides the ability to describe a
+  !! computational decomposition in PIO using degrees of freedom
+  !! method. This is a decomposition that can not be easily described
+  !! using a start and count method.
+  !!
+  !! Optional parameters for this subroutine allows for the
+  !! specififcation of io decomposition using iostart and iocount
+  !! arrays. If iostart and iocount arrays are not specified by the
+  !! user, and rearrangement is turned on then PIO will calculate an
+  !! suitable IO decomposition.
+  !!
+  !! @note This subroutine was previously called \em
+  !! initdecomp_1dof_nf_box.
+  !!
+  !! @param iosystem @copydoc iosystem_desc_t
+  !! @param basepiotype @copydoc use_PIO_kinds
+  !! @param dims An array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param compdof Mapping of the storage order for the computational
+  !! decomposition to its memory order
+  !! @param iodesc @copydoc iodesc_generate
+  !! @param rearr rearranger
+  !! @param iostart The start index for the block-cyclic io
+  !! decomposition
+  !! @param iocount The count for the block-cyclic io decomposition
+  !! @author Jim Edwards
+  !<
+  subroutine PIO_initdecomp_dof_i4(iosystem, basepiotype, dims, compdof, iodesc, rearr, iostart, iocount)
     type (iosystem_desc_t), intent(inout) :: iosystem
     integer(i4), intent(in)           :: basepiotype
     integer(i4), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
@@ -757,7 +880,6 @@ subroutine PIO_initdecomp_dof_i4(iosystem,basepiotype,dims,compdof, iodesc, rear
 
   end subroutine PIO_initdecomp_dof_i4
 
-
   subroutine PIO_initdecomp_internal(iosystem,basepiotype,dims,maplen, compdof, iodesc, rearr, iostart, iocount)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -815,16 +937,19 @@ end function PIOc_InitDecomp
             maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount))
        deallocate(cstart, ccount)
     else
-        ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, &
+       ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, &
             maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR)
     end if
 
     deallocate(cdims)
 
-
   end subroutine PIO_initdecomp_internal
 
-
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! I8 version of PIO_initdecomp_dof_i4.
+  !! @author Jim Edwards
   subroutine PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc, rearr, iostart, iocount)
     type (iosystem_desc_t), intent(in) :: iosystem
     integer(i4), intent(in)           :: basepiotype
@@ -843,29 +968,34 @@ subroutine PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc, rear
 
     call PIO_initdecomp_internal(iosystem, basepiotype, dims, maplen, compdof, iodesc, rearr, iostart,iocount)
 
-
 #ifdef TIMING
     call t_stopf("PIO:initdecomp_dof")
 #endif
 
   end subroutine PIO_initdecomp_dof_i8
 
-!>
-!! @public
-!! @ingroup PIO_init
-!! @brief initialize the pio subsystem.
-!! @details  This is a collective call.  Input parameters are read on comp_rank=0
-!!   values on other tasks are ignored.  This variation of PIO_init locates the IO tasks on a subset
-!!   of the compute tasks.
-!! @param comp_rank mpi rank of each participating task,
-!! @param comp_comm the mpi communicator which defines the collective.
-!! @param num_iotasks the number of iotasks to define.
-!! @param num_aggregator the mpi aggregator count
-!! @param stride the stride in the mpi rank between io tasks.
-!! @param rearr @copydoc PIO_rearr_method
-!! @param iosystem a derived type which can be used in subsequent pio operations (defined in PIO_types).
-!! @param base @em optional argument can be used to offset the first io task - default base is task 1.
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_init
+  !! Initialize the pio subsystem. This is a collective call. Input
+  !! parameters are read on comp_rank=0 values on other tasks are
+  !! ignored. This variation of PIO_init locates the IO tasks on a
+  !! subset of the compute tasks.
+  !!
+  !! @param comp_rank mpi rank of each participating task,
+  !! @param comp_comm the mpi communicator which defines the
+  !! collective.
+  !! @param num_iotasks the number of iotasks to define.
+  !! @param num_aggregator the mpi aggregator count
+  !! @param stride the stride in the mpi rank between io tasks.
+  !! @param rearr @copydoc PIO_rearr_method
+  !! @param iosystem a derived type which can be used in subsequent
+  !! pio operations (defined in PIO_types).
+  !! @param base @em optional argument can be used to offset the first
+  !! io task - default base is task 1.
+  !! @param rearr_opts the rearranger options.
+  !! @author Jim Edwards
+  !<
   subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
     use pio_types, only : pio_internal_error, pio_rearr_opt_t
     use iso_c_binding
@@ -892,7 +1022,7 @@ integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks,
          integer(C_INT), value :: stride
          integer(C_INT), value :: base
          integer(C_INT), value :: rearr
-          type(pio_rearr_opt_t) :: rearr_opts
+         type(pio_rearr_opt_t) :: rearr_opts
          integer(C_INT) :: iosysidp
        end function PIOc_Init_Intracomm_from_F90
     end interface
@@ -910,21 +1040,27 @@ end function PIOc_Init_Intracomm_from_F90
 #endif
   end subroutine init_intracom
 
-
-!>
-!! @public
-!! @ingroup PIO_init
-!! @brief Initialize the pio subsystem.
-!! @details  This is a collective call.  Input parameters are read on comp_rank=0
-!!   values on other tasks are ignored.  This variation of PIO_init sets up a distinct set of tasks
-!!   to handle IO, these tasks do not return from this call.  Instead they go to an internal loop
-!!   and wait to receive further instructions from the computational tasks
-!! @param component_count The number of computational components to associate with this IO component
-!! @param peer_comm  The communicator from which all other communicator arguments are derived
-!! @param comp_comms The computational communicator for each of the computational components
-!! @param io_comm    The io communicator
-!! @param iosystem a derived type which can be used in subsequent pio operations (defined in PIO_types).
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_init
+  !! Initialize the pio subsystem. This is a collective call. Input
+  !! parameters are read on comp_rank=0 values on other tasks are
+  !! ignored. This variation of PIO_init sets up a distinct set of
+  !! tasks to handle IO, these tasks do not return from this
+  !! call. Instead they go to an internal loop and wait to receive
+  !! further instructions from the computational tasks.
+  !!
+  !! @param component_count The number of computational components to
+  !! associate with this IO component.
+  !! @param peer_comm The communicator from which all other
+  !! communicator arguments are derived.
+  !! @param comp_comms The computational communicator for each of the
+  !! computational components.
+  !! @param io_comm The io communicator.
+  !! @param iosystem a derived type which can be used in subsequent
+  !! pio operations (defined in PIO_types).
+  !! @author Jim Edwards
+  !<
   subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosystem)
     use pio_types, only : pio_internal_error, pio_rearr_box
     integer, intent(in) :: component_count
@@ -948,7 +1084,7 @@ subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosyst
 #endif
 #if defined(NO_MPI2) || defined(_MPISERIAL)
     call piodie( __PIO_FILE__,__LINE__, &
-     'The PIO async interface requires an MPI2 complient MPI library')
+         'The PIO async interface requires an MPI2 complient MPI library')
 #else
     do i=1,component_count
        iosystem(i)%error_handling = PIO_internal_error
@@ -1102,16 +1238,16 @@ subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosyst
        call mpi_info_create(iosystem(i)%info,ierr)
        ! turn on mpi-io aggregation
        !DBG    print *,'PIO_init: before call to setnumagg'
-!       itmp = num_aggregator
-!       call mpi_bcast(itmp, 1, mpi_integer, 0, iosystem%union_comm, ierr)
-!       if(itmp .gt. 0) then
-!          write(cb_nodes,('(i5)')) itmp
-!#ifdef BGQ
-!          call PIO_set_hint(iosystem(i),"bgl_nodes_pset",trim(adjustl(cb_nodes)))
-!#else
-!          call PIO_set_hint(iosystem(i),"cb_nodes",trim(adjustl(cb_nodes)))
-!#endif
-!       endif
+       !       itmp = num_aggregator
+       !       call mpi_bcast(itmp, 1, mpi_integer, 0, iosystem%union_comm, ierr)
+       !       if(itmp .gt. 0) then
+       !          write(cb_nodes,('(i5)')) itmp
+       !#ifdef BGQ
+       !          call PIO_set_hint(iosystem(i),"bgl_nodes_pset",trim(adjustl(cb_nodes)))
+       !#else
+       !          call PIO_set_hint(iosystem(i),"cb_nodes",trim(adjustl(cb_nodes)))
+       !#endif
+       !       endif
 
 #ifdef PIO_GPFS_HINTS
        call PIO_set_hint(iosystem(i),"ibm_largeblock_io","true")
@@ -1140,17 +1276,17 @@ subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosyst
 #endif
   end subroutine init_intercom
 
-
-!>
-!! @public
-!! @defgroup PIO_set_hint  PIO_set_hint
-!! @brief set file system hints using mpi_info_set
-!! @details This is a collective call which expects the following parameters:
-!! @param iosystem @copydoc io_desc_t
-!! @param hint  the string name of the hint to define
-!! @param hintval  the string value to set the hint to
-!! @retval ierr @copydoc  error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_set_hint
+  !! Set file system hints using mpi_info_set. This is a collective
+  !! call.
+  !!
+  !! @param iosystem @copydoc io_desc_t
+  !! @param hint the string name of the hint to define
+  !! @param hintval the string value to set the hint to
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
   subroutine PIO_set_hint(iosystem, hint, hintval)
     type (iosystem_desc_t), intent(inout)  :: iosystem  ! io descriptor to initalize
     character(len=*), intent(in) :: hint, hintval
@@ -1166,24 +1302,22 @@ integer(C_INT) function PIOc_set_hint(iosysid, key, val) &
        end function PIOc_set_hint
     end interface
 
-
     ierr = PIOc_set_hint(iosystem%iosysid, hint, hintval)
 
-
   end subroutine PIO_set_hint
 
-
-!>
-!! @public
-!! @ingroup PIO_finalize
-!! @brief finalizes the pio subsystem.
-!! @details This is a collective call which expects the following parameters
-!! @param iosystem : @copydoc io_desc_t
-!! @retval ierr @copydoc  error_return
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_finalize
+  !! Finalizes an IO System. This is a collective call.
+  !!
+  !! @param iosystem @copydoc io_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   subroutine finalize(iosystem,ierr)
-     type (iosystem_desc_t), intent(inout) :: iosystem
-     integer(i4), intent(out) :: ierr
+    type (iosystem_desc_t), intent(inout) :: iosystem
+    integer(i4), intent(out) :: ierr
     interface
        integer(C_INT) function PIOc_finalize(iosysid) &
             bind(C,name="PIOc_finalize")
@@ -1196,60 +1330,61 @@ end function PIOc_finalize
     endif
   end subroutine finalize
 
+  !>
+  !! @public
+  !! @ingroup PIO_getnumiotasks
+  !! Return the number of IO-tasks that PIO is using.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param numiotasks the number of IO-tasks
+  !! @author Jim Edwards
+  !<
+  subroutine getnumiotasks(iosystem,numiotasks)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(out) :: numiotasks
+    integer :: ierr
+    interface
+       integer(C_INT) function PIOc_get_numiotasks(iosysid,numiotasks) &
+            bind(C,name="PIOc_get_numiotasks")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+         integer(C_INT), intent(out) :: numiotasks
+       end function PIOc_get_numiotasks
+    end interface
+    ierr = PIOc_get_numiotasks(iosystem%iosysid, numiotasks)
 
-!>
-!! @public
-!! @ingroup PIO_getnumiotasks
-!! @brief This returns the number of IO-tasks that PIO is using
-!! @param iosystem : a defined pio system descriptor, see PIO_types
-!! @param numiotasks : the number of IO-tasks
-!<
-   subroutine getnumiotasks(iosystem,numiotasks)
-       type (iosystem_desc_t), intent(in) :: iosystem
-       integer(i4), intent(out) :: numiotasks
-       integer :: ierr
-       interface
-          integer(C_INT) function PIOc_get_numiotasks(iosysid,numiotasks) &
-               bind(C,name="PIOc_get_numiotasks")
-            use iso_c_binding
-            integer(C_INT), intent(in), value :: iosysid
-            integer(C_INT), intent(out) :: numiotasks
-          end function PIOc_get_numiotasks
-       end interface
-       ierr = PIOc_get_numiotasks(iosystem%iosysid, numiotasks)
-
-   end subroutine getnumiotasks
-
-   logical function pio_iotype_available( iotype) result(available)
-     integer, intent(in) :: iotype
-     interface
-        integer(C_INT) function PIOc_iotype_available(iotype) &
-             bind(C,name="PIOc_iotype_available")
-          use iso_c_binding
-          integer(C_INT), intent(in), value :: iotype
-        end function PIOc_iotype_available
-     end interface
-     available= (PIOc_iotype_available(iotype) == 1)
-
-   end function pio_iotype_available
-
+  end subroutine getnumiotasks
 
-!>
-!! @public
-!! @ingroup PIO_createfile
-!! @brief  Create a NetCDF or PNetCDF file using PIO.
-!! @details  Input parameters are read on comp task 0 and ignored elsewhere
-!! @param iosystem : A defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file  :  The returned file descriptor
-!! @param iotype : @copydoc PIO_iotype
-!! @param fname : The name of the file to open
-!! @param amode_in : The NetCDF creation mode flag. the following flags are available:
-!! (1) zero value or NC_NOWRITE is default and opens the file with read-only access.
-!! (2) NC_WRITE for read-write access.
-!! (3) NC_SHARE is used for NetCDF classic, and dangerous with this application.
-!! (4) NC_WRITE|NC_SHARE
-!! @retval ierr @copydoc error_return
-!<
+  !> Is an iotype available?
+  logical function pio_iotype_available( iotype) result(available)
+    integer, intent(in) :: iotype
+    interface
+       integer(C_INT) function PIOc_iotype_available(iotype) &
+            bind(C,name="PIOc_iotype_available")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iotype
+       end function PIOc_iotype_available
+    end interface
+    available= (PIOc_iotype_available(iotype) == 1)
+
+  end function pio_iotype_available
+
+  !>
+  !! @public
+  !! @ingroup PIO_createfile
+  !! Create a NetCDF file using PIO. Input parameters are read on
+  !! comp task 0 and ignored elsewhere.
+  !!
+  !! @param iosystem A defined PIO system descriptor created by a
+  !! call to @ref PIO_init (see PIO_init)
+  !! @param file The returned file descriptor
+  !! @param iotype @copydoc PIO_iotype
+  !! @param fname The name of the file to open
+  !! @param amode_in The NetCDF creation mode flag - NC_NOWRITE for
+  !! read-only access or NC_WRITE for read-write access.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function createfile(iosystem, file,iotype, fname, amode_in) result(ierr)
     type (iosystem_desc_t), intent(inout), target :: iosystem
     type (file_desc_t), intent(out) :: file
@@ -1259,7 +1394,7 @@ integer function createfile(iosystem, file,iotype, fname, amode_in) result(ierr)
     integer :: mode
     interface
        integer(C_INT) function PIOc_createfile(iosysid, fh, iotype, fname,mode) &
-         bind(C,NAME='PIOc_createfile')
+            bind(C,NAME='PIOc_createfile')
          use iso_c_binding
          implicit none
          integer(c_int), value :: iosysid
@@ -1289,27 +1424,25 @@ end function PIOc_createfile
     call t_stopf("PIO:createfile")
 #endif
   end function createfile
-!>
-!! @public
-!! @ingroup PIO_openfile
-!! @brief open an existing file using pio
-!! @details  Input parameters are read on comp task 0 and ignored elsewhere.
-!! @param iosystem : a defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file  :  the returned file descriptor
-!! @param iotype : @copybrief PIO_iotype
-!! @param fname : the name of the file to open
-!! @param mode : a zero value (or PIO_nowrite) specifies the default
-!! behavior: open the dataset with read-only access, buffering and
-!! caching accesses for efficiency otherwise, the creation mode is
-!! PIO_write. setting the PIO_write flag opens the dataset with
-!! read-write access. ("writing" means any kind of change to the dataset,
-!! including appending or changing data, adding or renaming dimensions,
-!! variables, and attributes, or deleting attributes.)
-!! @retval ierr @copydoc error_return
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_openfile
+  !! Open an existing file using PIO. Input parameters are read on
+  !! comp task 0 and ignored elsewhere.
+  !!
+  !! @param iosystem a defined PIO system descriptor created by a call
+  !! to @ref PIO_init (see PIO_int)
+  !! @param file the returned file descriptor
+  !! @param iotype @copybrief PIO_iotype
+  !! @param fname the name of the file to open
+  !! @param mode PIO_nowrite or PIO_write.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   integer function PIO_openfile(iosystem, file, iotype, fname,mode) result(ierr)
 
-!    use ifcore, only: tracebackqq
+    !    use ifcore, only: tracebackqq
     type (iosystem_desc_t), intent(inout), target :: iosystem
     type (file_desc_t), intent(out) :: file
     integer, intent(in) :: iotype
@@ -1318,7 +1451,7 @@ integer function PIO_openfile(iosystem, file, iotype, fname,mode) result(ierr)
     integer :: iorank
     interface
        integer(C_INT) function PIOc_openfile(iosysid, fh, iotype, fname,mode) &
-         bind(C,NAME='PIOc_openfile')
+            bind(C,NAME='PIOc_openfile')
          use iso_c_binding
          implicit none
          integer(c_int), value :: iosysid
@@ -1349,13 +1482,16 @@ end function PIOc_openfile
 #endif
   end function PIO_openfile
 
-!>
-!! @public
-!! @ingroup PIO_syncfile
-!! @brief synchronizing a file forces all writes to complete before the subroutine returns.
-!!
-!! @param file @copydoc file_desc_t
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_syncfile
+  !! Synchronizing a file, forcing all writes to complete before the
+  !! subroutine returns.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
   subroutine syncfile(file)
     implicit none
     type (file_desc_t), target :: file
@@ -1371,14 +1507,17 @@ end function PIOc_sync
     ierr = PIOc_sync(file%fh)
 
   end subroutine syncfile
-!>
-!! @public
-!! @ingroup PIO_freedecomp
-!! @brief free all allocated storage associated with this decomposition
-!! @details
-!! @param ios :  a defined pio system descriptor created by call to @ref PIO_init (see PIO_types)
-!! @param iodesc @copydoc io_desc_t
-!<
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_freedecomp
+  !! @brief free all allocated storage associated with this decomposition
+  !! @details
+  !! @param ios :  a defined pio system descriptor created by call to @ref PIO_init (see PIO_types)
+  !! @param iodesc @copydoc io_desc_t
+  !! @author Jim Edwards
+  !<
   subroutine freedecomp_ios(ios,iodesc)
     implicit none
     type (iosystem_desc_t) :: ios
@@ -1395,15 +1534,18 @@ end function PIOc_freedecomp
     ierr = PIOc_freedecomp(ios%iosysid, iodesc%ioid)
 
   end subroutine freedecomp_ios
-!>
-!! @public
-!! @ingroup PIO_freedecomp
-!! @brief free all allocated storage associated with this decomposition
-!! @details
-!! @param file @copydoc file_desc_t
-!! @param iodesc : @copydoc io_desc_t
-!! @retval ierr @copydoc error_return
-!<
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_freedecomp
+  !! Free all allocated storage associated with this decomposition.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @param iodesc : @copydoc io_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
   subroutine freedecomp_file(file,iodesc)
     implicit none
     type (file_desc_t) :: file
@@ -1415,13 +1557,15 @@ subroutine freedecomp_file(file,iodesc)
 
   end subroutine freedecomp_file
 
-!>
-!! @public
-!! @ingroup PIO_closefile
-!! @brief close a disk file
-!! @details
-!! @param file @copydoc file_desc_t
-!<
+
+  !>
+  !! @public
+  !! @ingroup PIO_closefile
+  !! Close a disk file.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
   subroutine closefile(file)
     type(file_desc_t) :: file
     integer :: ierr
@@ -1444,55 +1588,15 @@ end function PIOc_closefile
   end subroutine closefile
 
 
-  !******************************
-  ! read_ascii
-  !
-
-  subroutine read_ascii(rank,iobuf,size)
-
-    integer, intent(in) :: rank
-    real (r8), dimension(:) :: iobuf
-    integer, intent(in) :: size
-
-    character(len=80) filename
-    integer lun
-    integer ios
-    integer i
-
-    lun=10+rank
-    write(filename,"('fort.',i2)" ) lun
-    write(6,*) 'filename is:', filename
-
-    open(lun,file=filename,status='old',iostat=ios)
-    if (ios /= 0) then
-       write(6,*) rank,': could not open ascii file: ',filename
-    endif
-
-    do i=1,size
-       read(unit=lun,fmt=*,iostat=ios) iobuf(i)
-       if (ios /= 0) then
-          write (6,*) rank,': error reading item ',i,' of ',size
-#ifndef CPRNAG
-          call abort
-#else
-          stop
-#endif
-       endif
-
-    end do
-
-    close(lun)
-
-  end subroutine read_ascii
-
-!>
-!! @public
-!! @ingroup PIO_deletefile
-!! @brief Delete a file
-!! @details
-!! @param ios : a pio system handle
-!! @param fname : a filename
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_deletefile
+  !! Delete a file.
+  !!
+  !! @param ios a pio system handle
+  !! @param fname a filename
+  !! @author Jim Edwards
+  !<
   subroutine pio_deletefile(ios, fname)
     type(iosystem_desc_t) :: ios
     character(len=*) :: fname
@@ -1510,27 +1614,30 @@ end function PIOc_deletefile
 
   end subroutine pio_deletefile
 
-!>
-!! @public
-!! @ingroup PIO_set_rearr_opts
-!! @brief Set the rerranger options
-!! @details
-!! @param ios : handle to pio iosystem
-!! @param comm_type : @copydoc PIO_rearr_comm_t
-!! @param fcd : @copydoc PIO_rearr_comm_dir
-!! @param enable_hs_c2i : Enable handshake (compute procs to io procs)
-!! @param enable_isend_c2i : Enable isends (compute procs to io procs)
-!! @param max_pend_req_c2i: Maximum pending requests (compute procs to io procs)
-!! @param enable_hs_i2c : Enable handshake (io procs to compute procs)
-!! @param enable_isend_i2c : Enable isends (io procs to compute procs)
-!! @param max_pend_req_i2c: Maximum pending requests (io procs to compute procs)
-!! @copydoc PIO_rearr_comm_fc_options
-!<
+  !>
+  !! @public
+  !! @ingroup PIO_set_rearr_opts
+  !! Set the rerranger options.
+  !!
+  !! @param ios handle to pio iosystem
+  !! @param comm_type @copydoc PIO_rearr_comm_t
+  !! @param fcd : @copydoc PIO_rearr_comm_dir
+  !! @param enable_hs_c2i Enable handshake (compute procs to io procs)
+  !! @param enable_isend_c2i Enable isends (compute procs to io procs)
+  !! @param max_pend_req_c2i Maximum pending requests (compute procs
+  !! to io procs)
+  !! @param enable_hs_i2c Enable handshake (io procs to compute procs)
+  !! @param enable_isend_i2c Enable isends (io procs to compute procs)
+  !! @param max_pend_req_i2c Maximum pending requests (io procs to
+  !! compute procs)
+  !! @copydoc PIO_rearr_comm_fc_options
+  !! @author Jim Edwards
+  !<
   function pio_set_rearr_opts(ios, comm_type, fcd,&
-                              enable_hs_c2i, enable_isend_c2i,&
-                              max_pend_req_c2i,&
-                              enable_hs_i2c, enable_isend_i2c,&
-                              max_pend_req_i2c) result(ierr)
+       enable_hs_c2i, enable_isend_c2i,&
+       max_pend_req_c2i,&
+       enable_hs_i2c, enable_isend_i2c,&
+       max_pend_req_i2c) result(ierr)
 
     type(iosystem_desc_t), intent(inout) :: ios
     integer, intent(in) :: comm_type, fcd
@@ -1539,36 +1646,33 @@ function pio_set_rearr_opts(ios, comm_type, fcd,&
     integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c
     integer :: ierr
     interface
-      integer(c_int) function PIOc_set_rearr_opts(iosysid, comm_type, fcd,&
-                                                  enable_hs_c2i, enable_isend_c2i,&
-                                                  max_pend_req_c2i,&
-                                                  enable_hs_i2c, enable_isend_i2c,&
-                                                  max_pend_req_i2c)&
-        bind(C,name="PIOc_set_rearr_opts")
-        use iso_c_binding
-        integer(C_INT), intent(in), value :: iosysid
-        integer(C_INT), intent(in), value :: comm_type
-        integer(C_INT), intent(in), value :: fcd
-        logical(C_BOOL), intent(in), value :: enable_hs_c2i
-        logical(C_BOOL), intent(in), value :: enable_isend_c2i
-        integer(C_INT), intent(in), value :: max_pend_req_c2i
-        logical(C_BOOL), intent(in), value :: enable_hs_i2c
-        logical(C_BOOL), intent(in), value :: enable_isend_i2c
-        integer(C_INT), intent(in), value :: max_pend_req_i2c
-      end function PIOc_set_rearr_opts
+       integer(c_int) function PIOc_set_rearr_opts(iosysid, comm_type, fcd,&
+            enable_hs_c2i, enable_isend_c2i,&
+            max_pend_req_c2i,&
+            enable_hs_i2c, enable_isend_i2c,&
+            max_pend_req_i2c)&
+            bind(C,name="PIOc_set_rearr_opts")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+         integer(C_INT), intent(in), value :: comm_type
+         integer(C_INT), intent(in), value :: fcd
+         logical(C_BOOL), intent(in), value :: enable_hs_c2i
+         logical(C_BOOL), intent(in), value :: enable_isend_c2i
+         integer(C_INT), intent(in), value :: max_pend_req_c2i
+         logical(C_BOOL), intent(in), value :: enable_hs_i2c
+         logical(C_BOOL), intent(in), value :: enable_isend_i2c
+         integer(C_INT), intent(in), value :: max_pend_req_i2c
+       end function PIOc_set_rearr_opts
     end interface
 
     ierr = PIOc_set_rearr_opts(ios%iosysid, comm_type, fcd,&
-                                logical(enable_hs_c2i, kind=c_bool),&
-                                logical(enable_isend_c2i, kind=c_bool),&
-                                max_pend_req_c2i,&
-                                logical(enable_hs_i2c, kind=c_bool),&
-                                logical(enable_isend_i2c, kind=c_bool),&
-                                max_pend_req_i2c)
+         logical(enable_hs_c2i, kind=c_bool),&
+         logical(enable_isend_c2i, kind=c_bool),&
+         max_pend_req_c2i,&
+         logical(enable_hs_i2c, kind=c_bool),&
+         logical(enable_isend_i2c, kind=c_bool),&
+         max_pend_req_i2c)
 
   end function pio_set_rearr_opts
 
-
 end module piolib_mod
-
-  !|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
diff --git a/src/flib/pionfatt_mod.F90.in b/src/flib/pionfatt_mod.F90.in
index daaca6dca81e..74b76082ab0c 100644
--- a/src/flib/pionfatt_mod.F90.in
+++ b/src/flib/pionfatt_mod.F90.in
@@ -3,6 +3,15 @@
 !! @file
 !! @brief NetCDF attribute interface to PIO
 !<
+!>
+!!  @defgroup PIO_put_att Write Attributes
+!!  Writes an attribute to a file in Fortran.
+!<
+!>
+!! @defgroup PIO_get_att Read Attributes
+!! Reads an attribute from a file in Fortran.
+!<
+
 module pionfatt_mod
   use pio_kinds, only : r4, r8, i4, pio_offset_kind
   use pio_types
@@ -36,21 +45,10 @@ module pionfatt_mod
      module procedure get_att_1d_{TYPE}, get_att_desc_1d_{TYPE}, get_att_1d_id_{TYPE}
   end interface
 
-  !>
-  !!  @public
-  !!  @defgroup PIO_put_att PIO_put_att
-  !!  @brief Writes an netcdf attribute to a file
-  !<
-  !>
-  !! @public
-  !! @defgroup PIO_get_att PIO_get_att
-  !! @brief Reads  an netcdf attribute from a file
-  !<
-
+  !> @cond doxygen_cant_handle_so_exclude
   private :: modName
   character(len=*), parameter :: modName='pionfatt_mod'
   interface
-     !> @brief Write a netCDF text attribute.
      integer(C_INT) function PIOc_put_att_text (ncid, varid, name, len, op) &
           bind(C,name="PIOc_put_att_text")
        use iso_c_binding
@@ -143,7 +141,7 @@ module pionfatt_mod
        real(C_DOUBLE), intent(out)   :: op
      end function PIOc_get_att_double
   end interface
-
+  !> @endcond
 
 contains
 
@@ -153,9 +151,9 @@ contains
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
-  !! @param varid : The netcdf variable identifier
+  !! @param vdesc : The netcdf variable descriptor
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_desc_{TYPE} (File, vdesc, name, values) result(ierr)
@@ -271,7 +269,7 @@ contains
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
-  !! @param File @copydoc file_desc_t
+  !! @param ncid @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to add
   !! @param values : The value for the netcdf attribute
@@ -312,7 +310,7 @@ contains
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_1d_desc_{TYPE} (File,varDesc,name,values) result(ierr)
@@ -337,7 +335,6 @@ contains
 
   end function put_att_1d_vid_{TYPE}
 
-
   !>
   !! @public
   !! @ingroup PIO_get_att
@@ -390,7 +387,7 @@ contains
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
-  !! @param File @copydoc file_desc_t
+  !! @param ncid : The netcdf file ID
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to get
   !! @param values : The value for the netcdf attribute
diff --git a/src/flib/pionfget_mod.F90.in b/src/flib/pionfget_mod.F90.in
index ebd335dde2ed..17fe764b951d 100644
--- a/src/flib/pionfget_mod.F90.in
+++ b/src/flib/pionfget_mod.F90.in
@@ -3,6 +3,13 @@
 !! @file
 !! @brief Read Routines for non-decomposed NetCDF data.
 !<
+!>
+!! @defgroup PIO_get_var Read Non-Decomposed Data
+!! Reads non-decomposed data from a NetCDF file in Fortran.  The
+!! get_var interface is provided as a simplified interface to read
+!! variables from a NetCDF format file. The variable is read on the
+!! root IO task and broadcast in its entirety to all tasks.
+!<
 module pionfget_mod
   use iso_c_binding
 #ifdef TIMING
@@ -13,13 +20,6 @@ module pionfget_mod
   use pio_support, only : replace_c_null
   implicit none
   private
-!>
-!! @defgroup PIO_get_var PIO_get_var
-!! @brief Reads non-decomposed data from a NetCDF file
-!! @details The get_var interface is provided as a simplified interface to
-!!  read variables from a NetCDF format file.   The variable is read on the
-!!  root IO task and broadcast in its entirety to all tasks.
-!<
   public :: get_var
   interface get_var
      module procedure get_var_{DIMS}d_{TYPE}, get_var_vdesc_{DIMS}d_{TYPE}
@@ -450,7 +450,7 @@ CONTAINS
     integer, intent(in) :: start(:)
     integer, intent(in) :: count(:)
     integer, intent(in) :: nstrs
-    character, intent(out) :: ival(*)
+    character(len=*), intent(out) :: ival(*)
     integer :: j
     integer(C_SIZE_T), allocatable :: cstart(:), ccount(:)
     integer :: i, ndims
diff --git a/src/flib/pionfput_mod.F90.in b/src/flib/pionfput_mod.F90.in
index d4b54220a941..1ea42af743cc 100644
--- a/src/flib/pionfput_mod.F90.in
+++ b/src/flib/pionfput_mod.F90.in
@@ -3,6 +3,12 @@
 !! @file
 !! @brief Write routines for non-decomposed NetCDF data.
 !<
+!>
+!! @defgroup PIO_put_var Write Variable
+!! Writes data to a variable.
+!! @warning Although this is a collective call the variable is written from the
+!! root IO task, no consistancy check is made with data passed on other tasks.
+!<
 module pionfput_mod
 #ifdef TIMING
   use perf_mod, only : t_startf, t_stopf      ! _EXTERNAL
@@ -14,15 +20,6 @@ module pionfput_mod
   implicit none
   private
 
-!>
-!! @defgroup PIO_put_var PIO_put_var
-!! @brief Writes data to a netCDF file.
-!! @details The put_var interface is provided as a simplified interface to
-!!  write variables to a netcdf format file.
-!! @warning Although this is a collective call the variable is written from the
-!!  root IO task, no consistancy check is made with data passed on other tasks.
-!!
-!<
   public :: put_var
   interface put_var
      !  DIMS 0,1,2,3,4,5
@@ -178,7 +175,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param start :
+!! @param index :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -266,8 +263,7 @@ contains
 !! @ingroup PIO_put_var
 !! @brief Writes {TYPE} data to a netCDF variable.
 !! @details
-!! @param File @copydoc file_desc_t
-!! @param File : A file handle returne from \ref PIO_openfile or \ref PIO_createfile.
+!! @param ncid : The netcdf file id.
 !! @param varid : The netcdf variable identifier
 !! @param ival : The data to write.
 !! @retval ierr @copydoc error_return
diff --git a/src/gptl/Makefile.am b/src/gptl/Makefile.am
new file mode 100644
index 000000000000..54216d4ffa99
--- /dev/null
+++ b/src/gptl/Makefile.am
@@ -0,0 +1,23 @@
+# This is part of PIO. It creates the Makefile for the GPTL directory.
+
+# Ed Hartnett 4/9/19
+
+# Turn off parallel builds in this directory.
+.NOTPARALLEL:
+
+# Build these uninstalled convenience libraries.
+noinst_LTLIBRARIES = libperf_utils.la libperf_mod.la
+
+# The convenience libraries depends on their source.
+libperf_utils_la_SOURCES = perf_utils.F90
+libperf_mod_la_SOURCES = perf_mod.F90 f_wrappers_2.c private.h
+
+# Each mod file depends on the .o file.
+perf_utils.mod: perf_utils.$(OBJEXT)
+perf_mod.mod: perf_mod.$(OBJEXT)
+
+# Does the user want to build fortran?
+#if BUILD_FORTRAN
+#endif
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/src/gptl/f_wrappers_2.c b/src/gptl/f_wrappers_2.c
new file mode 100644
index 000000000000..1096573a0b7a
--- /dev/null
+++ b/src/gptl/f_wrappers_2.c
@@ -0,0 +1,185 @@
+/*
+** Fortran wrappers for timing library routines that are not in GPTL.
+* Ed Hartnett 4/9/19
+*/
+
+#include 
+#include 
+#include "private.h" /* MAX_CHARS, bool */
+#include "gptl.h"    /* function prototypes and HAVE_MPI logic*/
+#ifdef HAVE_PAPI
+#include 
+#endif /* HAVE_PAPI */
+
+#define gptlevent_name_to_code gptlevent_name_to_code_
+#define gptlevent_code_to_name gptlevent_code_to_name_
+#define gptlpr_set_append gptlpr_set_append_
+#define gptlpr_query_append gptlpr_query_append_
+#define gptlpr_set_write gptlpr_set_write_
+#define gptlpr_query_write gptlpr_query_write_
+
+/*
+** Local function prototypes
+*/
+
+int gptlpr_set_append (void);
+int gptlpr_query_append (void);
+int gptlpr_set_write (void);
+int gptlpr_query_write (void);
+static int pr_append;
+
+#ifdef HAVE_PAPI
+/* int gptl_papilibraryinit (void); */
+int gptlevent_name_to_code (const char *str, int *code, int nc);
+int gptlevent_code_to_name (int *code, char *str, int nc);
+
+/** GPTL_PAPIlibraryinit: Call PAPI_library_init if necessary
+ **
+ ** Return value: 0 (success) or GPTLerror (failure)
+ */
+
+int GPTL_PAPIlibraryinit ()
+{
+    int ret;
+
+    if ((ret = PAPI_is_initialized ()) == PAPI_NOT_INITED) {
+        if ((ret = PAPI_library_init (PAPI_VER_CURRENT)) != PAPI_VER_CURRENT) {
+            fprintf (stderr, "GPTL_PAPIlibraryinit: ret=%d PAPI_VER_CURRENT=%d\n",
+                     ret, (int) PAPI_VER_CURRENT);
+            return GPTLerror ("GPTL_PAPIlibraryinit: PAPI_library_init failure:%s\n",
+                              PAPI_strerror (ret));
+        }
+    }
+    return 0;
+}
+
+#endif
+
+/*
+** GPTLpr_set_append: set GPTLpr_file and GPTLpr_summary_file
+** to use append mode
+*/
+
+int GPTLpr_set_append (void)
+{
+    pr_append = true;
+    return 0;
+}
+
+/*
+** GPTLpr_query_append: query whether GPTLpr_file and GPTLpr_summary_file
+** use append mode
+*/
+
+int GPTLpr_query_append (void)
+{
+    if (pr_append)
+        return 1;
+    else
+        return 0;
+}
+
+/*
+** GPTLpr_set_write: set GPTLpr_file and GPTLpr_summary_file
+** to use write mode
+*/
+
+int GPTLpr_set_write (void)
+{
+    pr_append = false;
+    return 0;
+}
+
+/*
+** GPTLpr_query_write: query whether GPTLpr_file and GPTLpr_summary_file
+** use write mode
+*/
+
+int GPTLpr_query_write (void)
+{
+    if (pr_append)
+        return 0;
+    else
+        return 1;
+}
+
+
+/*
+** Fortran wrapper functions start here
+*/
+
+int gptlpr_set_append (void)
+{
+    return GPTLpr_set_append ();
+}
+
+int gptlpr_query_append (void)
+{
+    return GPTLpr_set_append ();
+}
+
+int gptlpr_set_write (void)
+{
+    return GPTLpr_set_append ();
+}
+
+int gptlpr_query_write (void)
+{
+    return GPTLpr_set_append ();
+}
+
+#ifdef HAVE_PAPI
+
+int gptl_papilibraryinit (void)
+{
+    return GPTL_PAPIlibraryinit ();
+}
+
+int gptlevent_name_to_code (const char *str, int *code, int nc)
+{
+    char cname[PAPI_MAX_STR_LEN+1];
+    int numchars = MIN (nc, PAPI_MAX_STR_LEN);
+
+    strncpy (cname, str, numchars);
+    cname[numchars] = '\0';
+
+    /* "code" is an int* and is an output variable */
+
+    return GPTLevent_name_to_code (cname, code);
+}
+
+int gptlevent_code_to_name (int *code, char *str, int nc)
+{
+
+    if (nc < PAPI_MAX_STR_LEN)
+        return GPTLerror ("gptl_event_code_to_name: output name must hold at least %d characters\n",
+                          PAPI_MAX_STR_LEN);
+
+    if (GPTLevent_code_to_name (*code, str) == 0) {
+        int i;
+        for (i = strlen(str); i < nc; ++i)
+            str[i] = ' ';
+    } else {
+        return GPTLerror ("");
+    }
+    return 0;
+}
+
+#else
+
+int gptl_papilibraryinit (void)
+{
+    return 0;
+}
+
+int gptlevent_name_to_code (const char *str, int *code, int nc)
+{
+    return GPTLevent_name_to_code (str, code);
+}
+
+int gptlevent_code_to_name (const int *code, char *str, int nc)
+{
+    return GPTLevent_code_to_name (*code, str);
+}
+
+#endif
diff --git a/src/gptl/perf_mod.F90 b/src/gptl/perf_mod.F90
index e62059de98ea..622d5cf2c120 100644
--- a/src/gptl/perf_mod.F90
+++ b/src/gptl/perf_mod.F90
@@ -913,12 +913,12 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
    unitn = shr_file_getUnit()
 
    ! determine what the current output mode is (append or write)
-   if (GPTLpr_query_write() == 1) then
-     pr_write = .true.
-     ierr = GPTLpr_set_append()
-   else 
-     pr_write=.false.
-   endif
+   ! if (GPTLpr_query_write() == 1) then
+   !   pr_write = .true.
+   !   ierr = GPTLpr_set_append()
+   ! else
+   pr_write=.false.
+   ! endif
 
    ! Determine whether to write all data to a single fie
    if (present(single_file)) then
@@ -1100,9 +1100,9 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
    call shr_file_freeUnit( unitn )
 
    ! reset GPTL output mode
-   if (pr_write) then
-     ierr = GPTLpr_set_write()
-   endif
+   ! if (pr_write) then
+   !   ierr = GPTLpr_set_write()
+   ! endif
 
 !$OMP END MASTER
    call t_stopf("t_prf")
diff --git a/tests/Makefile.am b/tests/Makefile.am
index b2c4a95e64d3..a643bbcd9760 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1 +1,18 @@
-SUBDIRS = cunit
+# This file is part of PIO. It generates the Makefiles for the tests
+# directory.
+
+# Ed Hartnett
+
+# Does the user want to build fortran? If so, there are two additional
+# test directories.
+if BUILD_FORTRAN
+UNIT = unit
+GENERAL = general
+if USE_GPTL
+PERFORMANCE = performance
+endif
+endif
+
+SUBDIRS = cunit ${UNIT} ${GENERAL} ${PERFORMANCE}
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/tests/cunit/CMakeLists.txt b/tests/cunit/CMakeLists.txt
index 21553eb66452..64b6a20b7275 100644
--- a/tests/cunit/CMakeLists.txt
+++ b/tests/cunit/CMakeLists.txt
@@ -88,7 +88,13 @@ if (NOT PIO_USE_MPISERIAL)
   target_link_libraries (test_decomps pioc)
   add_executable (test_rearr EXCLUDE_FROM_ALL test_rearr.c test_common.c)
   target_link_libraries (test_rearr pioc)
+  add_executable (test_darray_fill EXCLUDE_FROM_ALL test_darray_fill.c test_common.c)
+  target_link_libraries (test_darray_fill pioc)
+  add_executable (test_decomp_frame EXCLUDE_FROM_ALL test_decomp_frame.c test_common.c)
+  target_link_libraries (test_decomp_frame pioc)
   if (PIO_USE_MALLOC)
+    add_executable (test_perf2 EXCLUDE_FROM_ALL test_perf2.c test_common.c)
+    target_link_libraries (test_perf2)
     add_executable (test_darray_async_simple EXCLUDE_FROM_ALL test_darray_async_simple.c test_common.c)
     target_link_libraries (test_darray_async_simple pioc)
     add_executable (test_darray_async EXCLUDE_FROM_ALL test_darray_async.c test_common.c)
@@ -123,7 +129,10 @@ add_dependencies (tests test_darray_1d)
 add_dependencies (tests test_darray_3d)
 add_dependencies (tests test_decomp_uneven)
 add_dependencies (tests test_decomps)
+add_dependencies (tests test_darray_fill)
+add_dependencies (tests test_decomp_frame)
 if(PIO_USE_MALLOC)
+#  add_dependencies (tests test_perf2)
   add_dependencies (tests test_darray_async_simple)
   add_dependencies (tests test_darray_async)
   add_dependencies (tests test_darray_async_many)
@@ -252,6 +261,10 @@ else ()
     EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d
     NUMPROCS ${AT_LEAST_FOUR_TASKS}
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_fill
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_fill
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   if(PIO_USE_MALLOC)
     add_mpi_test(test_darray_2sync
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_2sync
@@ -261,6 +274,10 @@ else ()
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_async_simple
       NUMPROCS ${AT_LEAST_FOUR_TASKS}
       TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    # add_mpi_test(test_perf2
+    #   EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_perf2
+    #   NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    #   TIMEOUT ${DEFAULT_TEST_TIMEOUT})
     add_mpi_test(test_darray_async
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_async
       NUMPROCS ${AT_LEAST_FOUR_TASKS}
diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am
index 16934faafebb..0b7042883b42 100644
--- a/tests/cunit/Makefile.am
+++ b/tests/cunit/Makefile.am
@@ -1,13 +1,15 @@
 ## This is the automake file for building the C tests for the PIO
 ## library.
+
 # Ed Hartnett 8/17/17
 
 # Link to our assembled library.
 AM_LDFLAGS = ${top_builddir}/src/clib/libpio.la
 AM_CPPFLAGS = -I$(top_srcdir)/src/clib
+LDADD = ${top_builddir}/src/clib/libpio.la
 
-# The tests that will be run.
-PIO_TESTS = test_async_mpi test_spmd test_intercomm2			\
+# Build the tests for make check.
+check_PROGRAMS = test_intercomm2 test_async_mpi test_spmd		\
 test_async_simple test_async_3proc test_async_4proc			\
 test_iosystem2_simple test_iosystem2_simple2 test_iosystem2		\
 test_iosystem3_simple test_iosystem3_simple2 test_iosystem3 test_pioc	\
@@ -16,13 +18,18 @@ test_darray_multi test_darray_multivar test_darray_multivar2		\
 test_darray_multivar3 test_darray_1d test_darray_3d			\
 test_decomp_uneven test_decomps test_rearr test_darray_async_simple	\
 test_darray_async test_darray_async_many test_darray_2sync		\
-test_async_multicomp test_async_multi2 test_async_manyproc
+test_async_multicomp test_async_multi2 test_async_manyproc		\
+test_darray_fill test_decomp_frame test_perf2 test_async_perf		\
+test_darray_vard
+
+if RUN_TESTS
+# Tests will run from a bash script.
+TESTS = run_tests.sh
+endif # RUN_TESTS
 
 # Source code for each test.
-test_async_mpi_SOURCES = test_async_mpi.c
 test_intercomm2_SOURCES = test_intercomm2.c test_common.c pio_tests.h
 test_async_simple_SOURCES = test_async_simple.c test_common.c pio_tests.h
-test_async_3proc_SOURCES = test_async_3proc.c test_common.c pio_tests.h
 test_async_4proc_SOURCES = test_async_4proc.c test_common.c pio_tests.h
 test_iosystem2_simple_SOURCES = test_iosystem2_simple.c test_common.c pio_tests.h
 test_iosystem2_simple2_SOURCES = test_iosystem2_simple2.c test_common.c pio_tests.h
@@ -49,26 +56,18 @@ test_darray_async_SOURCES = test_darray_async.c test_common.c pio_tests.h
 test_darray_async_many_SOURCES = test_darray_async_many.c test_common.c pio_tests.h
 test_darray_2sync_SOURCES = test_darray_2sync.c test_common.c pio_tests.h
 test_spmd_SOURCES = test_spmd.c test_common.c pio_tests.h
-test_intercomm2_SOURCES = test_async_simple.c test_common.c pio_tests.h
 test_async_3proc_SOURCES = test_async_3proc.c test_common.c pio_tests.h
 test_async_multicomp_SOURCES = test_async_multicomp.c test_common.c pio_tests.h
 test_async_multi2_SOURCES = test_async_multi2.c test_common.c pio_tests.h
 test_async_manyproc_SOURCES = test_async_manyproc.c test_common.c pio_tests.h
-
-# Build the tests for the tests target.
-tests: ${PIO_TESTS}
-
-# Build the tests for make check.
-check_PROGRAMS = $(PIO_TESTS)
-
-# Tests will run from a bash script.
-TESTS = run_tests.sh
-
-# Bash script needs all tests built.
-run_tests.sh : tests
+test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h
+test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h
+test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h
+test_async_perf_SOURCES = test_async_perf.c test_common.c pio_tests.h
+test_darray_vard_SOURCES = test_darray_vard.c test_common.c pio_tests.h
 
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh
 
 # Clean up files produced during testing.
-CLEANFILES = *.nc *.log
+CLEANFILES = *.nc *.log decomp*.txt
diff --git a/tests/cunit/pio_tests.h b/tests/cunit/pio_tests.h
index d1d0cae98941..551c570a3a71 100644
--- a/tests/cunit/pio_tests.h
+++ b/tests/cunit/pio_tests.h
@@ -61,7 +61,7 @@
 #endif /* _NETCDF4 */
 
 /** Handle MPI errors. This should only be used with MPI library
- * function calls. */
+ * function calls. Finalize and return. */
 #define MPIERR(e) do {                                                  \
         MPI_Error_string(e, err_buffer, &resultlen);                    \
         fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
@@ -69,14 +69,32 @@
         return ERR_AWFUL;                                               \
     } while (0)
 
+/** Handle MPI errors. This should only be used with MPI library
+ * function calls. Finalize and goto exit. */
+#define MPIBAIL(e) do {                                                  \
+        MPI_Error_string(e, err_buffer, &resultlen);                    \
+        fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+        MPI_Finalize();                                                 \
+        ret = NC_EIO;                                                  \
+        goto exit;                                                      \
+    } while (0)
+
 /** Handle non-MPI errors by finalizing the MPI library and exiting
- * with an exit code. */
+ * with an exit code. Finalize and return. */
 #define ERR(e) do {                                                     \
         fprintf(stderr, "%d Error %d in %s, line %d\n", my_rank, e, __FILE__, __LINE__); \
         MPI_Finalize();                                                 \
         return e;                                                       \
     } while (0)
 
+/** Handle non-MPI errors by finalizing the MPI library and goto
+ * exit. Finalize and goto exit. */
+#define BAIL(e) do {                                                     \
+        fprintf(stderr, "%d Error %d in %s, line %d\n", my_rank, e, __FILE__, __LINE__); \
+        MPI_Finalize();                                                 \
+        goto exit;                                                      \
+    } while (0)
+
 /** Global err buffer for MPI. When there is an MPI error, this buffer
  * is used to store the error message that is associated with the MPI
  * error. */
diff --git a/tests/cunit/run_tests.sh b/tests/cunit/run_tests.sh
index 8b1b8698be98..c9106617ae73 100755
--- a/tests/cunit/run_tests.sh
+++ b/tests/cunit/run_tests.sh
@@ -1,37 +1,50 @@
+#!/bin/sh
+# This is a test script for PIO.
+# Ed Hartnett
+
 # Stop execution of script if error is returned.
 set -e
 
 # Stop loop if ctrl-c is pressed.
-trap exit SIGINT SIGTERM
+trap exit INT TERM
 
 printf 'running PIO tests...\n'
 
-PIO_TESTS='test_async_mpi test_spmd test_rearr test_intercomm2 test_async_simple '\
+PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple '\
 'test_async_3proc test_async_4proc test_iosystem2_simple test_iosystem2_simple2 '\
 'test_iosystem2 test_iosystem3_simple test_iosystem3_simple2 test_iosystem3 test_pioc '\
 'test_pioc_unlim test_pioc_putget test_pioc_fill test_darray test_darray_multi '\
 'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\
 'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\
-'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '
+'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\
+'test_darray_fill test_darray_vard'
 
+success1=true
+success2=true
 for TEST in $PIO_TESTS
 do
-    success=false
+    success1=false
     echo "running ${TEST}"
-    mpiexec -n 4 ./${TEST} && success=true || break
+    mpiexec -n 4 ./${TEST} && success1=true
+    if test $success1 = false; then
+        break
+    fi
 done
 
-PIO_TESTS_8='test_async_multi2'
+PIO_TESTS_8='test_async_multi2  test_async_manyproc'
 
 for TEST in $PIO_TESTS_8
 do
-    success=false
+    success2=false
     echo "running ${TEST}"
-    mpiexec -n 8 ./${TEST} && success=true || break
+    mpiexec -n 8 ./${TEST} && success2=true
+    if test $success2 = false; then
+        break
+    fi
 done
 
 # Did we succeed?
-if test x$success = xtrue; then
+if test x$success1 = xtrue -a x$success2 = xtrue; then
     exit 0
 fi
 exit 1
diff --git a/tests/cunit/test_async_3proc.c b/tests/cunit/test_async_3proc.c
index 20c42d1ce831..8eff69fdf241 100644
--- a/tests/cunit/test_async_3proc.c
+++ b/tests/cunit/test_async_3proc.c
@@ -71,12 +71,12 @@ int main(int argc, char **argv)
             {
                 for (int flv = 0; flv < num_flavors; flv++)
                 {
-                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
                     int my_comp_idx = 0; /* Index in iosysid array. */
 
                     for (int sample = 0; sample < NUM_SAMPLES; sample++)
                     {
-                        char iotype_name[NC_MAX_NAME + 1];
+                        char iotype_name[PIO_MAX_NAME + 1];
 
                         /* Create a filename. */
                         if ((ret = get_iotype_name(flavor[flv], iotype_name)))
@@ -96,7 +96,7 @@ int main(int argc, char **argv)
                 /* Finalize the IO system. Only call this from the computation tasks. */
                 for (int c = 0; c < COMPONENT_COUNT; c++)
                 {
-                    if ((ret = PIOc_finalize(iosysid[c])))
+                    if ((ret = PIOc_free_iosystem(iosysid[c])))
                         ERR(ret);
                 }
             } /* endif comp_task */
diff --git a/tests/cunit/test_async_4proc.c b/tests/cunit/test_async_4proc.c
index 5eb23472db40..833110388f37 100644
--- a/tests/cunit/test_async_4proc.c
+++ b/tests/cunit/test_async_4proc.c
@@ -69,12 +69,12 @@ int main(int argc, char **argv)
             {
                 for (int flv = 0; flv < num_flavors; flv++)
                 {
-                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
                     int my_comp_idx = 0; /* Index in iosysid array. */
 
                     for (int sample = 0; sample < NUM_SAMPLES; sample++)
                     {
-                        char iotype_name[NC_MAX_NAME + 1];
+                        char iotype_name[PIO_MAX_NAME + 1];
 
                         /* Create a filename. */
                         if ((ret = get_iotype_name(flavor[flv], iotype_name)))
@@ -93,7 +93,7 @@ int main(int argc, char **argv)
 
                 /* Finalize the IO system. Only call this from the computation tasks. */
                 for (int c = 0; c < COMPONENT_COUNT; c++)
-                    if ((ret = PIOc_finalize(iosysid[c])))
+                    if ((ret = PIOc_free_iosystem(iosysid[c])))
                         ERR(ret);
             } /* endif comp_task */
 
diff --git a/tests/cunit/test_async_manyproc.c b/tests/cunit/test_async_manyproc.c
index 7a203a1d37d8..7c943ad06194 100644
--- a/tests/cunit/test_async_manyproc.c
+++ b/tests/cunit/test_async_manyproc.c
@@ -34,16 +34,9 @@ int main(int argc, char **argv)
 {
     int my_rank; /* Zero-based rank of processor. */
     int ntasks; /* Number of processors involved in current execution. */
-    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
     int num_iotypes; /* Number of PIO netCDF iotypes in this build. */
-    int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
-    int num_procs[COMPONENT_COUNT] = {NUM_COMP_PROCS, NUM_COMP_PROCS}; /* Num procs for IO and computation. */
     int io_proc_list[NUM_IO_PROCS];
-    int comp_proc_list1[NUM_COMP_PROCS] = {NUM_IO_PROCS, NUM_IO_PROCS + 1};
-    int comp_proc_list2[NUM_COMP_PROCS] = {NUM_IO_PROCS + 2, NUM_IO_PROCS + 3};
-    int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
     MPI_Comm test_comm;
-    int verbose = 0;
     int ret; /* Return code. */
 
     /* Initialize our list of IO tasks. */
@@ -61,6 +54,13 @@ int main(int argc, char **argv)
     /* Only do something on TARGET_NTASKS tasks. */
     if (my_rank < TARGET_NTASKS)
     {
+        int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+        int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
+        int num_procs[COMPONENT_COUNT] = {NUM_COMP_PROCS, NUM_COMP_PROCS}; /* Num procs for IO and computation. */
+        int comp_proc_list1[NUM_COMP_PROCS] = {NUM_IO_PROCS, NUM_IO_PROCS + 1};
+        int comp_proc_list2[NUM_COMP_PROCS] = {NUM_IO_PROCS + 2, NUM_IO_PROCS + 3};
+        int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_iotypes, iotype)))
             ERR(ret);
@@ -71,9 +71,6 @@ int main(int argc, char **argv)
         if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT,
                                    num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid)))
             ERR(ERR_INIT);
-        if (verbose)
-            for (int c = 0; c < COMPONENT_COUNT; c++)
-                printf("my_rank %d cmp %d iosysid[%d] %d\n", my_rank, c, c, iosysid[c]);
 
         /* All the netCDF calls are only executed on the computation
          * tasks. */
@@ -81,25 +78,25 @@ int main(int argc, char **argv)
         {
             for (int i = 0; i < num_iotypes; i++)
             {
-                /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */
-                /* /\* Ranks 0, 1, 2 are IO. 3, 4 are the first */
-                /*  * computation component. 5, 6 are the second. *\/ */
-                /* int my_comp_idx = my_rank < NUM_IO_PROCS + NUM_COMP_PROCS ? 0 : 1;  /\* Index in iosysid array. *\/ */
-
-                /* /\* Create sample file. *\/ */
-                /* if ((ret = create_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx, */
-                /*                               filename, TEST_NAME, verbose, 0, 0))) */
-                /*     ERR(ret); */
-
-                /* /\* Check the file for correctness. *\/ */
-                /* if ((ret = check_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx, */
-                /*                              filename, verbose, 0, 0))) */
-                /*     ERR(ret); */
+                char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                /* Ranks 0, 1, 2 are IO. 3, 4 are the first
+                 * computation component. 5, 6 are the second. */
+                int my_comp_idx = my_rank < NUM_IO_PROCS + NUM_COMP_PROCS ? 0 : 1;  /* Index in iosysid array. */
+
+                /* Create sample file. */
+                if ((ret = create_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
+                                              filename, TEST_NAME, 0, 0, 0)))
+                    ERR(ret);
+
+                /* Check the file for correctness. */
+                if ((ret = check_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
+                                             filename, 0, 0, 0)))
+                    ERR(ret);
             } /* next netcdf iotype */
 
             /* Finalize the IO system. Only call this from the computation tasks. */
             for (int c = 0; c < COMPONENT_COUNT; c++)
-                if ((ret = PIOc_finalize(iosysid[c])))
+                if ((ret = PIOc_free_iosystem(iosysid[c])))
                     ERR(ret);
         } /* endif comp_task */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_async_mpi.c b/tests/cunit/test_async_mpi.c
index 430a3b820a7f..e4e615a6c586 100644
--- a/tests/cunit/test_async_mpi.c
+++ b/tests/cunit/test_async_mpi.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * This program tests some MPI functionality that is used in PIO. This
  * runs on three processors, and does the same MPI commands that are
  * done when async mode is used, with 1 IO task, and two computation
@@ -116,8 +116,9 @@ int get_test_comm(int my_rank, int ntasks, int min_ntasks, int max_ntasks, MPI_C
  * @returns 0 for success, error code otherwise.
  * @author Ed Hartnett
  */
-int msg_handler(int verbose, int my_rank, int io_rank, int component_count, MPI_Comm *union_comm,
-                MPI_Comm *comp_comm, int *comproot, MPI_Comm io_comm)
+int msg_handler(int verbose, int my_rank, int io_rank, int component_count,
+                MPI_Comm *union_comm, MPI_Comm *comp_comm, int *comproot,
+                MPI_Comm io_comm)
 {
     int msg = 0;
     MPI_Request req[component_count];
@@ -133,13 +134,15 @@ int msg_handler(int verbose, int my_rank, int io_rank, int component_count, MPI_
         for (int cmp = 0; cmp < component_count; cmp++)
         {
             if (verbose)
-                printf("my_rank %d cmp %d about to call MPI_Irecv comproot[cmp] %d union_comm[cmp] %d\n",
-                       my_rank, cmp, comproot[cmp], union_comm[cmp]);
+                printf("my_rank %d cmp %d about to call MPI_Irecv comproot[cmp] %d "
+                       "union_comm[cmp] %lld\n", my_rank, cmp, comproot[cmp],
+                       (long long int)(union_comm[cmp]));
             if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, comproot[cmp], MPI_ANY_TAG,
                                     union_comm[cmp], &req[cmp])))
                 MPIERR(mpierr);
             if (verbose)
-                printf("my_rank %d MPI_Irecv req[%d] = %d\n", my_rank, cmp, req[cmp]);
+                printf("my_rank %d MPI_Irecv req[%d] = %lld\n", my_rank, cmp,
+                       (long long int)(req[cmp]));
         }
     }
 
@@ -156,28 +159,30 @@ int msg_handler(int verbose, int my_rank, int io_rank, int component_count, MPI_
         {
             if (verbose)
             {
-                printf("my_rank %d about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d\n",
-                       my_rank, req[0], MPI_REQUEST_NULL);
+                printf("my_rank %d about to call MPI_Waitany req[0] = %lld\n",
+                       my_rank, (long long int)(req[0]));
                 for (int c = 0; c < component_count; c++)
-                    printf("my_rank %d req[%d] = %d\n", my_rank, c, req[c]);
+                    printf("my_rank %d req[%d] = %lld\n", my_rank, c,
+                           (long long int)(req[c]));
             }
             if ((mpierr = MPI_Waitany(component_count, req, &index, &status)))
                 MPIERR(mpierr);
             if (verbose)
-                printf("my_rank %d Waitany returned index = %d req[%d] = %d\n", my_rank, index, index, req[index]);
+                printf("my_rank %d Waitany returned index = %d req[%d] = %lld\n",
+                       my_rank, index, index, (long long int)req[index]);
         }
 
         /* Broadcast the index and msg value to the rest of the IO tasks. */
         if (verbose)
-            printf("my_rank %d about to MPI_Bcast io_comm %d index %d msg %d\n", my_rank, io_comm,
-                   index, msg);
+            printf("my_rank %d about to MPI_Bcast io_comm %lld index %d msg %d\n",
+                   my_rank, (long long int)io_comm, index, msg);
         if ((mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, io_comm)))
             MPIERR(mpierr);
         if ((mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, io_comm)))
             MPIERR(mpierr);
         if (verbose)
-            printf("my_rank %d MPI_Bcast io_comm %d index %d msg %d\n", my_rank, io_comm,
-                   index, msg);
+            printf("my_rank %d MPI_Bcast io_comm %lld index %d msg %d\n",
+                   my_rank, (long long int)io_comm, index, msg);
 
         /* Handle the message. This code is run on all IO tasks. */
         switch (msg)
@@ -196,17 +201,19 @@ int msg_handler(int verbose, int my_rank, int io_rank, int component_count, MPI_
         if (!io_rank && msg != -1)
         {
             if (verbose)
-                printf("my_rank %d msg_handler about to Irecv index = %d comproot = %d union_comm = %d\n",
-                       my_rank, index, comproot[index], union_comm[index]);
+                printf("my_rank %d msg_handler about to Irecv index = %d comproot = %d union_comm = %lld\n",
+                       my_rank, index, comproot[index], (long long int)union_comm[index]);
             if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, comproot[index], MPI_ANY_TAG, union_comm[index],
                                     &req[index])))
                 MPIERR(mpierr);
             if (verbose)
-                printf("my_rank %d msg_handler called MPI_Irecv req[%d] = %d\n", my_rank, index, req[index]);
+                printf("my_rank %d msg_handler called MPI_Irecv req[%d] = %lld\n",
+                       my_rank, index, (long long int)req[index]);
         }
 
         if (verbose)
-            printf("my_rank %d msg_handler done msg = %d open_components = %d\n", my_rank, msg, open_components);
+            printf("my_rank %d msg_handler done msg = %d open_components = %d\n",
+                   my_rank, msg, open_components);
 
         /* If there are no more open components, exit. */
         if (msg == -1)
@@ -231,7 +238,6 @@ int main(int argc, char **argv)
     int ntasks;         /* Number of processors involved in current execution. */
     MPI_Comm test_comm; /* Communicator for tasks running tests. */
     int mpierr;         /* Return code from MPI functions. */
-    int verbose = 0;    /* Non-zero to turn on printf statements. */
     int ret;            /* Return code from function calls. */
 
     /* Initialize MPI. */
@@ -239,10 +245,10 @@ int main(int argc, char **argv)
         MPIERR(ret);
 
     /* Learn my rank and the total number of processors. */
-    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-        MPIERR(ret);
-    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
-        MPIERR(ret);
+    if ((mpierr = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+        MPIERR(mpierr);
+    if ((mpierr = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+        MPIERR(mpierr);
 
     /* Get test_comm. */
     if ((ret = get_test_comm(my_rank, ntasks, TARGET_NTASKS, TARGET_NTASKS, &test_comm)))
@@ -268,14 +274,12 @@ int main(int argc, char **argv)
         MPI_Comm union_comm[COMPONENT_COUNT];
         MPI_Comm intercomm[COMPONENT_COUNT];
         int in_cmp[COMPONENT_COUNT] = {0, 0};    /* Is this process in this computation component? */
+	int verbose = 0;    /* Non-zero to turn on printf statements. */
         
         /* Create group for world. */
         if ((ret = MPI_Comm_group(test_comm, &world_group)))
             MPIERR(ret);
 
-        if (verbose)
-            printf("MPI_GROUP_NULL %d MPI_COMM_NULL %d\n", MPI_GROUP_NULL, MPI_COMM_NULL);
-
         /* There is one shared IO comm. Create it. */
         if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group)))
             MPIERR(ret);
@@ -283,7 +287,8 @@ int main(int argc, char **argv)
             MPIERR(ret);
         MPI_Group_free(&io_group);
         if (verbose)
-            printf("my_rank %d created io comm io_comm = %d\n", my_rank, io_comm);
+            printf("my_rank %d created io comm io_comm = %lld\n", my_rank,
+                   (long long int)io_comm);
 
         /* For processes in the IO component, get their rank within the IO
          * communicator. */
@@ -317,7 +322,8 @@ int main(int argc, char **argv)
             if ((ret = MPI_Group_incl(world_group, 1, my_proc_list[cmp], &group[cmp])))
                 MPIERR(ret);
             if (verbose)
-                printf("my_rank %d created component MPI group - group[%d] = %d\n", my_rank, cmp, group[cmp]);
+                printf("my_rank %d created component MPI group - group[%d] = %lld\n",
+                       my_rank, cmp, (long long int)group[cmp]);
 
             /* Add proc numbers from IO. */
             proc_list_union[0] = 0;
@@ -347,8 +353,8 @@ int main(int argc, char **argv)
                     MPIERR(ret);
             }
             if (verbose)
-                printf("my_rank %d intracomm created for cmp = %d comp_comm[cmp] = %d comp_rank = %d\n",
-                       my_rank, cmp, comp_comm[cmp], comp_rank);
+                printf("my_rank %d intracomm created for cmp = %d comp_comm[cmp] = %lld comp_rank = %d\n",
+                       my_rank, cmp, (long long int)comp_comm[cmp], comp_rank);
             
             /* If this is the IO component, make a copy of the IO comm for
              * each computational component. */
@@ -357,7 +363,8 @@ int main(int argc, char **argv)
                 if ((ret = MPI_Comm_dup(io_comm, &io_comm2)))
                     MPIERR(ret);
                 if (verbose)
-                    printf("my_rank %d dup of io_comm = %d io_rank = %d\n", my_rank, io_comm, io_rank);
+                    printf("my_rank %d dup of io_comm = %lld io_rank = %d\n", my_rank,
+                           (long long int)io_comm, io_rank);
             }
 
             /* Create a group for the union of the IO component
@@ -368,8 +375,9 @@ int main(int argc, char **argv)
                 MPIERR(ret);
             MPI_Group_free(&union_group[cmp]);
             if (verbose)
-                printf("my_rank %d created union - union_group[%d] %d with %d procs union_comm[%d] %d\n",
-                       my_rank, cmp, union_group[cmp], nprocs_union, cmp, union_comm[cmp]);
+                printf("my_rank %d created union - union_group[%d] %lld with %d procs union_comm[%d] %lld\n",
+                       my_rank, cmp, (long long int)union_group[cmp], nprocs_union, cmp,
+                       (long long int)union_comm[cmp]);
 
 
             if (in_io || in_cmp[cmp])
@@ -403,23 +411,25 @@ int main(int argc, char **argv)
         } /* next computation component. */
 
         /* Now launch IO message processing on the IO task. */
-        int comproot[COMPONENT_COUNT] = {1, 1};
         if (in_io)
+	{
+	    int comproot[COMPONENT_COUNT] = {1, 1};
+	    
             if ((ret = msg_handler(verbose, my_rank, 0, COMPONENT_COUNT, union_comm, comp_comm,
                                    comproot, io_comm)))
                 ERR(ret);
+	}
 
         /* Send exit messages. */
         if (!in_io)
         {
             for (int cmp = 0; cmp < COMPONENT_COUNT; cmp++)
             {
-            
-                int msg = MSG_EXIT;
-                int ioroot = 0;
-
                 if (in_cmp[cmp])
                 {
+		    int ioroot = 0;
+		    int msg = MSG_EXIT;
+		    
                     if (verbose)
                         printf("my_rank %d sending exit message on union_comm %d\n", my_rank, union_comm[cmp]);
                     if ((mpierr = MPI_Send(&msg, 1, MPI_INT, ioroot, 1, union_comm[cmp])))
diff --git a/tests/cunit/test_async_multi2.c b/tests/cunit/test_async_multi2.c
index ebf538565c13..94512f536a56 100644
--- a/tests/cunit/test_async_multi2.c
+++ b/tests/cunit/test_async_multi2.c
@@ -32,16 +32,8 @@ int main(int argc, char **argv)
 {
     int my_rank; /* Zero-based rank of processor. */
     int ntasks; /* Number of processors involved in current execution. */
-    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
     int num_iotypes; /* Number of PIO netCDF iotypes in this build. */
-    int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
-    int num_procs[COMPONENT_COUNT] = {1, 1}; /* Num procs for IO and computation. */
-    int io_proc_list[NUM_IO_PROCS] = {0};
-    int comp_proc_list1[NUM_COMP_PROCS] = {1};
-    int comp_proc_list2[NUM_COMP_PROCS] = {2};
-    int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
     MPI_Comm test_comm;
-    int verbose = 1;
     int ret; /* Return code. */
 
     /* Initialize test. */
@@ -55,6 +47,14 @@ int main(int argc, char **argv)
     /* Only do something on TARGET_NTASKS tasks. */
     if (my_rank < TARGET_NTASKS)
     {
+        int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+        int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
+        int num_procs[COMPONENT_COUNT] = {1, 1}; /* Num procs for IO and computation. */
+        int io_proc_list[NUM_IO_PROCS] = {0};
+        int comp_proc_list1[NUM_COMP_PROCS] = {1};
+        int comp_proc_list2[NUM_COMP_PROCS] = {2};
+        int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_iotypes, iotype)))
             ERR(ret);
@@ -65,9 +65,6 @@ int main(int argc, char **argv)
         if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT,
                                    num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid)))
             ERR(ERR_INIT);
-        if (verbose)
-            for (int c = 0; c < COMPONENT_COUNT; c++)
-                printf("my_rank %d cmp %d iosysid[%d] %d\n", my_rank, c, c, iosysid[c]);
 
         /* All the netCDF calls are only executed on the computation
          * tasks. */
@@ -89,12 +86,12 @@ int main(int argc, char **argv)
             
                 /* Create sample file. */
                 if ((ret = create_nc_sample_4(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
-                                              filename, TEST_NAME, verbose, num_types)))
+                                              filename, TEST_NAME, 0, num_types)))
                     ERR(ret);
 
                 /* Check the file for correctness. */
                 if ((ret = check_nc_sample_4(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
-                                             filename, verbose, num_types)))
+                                             filename, 0, num_types)))
                     ERR(ret);
 
                 /* Free the decompositions. */
@@ -105,7 +102,7 @@ int main(int argc, char **argv)
 
             /* Finalize the IO system. Only call this from the computation tasks. */
             for (int c = 0; c < COMPONENT_COUNT; c++)
-                if ((ret = PIOc_finalize(iosysid[c])))
+                if ((ret = PIOc_free_iosystem(iosysid[c])))
                     ERR(ret);
         } /* endif comp_task */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_async_multicomp.c b/tests/cunit/test_async_multicomp.c
index 9822106fdd85..9f1819b9fe36 100644
--- a/tests/cunit/test_async_multicomp.c
+++ b/tests/cunit/test_async_multicomp.c
@@ -38,16 +38,8 @@ int main(int argc, char **argv)
 {
     int my_rank; /* Zero-based rank of processor. */
     int ntasks; /* Number of processors involved in current execution. */
-    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
     int num_iotypes; /* Number of PIO netCDF iotypes in this build. */
-    int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
-    int num_procs[COMPONENT_COUNT] = {1, 1}; /* Num procs for IO and computation. */
-    int io_proc_list[NUM_IO_PROCS] = {0};
-    int comp_proc_list1[NUM_COMP_PROCS] = {1};
-    int comp_proc_list2[NUM_COMP_PROCS] = {2};
-    int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
     MPI_Comm test_comm;
-    int verbose = 0;
     int ret; /* Return code. */
 
     /* Initialize test. */
@@ -55,12 +47,20 @@ int main(int argc, char **argv)
                               -1, &test_comm)))
         ERR(ERR_INIT);
 
-    /* Is the current process a computation task? */    
+    /* Is the current process a computation task? */
     int comp_task = my_rank < NUM_IO_PROCS ? 0 : 1;
-    
+
     /* Only do something on TARGET_NTASKS tasks. */
     if (my_rank < TARGET_NTASKS)
     {
+        int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+        int iotype[NUM_IOTYPES]; /* iotypes for the supported netCDF IO iotypes. */
+        int num_procs[COMPONENT_COUNT] = {1, 1}; /* Num procs for IO and computation. */
+        int io_proc_list[NUM_IO_PROCS] = {0};
+        int comp_proc_list1[NUM_COMP_PROCS] = {1};
+        int comp_proc_list2[NUM_COMP_PROCS] = {2};
+        int *proc_list[COMPONENT_COUNT] = {comp_proc_list1, comp_proc_list2};
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_iotypes, iotype)))
             ERR(ret);
@@ -76,9 +76,6 @@ int main(int argc, char **argv)
         if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT,
                                    num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid)))
             ERR(ERR_INIT);
-        if (verbose)
-            for (int c = 0; c < COMPONENT_COUNT; c++)
-                printf("my_rank %d cmp %d iosysid[%d] %d\n", my_rank, c, c, iosysid[c]);
 
         /* All the netCDF calls are only executed on the computation
          * tasks. */
@@ -90,7 +87,7 @@ int main(int argc, char **argv)
                 int my_comp_idx = my_rank - 1; /* Index in iosysid array. */
                 int dim_len_2d[NDIM2] = {DIM_LEN2, DIM_LEN3};
                 int ioid = 0;
-                
+
                 if ((ret = create_decomposition_2d(NUM_COMP_PROCS, my_rank, iosysid[my_comp_idx], dim_len_2d,
                                                    &ioid, PIO_SHORT)))
                     ERR(ret);
@@ -101,12 +98,12 @@ int main(int argc, char **argv)
 
                     /* Create sample file. */
                     if ((ret = create_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
-                                                  filename, TEST_NAME, verbose, use_darray, ioid)))
+                                                  filename, TEST_NAME, 0, use_darray, ioid)))
                         ERR(ret);
-                    
+
                     /* Check the file for correctness. */
                     if ((ret = check_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx,
-                                                 filename, verbose, 0, ioid)))
+                                                 filename, 0, 0, ioid)))
                         ERR(ret);
                 } /* next use_darray */
 
@@ -118,7 +115,7 @@ int main(int argc, char **argv)
 
             /* Finalize the IO system. Only call this from the computation tasks. */
             for (int c = 0; c < COMPONENT_COUNT; c++)
-                if ((ret = PIOc_finalize(iosysid[c])))
+                if ((ret = PIOc_free_iosystem(iosysid[c])))
                     ERR(ret);
         } /* endif comp_task */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c
new file mode 100644
index 000000000000..89adde1a1bd0
--- /dev/null
+++ b/tests/cunit/test_async_perf.c
@@ -0,0 +1,311 @@
+/*
+ * This program tests performance of darray writes with async.
+ *
+ * @author Ed Hartnett
+ * @date 5/4/17
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 1
+
+/* The name of this test. */
+#define TEST_NAME "test_async_perf"
+
+/* For 2-D use. */
+#define NDIM2 2
+
+/* For 3-D use. */
+#define NDIM3 3
+
+/* For 4-D use. */
+#define NDIM4 4
+
+/* For maplens of 2. */
+#define MAPLEN2 2
+
+/* Lengths of non-unlimited dimensions. */
+#define LAT_LEN 2
+#define LON_LEN 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 1024
+#define Y_DIM_LEN 1024
+#define Z_DIM_LEN 256
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 3
+
+/* Name of record test var. */
+#define REC_VAR_NAME "Duncan_McCloud_of_the_clan_McCloud"
+
+/* How many different number of IO tasks to check? */
+#define MAX_IO_TESTS 5
+
+#define COMPONENT_COUNT 1
+
+char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"unlim", "x", "y", "z"};
+
+/* Length of the dimension. */
+#define LEN3 3
+
+#define NUM_VAR_SETS 2
+
+/* Create the decomposition to divide the 4-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 3 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks (tasks doing
+ * computation).
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid, PIO_Offset *elements_per_pe)
+{
+    PIO_Offset my_elem_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+    int my_proc_rank = my_rank - 1;
+    int ret;
+
+    /* How many data elements per task? */
+    my_elem_per_pe = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks;
+    if (elements_per_pe)
+        *elements_per_pe = my_elem_per_pe;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(my_elem_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. */
+    for (int i = 0; i < my_elem_per_pe; i++)
+        compdof[i] = my_proc_rank * my_elem_per_pe + i;
+
+    /* Create the PIO decomposition for this test. */
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, my_elem_per_pe,
+                                compdof, ioid, 0, NULL, NULL)))
+        ERR(ret);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/* Run a simple test using darrays with async. */
+int
+run_darray_async_test(int iosysid, int fmt, int my_rank, int ntasks, int niotasks,
+                      MPI_Comm test_comm, MPI_Comm comp_comm, int *flavor, int piotype)
+{
+    int ioid3;
+    int dim_len[NDIM4] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+    PIO_Offset elements_per_pe2;
+    char decomp_filename[PIO_MAX_NAME + 1];
+    int ret;
+
+    sprintf(decomp_filename, "decomp_rdat_%s_.nc", TEST_NAME);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition_3d(ntasks - niotasks, my_rank, iosysid, &ioid3,
+                                       &elements_per_pe2)))
+        return ret;
+
+    {
+        int ncid;
+        PIO_Offset type_size;
+        int dimid[NDIM4];
+        int varid;
+        char data_filename[PIO_MAX_NAME + 1];
+        int *my_data_int;
+        int d, t;
+
+        if (!(my_data_int = malloc(elements_per_pe2 * sizeof(int))))
+            BAIL(PIO_ENOMEM);
+
+        for (d = 0; d < elements_per_pe2; d++)
+            my_data_int[d] = my_rank;
+
+        /* Create sample output file. */
+        /* sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], */
+        /*         piotype); */
+        sprintf(data_filename, "data_%s.nc", TEST_NAME);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], data_filename,
+                                   NC_CLOBBER)))
+            BAIL(ret);
+
+        /* Find the size of the type. */
+        if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size)))
+            BAIL(ret);
+
+        /* Define dimensions. */
+        for (int d = 0; d < NDIM4; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
+                BAIL(ret);
+
+        /* Define variables. */
+        if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM4, dimid, &varid)))
+            BAIL(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            BAIL(ret);
+
+        for (t = 0; t < NUM_TIMESTEPS; t++)
+        {
+            /* Set the record number for the record vars. */
+            if ((ret = PIOc_setframe(ncid, varid, t)))
+                BAIL(ret);
+
+            /* Write some data to the record vars. */
+            if ((ret = PIOc_write_darray(ncid, varid, ioid3, elements_per_pe2,
+                                         my_data_int, NULL)))
+                BAIL(ret);
+
+            /* Sync the file. */
+            if ((ret = PIOc_sync(ncid)))
+                BAIL(ret);
+
+        }
+
+
+        /* Close the file. */
+        if ((ret = PIOc_closefile(ncid)))
+            BAIL(ret);
+
+        free(my_data_int);
+    }
+
+    /* Free the decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid3)))
+        BAIL(ret);
+exit:
+    return ret;
+}
+
+/* Run Tests for pio_spmd.c functions. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int iosysid;
+    int num_computation_procs;
+    MPI_Comm io_comm;              /* Will get a duplicate of IO communicator. */
+    MPI_Comm comp_comm[COMPONENT_COUNT]; /* Will get duplicates of computation communicators. */
+    int num_io_procs[MAX_IO_TESTS] = {1, 4, 16, 64, 128}; /* Number of processors that will do IO. */
+    int num_io_tests; /* How many different num IO procs to try? */
+    int mpierr;
+    int fmt, niotest;
+    int ret;     /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, 1, 0, -1, &test_comm)))
+        ERR(ERR_INIT);
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+
+    /* How many processors for IO? */
+    num_io_tests = 1;
+    if (ntasks >= 32)
+        num_io_tests = 2;
+    if (ntasks >= 64)
+        num_io_tests = 3;
+    if (ntasks >= 128)
+        num_io_tests = 4;
+    if (ntasks >= 512)
+        num_io_tests = 5;
+
+    if (!my_rank)
+        printf("ntasks\tnio\trearr\tfill\tformat\ttime(s)\tdata size (MB)\t"
+               "performance(MB/s)\n");
+
+    for (niotest = 0; niotest < num_io_tests; niotest++)
+    {
+        num_computation_procs = ntasks - num_io_procs[niotest];
+
+        for (fmt = 0; fmt < num_flavors; fmt++)
+        {
+            struct timeval starttime, endtime;
+            long long startt, endt;
+            long long delta;
+            float num_megabytes;
+            float delta_in_sec;
+            float mb_per_sec;
+
+            /* Start the clock. */
+            if (!my_rank)
+            {
+                gettimeofday(&starttime, NULL);
+                startt = (1000000 * starttime.tv_sec) + starttime.tv_usec;
+            }
+
+            if ((ret = PIOc_init_async(test_comm, num_io_procs[niotest], NULL, COMPONENT_COUNT,
+                                       &num_computation_procs, NULL, &io_comm, comp_comm,
+                                       PIO_REARR_BOX, &iosysid)))
+                ERR(ERR_INIT);
+
+            /* This code runs only on computation components. */
+            if (my_rank >= num_io_procs[niotest])
+            {
+                /* Run the simple darray async test. */
+                if ((ret = run_darray_async_test(iosysid, fmt, my_rank, ntasks, num_io_procs[niotest],
+                                                 test_comm, comp_comm[0], flavor, PIO_INT)))
+                    return ret;
+
+                /* Finalize PIO system. */
+                if ((ret = PIOc_free_iosystem(iosysid)))
+                    return ret;
+
+                /* Free the computation conomponent communicator. */
+                if ((mpierr = MPI_Comm_free(comp_comm)))
+                    MPIERR(mpierr);
+            }
+            else
+            {
+                /* Free the IO communicator. */
+                if ((mpierr = MPI_Comm_free(&io_comm)))
+                    MPIERR(mpierr);
+            }
+
+            if (!my_rank)
+            {
+                /* Stop the clock. */
+                gettimeofday(&endtime, NULL);
+
+                /* Compute the time delta */
+                endt = (1000000 * endtime.tv_sec) + endtime.tv_usec;
+                delta = (endt - startt)/NUM_TIMESTEPS;
+                delta_in_sec = (float)delta / 1000000;
+                num_megabytes = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * NUM_TIMESTEPS *
+                                 sizeof(int))/(1024*1024);
+                mb_per_sec = num_megabytes / delta_in_sec;
+                printf("%d\t%d\t%d\t%d\t%d\t%8.3f\t%8.1f\t%8.3f\n", ntasks, num_io_procs[niotest],
+                       1, 0, fmt, delta_in_sec, num_megabytes, mb_per_sec);
+            }
+
+        } /* next fmt */
+    } /* next niotest */
+
+    /* printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); */
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    return 0;
+}
diff --git a/tests/cunit/test_async_simple.c b/tests/cunit/test_async_simple.c
index aa197713a30d..d8520efbd322 100644
--- a/tests/cunit/test_async_simple.c
+++ b/tests/cunit/test_async_simple.c
@@ -34,15 +34,9 @@ int main(int argc, char **argv)
 #define NUM_COMP_PROCS 1
     int my_rank; /* Zero-based rank of processor. */
     int ntasks; /* Number of processors involved in current execution. */
-    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
     int num_flavors; /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
-    int ret; /* Return code. */
-    int num_procs[COMPONENT_COUNT] = {1}; /* Num procs for IO and computation. */
-    int io_proc_list[NUM_IO_PROCS] = {0};
-    int comp_proc_list[NUM_COMP_PROCS] = {1};
-    int *proc_list[COMPONENT_COUNT] = {comp_proc_list};
     MPI_Comm test_comm;
+    int ret; /* Return code. */
 
     /* Initialize test. */
     if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS,
@@ -52,6 +46,13 @@ int main(int argc, char **argv)
     /* Only do something on TARGET_NTASKS tasks. */
     if (my_rank < TARGET_NTASKS)
     {
+        int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+        int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+        int num_procs[COMPONENT_COUNT] = {1}; /* Num procs for IO and computation. */
+        int io_proc_list[NUM_IO_PROCS] = {0};
+        int comp_proc_list[NUM_COMP_PROCS] = {1};
+        int *proc_list[COMPONENT_COUNT] = {comp_proc_list};
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
             ERR(ret);
@@ -89,8 +90,8 @@ int main(int argc, char **argv)
 
                 for (int sample = 0; sample < NUM_SAMPLES; sample++)
                 {
-                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
-                    char iotype_name[NC_MAX_NAME + 1];
+                    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+                    char iotype_name[PIO_MAX_NAME + 1];
 
                     /* Create a filename. */
                     if ((ret = get_iotype_name(flavor[flv], iotype_name)))
@@ -109,7 +110,7 @@ int main(int argc, char **argv)
 
             /* Finalize the IO system. Only call this from the computation tasks. */
             for (int c = 0; c < COMPONENT_COUNT; c++)
-                if ((ret = PIOc_finalize(iosysid[c])))
+                if ((ret = PIOc_free_iosystem(iosysid[c])))
                     ERR(ret);
         } /* endif comp_task */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_common.c b/tests/cunit/test_common.c
index 6086a4a19b02..2846a6bc1c78 100644
--- a/tests/cunit/test_common.c
+++ b/tests/cunit/test_common.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * Common test code for PIO C tests.
  *
  * Ed Hartnett
@@ -160,13 +160,13 @@ get_iotypes(int *num_flavors, int *flavors)
  *
  * @param iotype the IO type
  * @param name pointer that will get name of IO type. Must have enough
- * memory allocated (NC_MAX_NAME + 1 works.)
+ * memory allocated (PIO_MAX_NAME + 1 works.)
  * @returns 0 for success, error code otherwise.
  * @internal
  */
 int get_iotype_name(int iotype, char *name)
 {
-    char flavor_name[NUM_FLAVORS][NC_MAX_NAME + 1] = {"pnetcdf", "classic",
+    char flavor_name[NUM_FLAVORS][PIO_MAX_NAME + 1] = {"pnetcdf", "classic",
                                                       "serial4", "parallel4"};
 
     /* Check inputs. */
@@ -186,7 +186,9 @@ int get_iotype_name(int iotype, char *name)
  * @param my_rank pointer that gets this tasks rank.
  * @param ntasks pointer that gets the number of tasks in WORLD
  * communicator.
- * @param target_ntasks the number of tasks this test needs to run.
+ * @param min_ntasks the min number of tasks this test needs to run.
+ * @param max_ntasks the max number of tasks this test needs to run. 0
+ * means no max.
  * @param log_level PIOc_set_log_level() will be called with this value.
  * @param comm a pointer to an MPI communicator that will be created
  * for this test and contain target_ntasks tasks from WORLD.
@@ -220,7 +222,7 @@ int pio_test_init2(int argc, char **argv, int *my_rank, int *ntasks,
                 min_ntasks);
         return ERR_AWFUL;
     }
-    else if (*ntasks > max_ntasks)
+    else if (max_ntasks && *ntasks > max_ntasks)
     {
         /* If more tasks are available than we need for this test,
          * create a communicator with exactly the number of tasks we
@@ -311,7 +313,7 @@ int
 test_inq_type(int ncid, int format)
 {
 #define NUM_TYPES 11
-    char type_name[NC_MAX_NAME + 1];
+    char type_name[PIO_MAX_NAME + 1];
     PIO_Offset type_size;
     nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
                                 NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
@@ -538,9 +540,9 @@ check_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *nci
     int ret;
     int ndims, nvars, ngatts, unlimdimid;
     int ndims2, nvars2, ngatts2, unlimdimid2;
-    char dimname[NC_MAX_NAME + 1];
+    char dimname[PIO_MAX_NAME + 1];
     PIO_Offset dimlen;
-    char varname[NC_MAX_NAME + 1];
+    char varname[PIO_MAX_NAME + 1];
     nc_type vartype;
     int varndims, vardimids, varnatts;
 
@@ -634,7 +636,7 @@ create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nc
         return ret;
 
     /* Define a dimension. */
-    char dimname2[NC_MAX_NAME + 1];
+    char dimname2[PIO_MAX_NAME + 1];
     if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME_S2, DIM_LEN_S2, &dimid)))
         return ret;
     if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
@@ -645,7 +647,7 @@ create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nc
         return ret;
 
     /* Define a 1-D variable. */
-    char varname2[NC_MAX_NAME + 1];
+    char varname2[PIO_MAX_NAME + 1];
     if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME_S2, NC_INT, NDIM_S2, &dimid, &varid)))
         return ret;
     if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
@@ -660,7 +662,7 @@ create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nc
     short short_att_data = ATT_VALUE_S2;
     float float_att_data = ATT_VALUE_S2;
     double double_att_data = ATT_VALUE_S2;
-    char attname2[NC_MAX_NAME + 1];
+    char attname2[PIO_MAX_NAME + 1];
     /* Write an att and rename it. */
     if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NC_INT, 1, &att_data)))
         return ret;
@@ -730,14 +732,14 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci
     int ndims, nvars, ngatts, unlimdimid;
     int ndims2, nvars2, ngatts2, unlimdimid2;
     int dimid2;
-    char dimname[NC_MAX_NAME + 1];
+    char dimname[PIO_MAX_NAME + 1];
     PIO_Offset dimlen;
-    char dimname2[NC_MAX_NAME + 1];
+    char dimname2[PIO_MAX_NAME + 1];
     PIO_Offset dimlen2;
-    char varname[NC_MAX_NAME + 1];
+    char varname[PIO_MAX_NAME + 1];
     nc_type vartype;
     int varndims, vardimids, varnatts;
-    char varname2[NC_MAX_NAME + 1];
+    char varname2[PIO_MAX_NAME + 1];
     nc_type vartype2;
     int varndims2, vardimids2, varnatts2;
     int varid2;
@@ -747,7 +749,7 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci
     double double_att_data;
     nc_type atttype;
     PIO_Offset attlen;
-    char myattname[NC_MAX_NAME + 1];
+    char myattname[PIO_MAX_NAME + 1];
     int myid;
     PIO_Offset start[NDIM_S2] = {0}, count[NDIM_S2] = {DIM_LEN_S2};
     int data_in[DIM_LEN_S2];
@@ -967,7 +969,7 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
                        char *filename, char *test_name, int verbose, int use_darray,
                        int ioid)
 {
-    char iotype_name[NC_MAX_NAME + 1];
+    char iotype_name[PIO_MAX_NAME + 1];
     int ncid;
     signed char my_char_comp_idx = my_comp_idx;
     int varid[NVAR];
@@ -1223,7 +1225,7 @@ int check_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
 int create_nc_sample_4(int iosysid, int iotype, int my_rank, int my_comp_idx,
                        char *filename, char *test_name, int verbose, int num_types)
 {
-    char iotype_name[NC_MAX_NAME + 1];
+    char iotype_name[PIO_MAX_NAME + 1];
     int ncid;
     int scalar_varid[num_types];
     int varid[num_types];
diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c
index aaca7d88d614..40c4f7e289f0 100644
--- a/tests/cunit/test_darray.c
+++ b/tests/cunit/test_darray.c
@@ -76,11 +76,14 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
     int ncid;      /* The ncid of the netCDF file. */
     int ncid2;     /* The ncid of the re-opened netCDF file. */
     int varid;     /* The ID of the netCDF varable. */
-    int varid2;    /* The ID of a varable of different type. */
+    int varid2;     /* The ID of a netCDF varable of different type. */
     int wrong_varid = TEST_VAL_42;  /* A wrong ID. */
     int ret;       /* Return code. */
+    MPI_Datatype mpi_type;
+    int type_size; /* size of a variable of type pio_type */
+    int other_type; /* another variable of the same size but different type */
     PIO_Offset arraylen = 4;
-    void *fillvalue;
+    void *fillvalue, *ofillvalue;
     void *test_data;
     void *test_data_in;
     int fillvalue_int = NC_FILL_INT;
@@ -116,7 +119,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 /* Create the filename. */
                 sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
                         flavor[fmt], pio_type, test_multi, provide_fill);
-
                 /* Select the fill value and data. */
                 switch (pio_type)
                 {
@@ -152,9 +154,27 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid)))
                     ERR(ret);
 
-                /* Define a variable with a different type. */
-                int other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT;
-                if ((ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2)))
+                /* Define a variable with a different type but same size. */
+		if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size)))
+		    ERR(ret);
+		if (type_size == NETCDF_INT_FLOAT_SIZE)
+		    other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT;
+//		else if(type_size == NETCDF_DOUBLE_INT64_SIZE)
+//		    other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64;
+		else
+		    other_type = 0; /* skip the test */
+                switch (other_type)
+                {
+                case PIO_INT:
+                    ofillvalue = provide_fill ? &fillvalue_int : NULL;
+                    break;
+                case PIO_FLOAT:
+                    ofillvalue = provide_fill ? &fillvalue_float : NULL;
+                    break;
+                default:
+		    break;
+                }
+                if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2)))
                     ERR(ret);
 
                 /* End define mode. */
@@ -164,6 +184,8 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 /* Set the value of the record dimension. */
                 if ((ret = PIOc_setframe(ncid, varid, 0)))
                     ERR(ret);
+                if (other_type && (ret = PIOc_setframe(ncid, varid2, 0)))
+                    ERR(ret);
 
                 int frame = 0;
                 int flushtodisk = test_multi - 1;
@@ -178,16 +200,19 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                         ERR(ERR_WRONG);
                     if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR)
                         ERR(ERR_WRONG);
-                    if (PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, fillvalue) != PIO_EINVAL)
-                        ERR(ERR_WRONG);
+
+		    /* This should work - library type conversion */
+		    if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue)))
+                        ERR(ret);
 
                     /* Write the data. */
                     if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
                         ERR(ret);
+
                 }
                 else
                 {
-                    int varid_big = NC_MAX_VARS + TEST_VAL_42;
+                    int varid_big = PIO_MAX_VARS + TEST_VAL_42;
 
                     /* These will not work. */
                     if (PIOc_write_darray_multi(ncid + TEST_VAL_42, &varid, ioid, 1, arraylen, test_data, &frame,
@@ -205,9 +230,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                     if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame,
                                                 fillvalue, flushtodisk) != PIO_ENOTVAR)
                         ERR(ERR_WRONG);
+//		    pio_setloglevel(3);
                     if (PIOc_write_darray_multi(ncid, &wrong_varid, ioid, 1, arraylen, test_data, &frame,
                                                 fillvalue, flushtodisk) != PIO_ENOTVAR)
                         ERR(ERR_WRONG);
+//		    pio_setloglevel(0);
+
+		    /* This should work - library type conversion */
+		    if (other_type && (ret = PIOc_write_darray_multi(ncid, &varid2, ioid, 1, arraylen, test_data, &frame,
+						       fillvalue, flushtodisk)))
+                        ERR(ret);
 
                     /* Write the data with the _multi function. */
                     if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame,
@@ -219,10 +251,18 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 if ((ret = PIOc_closefile(ncid)))
                     ERR(ret);
 
+
                 /* Reopen the file. */
                 if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
                     ERR(ret);
 
+		PIO_Offset dimlen;
+		/* check the unlimited dim size - it should be 1 */
+		if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen)))
+		    ERR(ret);
+		if (dimlen != 1)
+		    ERR(ERR_WRONG);
+
                 /* These should not work. */
                 if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen,
                                      test_data_in) != PIO_EBADID)
@@ -239,6 +279,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
                     ERR(ret);
 
+                /* /\* Read the data. *\/ */
+                /* if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) */
+                /*     ERR(ret); */
+
                 /* Check the results. */
                 for (int f = 0; f < arraylen; f++)
                 {
@@ -273,7 +317,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                                                 fillvalue, flushtodisk) != PIO_EPERM)
                         ERR(ERR_WRONG);
                 }
-
                 /* Close the netCDF file. */
                 if ((ret = PIOc_closefile(ncid2)))
                     ERR(ret);
@@ -299,7 +342,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
 {
 #define NUM_TYPES_TO_TEST 3
     int ioid;
-    char filename[NC_MAX_NAME + 1];
+    char filename[PIO_MAX_NAME + 1];
     int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE};
     int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
     int ret; /* Return code. */
@@ -372,7 +415,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
         } /* next rearranger */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_darray_1d.c b/tests/cunit/test_darray_1d.c
index 189d2e3e9bb9..556c2ae1eb3c 100644
--- a/tests/cunit/test_darray_1d.c
+++ b/tests/cunit/test_darray_1d.c
@@ -579,11 +579,9 @@ int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors,
         if (!(test_data_in = malloc(type_size * arraylen)))
             ERR(PIO_ENOMEM);
 
-        /* Set the record number for the unlimited dimension. */
-        if ((ret = PIOc_setframe(ncid, varid, 0)))
-            ERR(ret);
-
-        /* Read the data. */
+        /* Read the data. We don't have to set the record number for
+         * the unlimited dimension. If we don't set it, PIO will
+         * assume a value of 0. */
         if ((ret = PIOc_read_darray(ncid, varid, ioid, arraylen, test_data_in)))
             ERR(ret);
 
@@ -867,7 +865,7 @@ int main(int argc, char **argv)
             }
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
         } /* next rearranger */
 
diff --git a/tests/cunit/test_darray_2sync.c b/tests/cunit/test_darray_2sync.c
index a5e27ccc4abe..406bd7685d1c 100644
--- a/tests/cunit/test_darray_2sync.c
+++ b/tests/cunit/test_darray_2sync.c
@@ -189,6 +189,10 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype,
             if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid)))
                 ERR(ret);
 
+            /* Turn on fill mode for this var. */
+            if ((ret = PIOc_def_var_fill(ncid, varid, 0, default_fillvalue)))
+                ERR(ret);
+
             /* End define mode. */
             if ((ret = PIOc_enddef(ncid)))
                 ERR(ret);
@@ -508,7 +512,7 @@ int run_async_tests(MPI_Comm test_comm, int my_rank, int num_iotypes, int *iotyp
             ERR(ret);
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             return ret;
 
         /* Free the computation conomponent communicator. */
@@ -544,7 +548,7 @@ int run_noasync_tests(MPI_Comm test_comm, int my_rank, int num_iotypes, int *iot
         ERR(ret);
 
     /* Finalize PIO system. */
-    if ((ret = PIOc_finalize(iosysid)))
+    if ((ret = PIOc_free_iosystem(iosysid)))
         return ret;
 
     return PIO_NOERR;
diff --git a/tests/cunit/test_darray_3d.c b/tests/cunit/test_darray_3d.c
index e927da7047b9..dc77f40ee3ab 100644
--- a/tests/cunit/test_darray_3d.c
+++ b/tests/cunit/test_darray_3d.c
@@ -248,12 +248,13 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
     char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
     char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
     int fortran_order_in; /* Indicates fortran vs. c order. */
-    int ret;              /* Return code. */
 
     /* Use PIO to create the decomp file in each of the four
      * available ways. */
     for (int fmt = 0; fmt < num_flavors; fmt++)
     {
+	int ret;              /* Return code. */
+	
         /* Create the filename. */
         sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
 
@@ -361,8 +362,6 @@ int main(int argc, char **argv)
 {
     int my_rank;
     int ntasks;
-    int num_flavors; /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
     MPI_Comm test_comm; /* A communicator for this test. */
     int ret;         /* Return code. */
 
@@ -381,7 +380,8 @@ int main(int argc, char **argv)
         int iosysid;  /* The ID for the parallel I/O system. */
         int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
         int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
-        int ret;      /* Return code. */
+	int num_flavors; /* Number of PIO netCDF flavors in this build. */
+	int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
 
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
@@ -401,7 +401,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
 
         } /* next rearranger */
diff --git a/tests/cunit/test_darray_async.c b/tests/cunit/test_darray_async.c
index 9d493fc51eb2..d08da44442ad 100644
--- a/tests/cunit/test_darray_async.c
+++ b/tests/cunit/test_darray_async.c
@@ -61,24 +61,24 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
 {
     int ncid;
     int varid[NVAR] = {0, 1, 2, 3};
-    void *data_in;
-    void *data_in_norec;
+    void *data_in = NULL;
+    void *data_in_norec = NULL;
     PIO_Offset type_size;
     int ret;
 
     /* Reopen the file. */
     if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE)))
-        ERR(ret);
+        BAIL(ret);
 
     /* Get the size of the type. */
     if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size)))
-        ERR(ret);
+        BAIL(ret);
 
     /* Allocate memory to read data. */
     if (!(data_in = malloc(LAT_LEN * LON_LEN * type_size * NREC)))
-        ERR(PIO_ENOMEM);
+        BAIL(PIO_ENOMEM);
     if (!(data_in_norec = malloc(LAT_LEN * LON_LEN * type_size)))
-        ERR(PIO_ENOMEM);
+        BAIL(PIO_ENOMEM);
 
     /* We have two sets of variables, those with unlimted, and those
      * without unlimited dimension. */
@@ -90,12 +90,12 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
         /* Read the record data. The values we expect are: 10, 11, 20, 21, 30,
          * 31, in each of three records. */
         if ((ret = PIOc_get_var(ncid, rec_varid, data_in)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Read the non-record data. The values we expect are: 10, 11, 20, 21, 30,
          * 31. */
         if ((ret = PIOc_get_var(ncid, norec_varid, data_in_norec)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Check the results. */
         for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++)
@@ -104,53 +104,53 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
             switch (piotype)
             {
             case PIO_BYTE:
-                if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_CHAR:
-                if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_SHORT:
-                if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_INT:
-                if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_FLOAT:
                 if (((float *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_DOUBLE:
                 if (((double *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
 #ifdef _NETCDF4
             case PIO_UBYTE:
-                if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_USHORT:
-                if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_UINT:
-                if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_INT64:
-                if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
             case PIO_UINT64:
-                if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2)
-                    ERR(ret);
+                if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2)
+                    BAIL(ret);
                 break;
 #endif /* _NETCDF4 */
             default:
-                ERR(ERR_WRONG);
+                BAIL(ERR_WRONG);
             }
         }
 
@@ -161,65 +161,68 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
             {
             case PIO_BYTE:
                 if (((signed char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_CHAR:
                 if (((char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_SHORT:
                 if (((short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_INT:
                 if (((int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_FLOAT:
                 if (((float *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_DOUBLE:
                 if (((double *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
 #ifdef _NETCDF4
             case PIO_UBYTE:
                 if (((unsigned char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_USHORT:
                 if (((unsigned short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_UINT:
                 if (((unsigned int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_INT64:
                 if (((long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
             case PIO_UINT64:
                 if (((unsigned long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2)
-                    ERR(ret);
+                    BAIL(ret);
                 break;
 #endif /* _NETCDF4 */
             default:
-                ERR(ERR_WRONG);
+                BAIL(ERR_WRONG);
             }
         }
     } /* next var set */
 
-    /* Free resources. */
-    free(data_in);
-    free(data_in_norec);
-
     /* Close the file. */
     if ((ret = PIOc_closefile(ncid)))
-        ERR(ret);
+        BAIL(ret);
 
-    return 0;
+exit:
+    /* Free resources. */
+    if (data_in)
+        free(data_in);
+    if (data_in_norec)
+        free(data_in_norec);
+
+    return ret;
 }
 
 /* Run a simple test using darrays with async. */
@@ -231,6 +234,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
     PIO_Offset elements_per_pe = LAT_LEN;
     PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1};
     char decomp_filename[PIO_MAX_NAME + 1];
+    void *my_data_multi;
     int ret;
 
     sprintf(decomp_filename, "decomp_rdat_%s_.nc", TEST_NAME);
@@ -238,7 +242,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
     /* Create the PIO decomposition for this test. */
     if ((ret = PIOc_init_decomp(iosysid, piotype, NDIM2, &dim_len[1], elements_per_pe,
                                 compdof, &ioid, PIO_REARR_BOX, NULL, NULL)))
-        ERR(ret);
+        BAIL(ret);
 
     /* Write the decomp file (on appropriate tasks). */
     if ((ret = PIOc_write_nc_decomp(iosysid, decomp_filename, 0, ioid, NULL, NULL, 0)))
@@ -252,7 +256,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
 
     /* Free the decomposition. */
     if ((ret = PIOc_freedecomp(iosysid, ioid2)))
-        ERR(ret);
+        BAIL(ret);
 
     /* Test each available iotype. */
     for (int fmt = 0; fmt < num_flavors; fmt++)
@@ -263,7 +267,6 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
         int varid[NVAR];
         char data_filename[PIO_MAX_NAME + 1];
         void *my_data;
-        void *my_data_multi;
         void *my_data_norec;
         signed char my_data_byte[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1};
         char my_data_char[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1};
@@ -350,7 +353,7 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
             break;
 #endif /* _NETCDF4 */
         default:
-            ERR(ERR_WRONG);
+            BAIL(ERR_WRONG);
         }
 
         /* Create sample output file. */
@@ -358,119 +361,123 @@ int run_darray_async_test(int iosysid, int my_rank, MPI_Comm test_comm, MPI_Comm
                 piotype);
         if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], data_filename,
                                    NC_CLOBBER)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Find the size of the type. */
         if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Create the data for the darray_multi call by making two
          * copies of the data. */
         if (!(my_data_multi = malloc(2 * type_size * elements_per_pe)))
-            ERR(PIO_ENOMEM);
+            BAIL(PIO_ENOMEM);
         memcpy(my_data_multi, my_data, type_size * elements_per_pe);
         memcpy((char *)my_data_multi + type_size * elements_per_pe, my_data, type_size * elements_per_pe);
 
         /* Define dimensions. */
         for (int d = 0; d < NDIM3; d++)
             if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
-                ERR(ret);
+                BAIL(ret);
 
         /* Define variables. */
         if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid[0])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_def_var(ncid, REC_VAR_NAME2, piotype, NDIM3, dimid, &varid[1])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME, piotype, NDIM2, &dimid[1],
                                 &varid[2])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME2, piotype, NDIM2, &dimid[1],
                                 &varid[3])))
-            ERR(ret);
+            BAIL(ret);
 
         /* End define mode. */
         if ((ret = PIOc_enddef(ncid)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Set the record number for the record vars. */
         if ((ret = PIOc_setframe(ncid, varid[0], 0)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_setframe(ncid, varid[1], 0)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Write some data to the record vars. */
         if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Write some data to the non-record vars. */
         if ((ret = PIOc_write_darray(ncid, varid[2], ioid, elements_per_pe, my_data_norec, NULL)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_write_darray(ncid, varid[3], ioid, elements_per_pe, my_data_norec, NULL)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Sync the file. */
         if ((ret = PIOc_sync(ncid)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Increment the record number for the record vars. */
         if ((ret = PIOc_advanceframe(ncid, varid[0])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_advanceframe(ncid, varid[1])))
-            ERR(ret);
+            BAIL(ret);
 
         /* Write another record. */
         if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Sync the file. */
         if ((ret = PIOc_sync(ncid)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Increment the record number for the record var. */
         if ((ret = PIOc_advanceframe(ncid, varid[0])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_advanceframe(ncid, varid[1])))
-            ERR(ret);
+            BAIL(ret);
 
         /* Write a third record. */
         if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Increment the record number for the record var. */
         if ((ret = PIOc_advanceframe(ncid, varid[0])))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_advanceframe(ncid, varid[1])))
-            ERR(ret);
+            BAIL(ret);
 
         /* Write a forth record, using darray_multi(). */
         int frame[2] = {3, 3};
         if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, 2, elements_per_pe, my_data_multi, frame, NULL, 0)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Close the file. */
         if ((ret = PIOc_closefile(ncid)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Free resources. */
         free(my_data_multi);
+        my_data_multi = NULL;
 
         /* Check the file for correctness. */
         if ((ret = check_darray_file(iosysid, data_filename, PIO_IOTYPE_NETCDF, my_rank, piotype)))
-            ERR(ret);
+            BAIL(ret);
 
     } /* next iotype */
 
     /* Free the decomposition. */
     if ((ret = PIOc_freedecomp(iosysid, ioid)))
-        ERR(ret);
+        BAIL(ret);
 
-    return 0;
+exit:
+    if (my_data_multi)
+        free(my_data_multi);
+    return ret;
 }
 
 /* Run Tests for pio_spmd.c functions. */
@@ -535,7 +542,7 @@ int main(int argc, char **argv)
                     return ret;
 
                 /* Finalize PIO system. */
-                if ((ret = PIOc_finalize(iosysid)))
+                if ((ret = PIOc_free_iosystem (iosysid)))
                     return ret;
 
                 /* Free the computation conomponent communicator. */
diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c
index d993bfaef29a..9af212ce81d8 100644
--- a/tests/cunit/test_darray_async_many.c
+++ b/tests/cunit/test_darray_async_many.c
@@ -62,6 +62,63 @@ int my_type[NTYPE] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT,
 /* Names of the dimensions. */
 char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"time", "vert_level", "lat", "lon"};
 
+int
+check_4d_vars(int my_rank, int ncid, int *varid_4d)
+{
+    void *data_in2 = NULL;
+    int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3};
+    float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5};
+    int ret;
+
+    for (int v = 0; v < NUM_4D_VARS; v++)
+    {
+        int xtype;
+        PIO_Offset size;
+
+        /* Get the type of the 4d var. */
+        if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype)))
+            BAIL(ret);
+
+        /* Get the size of this type. */
+        if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size)))
+            BAIL(ret);
+
+        /* Allocate memory for data. */
+        if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC)))
+            BAIL(PIO_ENOMEM);
+
+        /* Read the data. */
+        if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2)))
+            BAIL(ret);
+
+        /* Check each element of data. */
+        for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++)
+        {
+            switch (xtype)
+            {
+            case PIO_INT:
+                if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)])
+                    BAIL(ERR_WRONG);
+                break;
+            case PIO_FLOAT:
+                if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)])
+                    BAIL(ERR_WRONG);
+                break;
+            default:
+                BAIL(ERR_WRONG);
+            }
+        }
+        free(data_in2);
+        data_in2 = NULL;
+    }
+
+exit:
+    if (data_in2)
+        free(data_in2);
+
+    return ret;
+}
+
 /* Check the file that was created in this test. */
 int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
                       int *rec_varid, int *norec_varid, int num_types, int *varid_4d)
@@ -88,42 +145,40 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
                                                              9223372036854775827ULL, 9223372036854775828ULL,
                                                              9223372036854775837ULL, 9223372036854775838ULL};
 #endif /* _NETCDF4 */
-    int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3};
-    float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5};
+    void *data_in = NULL;
+    void *norec_data_in = NULL;
 
     /* Reopen the file. */
     if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE)))
-        ERR(ret);
+        BAIL(ret);
 
     /* Check metadata. */
     int ndims_in, nvars_in, ngatts_in, unlimdimid_in;
     if ((ret = PIOc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in)))
-        ERR(ret);
+        BAIL(ret);
     if (ndims_in != NDIM4 || nvars_in != num_types * 2 + NUM_4D_VARS || ngatts_in != 0 || unlimdimid_in != 0)
-        ERR(ERR_WRONG);
+        BAIL(ERR_WRONG);
 
     /* Check the vars. */
     for (int t = 0; t < num_types; t++)
     {
-        void *data_in;
-        void *norec_data_in;
         PIO_Offset type_size;
 
         /* Find size of type. */
         if ((ret = PIOc_inq_type(ncid, my_type[t], NULL, &type_size)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Allocate buffers to hold data. */
         if (!(data_in = malloc(LAT_LEN * LON_LEN * NREC * type_size)))
-            ERR(PIO_ENOMEM);
+            BAIL(PIO_ENOMEM);
         if (!(norec_data_in = malloc(LAT_LEN * LON_LEN * type_size)))
-            ERR(PIO_ENOMEM);
+            BAIL(PIO_ENOMEM);
 
         /* Read record and non-record vars for this type. */
         if ((ret = PIOc_get_var(ncid, rec_varid[t], data_in)))
-            ERR(ret);
+            BAIL(ret);
         if ((ret = PIOc_get_var(ncid, norec_varid[t], norec_data_in)))
-            ERR(ret);
+            BAIL(ret);
 
         /* Check each value of non-record data. */
         for (int r = 0; r < LAT_LEN * LON_LEN; r++)
@@ -132,52 +187,52 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
             {
             case PIO_BYTE:
                 if (((signed char *)norec_data_in)[r] != expected_byte[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_CHAR:
                 if (((char *)norec_data_in)[r] != expected_char[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_SHORT:
                 if (((short *)norec_data_in)[r] != expected_short[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_INT:
                 if (((int *)norec_data_in)[r] != expected_int[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_FLOAT:
                 if (((float *)norec_data_in)[r] != expected_float[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_DOUBLE:
                 if (((double *)norec_data_in)[r] != expected_double[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
 #ifdef _NETCDF4
             case PIO_UBYTE:
                 if (((unsigned char *)norec_data_in)[r] != expected_ubyte[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_USHORT:
                 if (((unsigned short *)norec_data_in)[r] != expected_ushort[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_UINT:
                 if (((unsigned int *)norec_data_in)[r] != expected_uint[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_INT64:
                 if (((long long *)norec_data_in)[r] != expected_int64[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_UINT64:
                 if (((unsigned long long *)norec_data_in)[r] != expected_uint64[r])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
 #endif /* _NETCDF4 */
             default:
-                ERR(ERR_WRONG);
+                BAIL(ERR_WRONG);
             }
         }
 
@@ -188,109 +243,76 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank,
             {
             case PIO_BYTE:
                 if (((signed char *)data_in)[r] != expected_byte[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_CHAR:
                 if (((char *)data_in)[r] != expected_char[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_SHORT:
                 if (((short *)data_in)[r] != expected_short[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_INT:
                 if (((int *)data_in)[r] != expected_int[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_FLOAT:
                 if (((float *)data_in)[r] != expected_float[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_DOUBLE:
                 if (((double *)data_in)[r] != expected_double[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
 #ifdef _NETCDF4
             case PIO_UBYTE:
                 if (((unsigned char *)data_in)[r] != expected_ubyte[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_USHORT:
                 if (((unsigned short *)data_in)[r] != expected_ushort[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_UINT:
                 if (((unsigned int *)data_in)[r] != expected_uint[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_INT64:
                 if (((long long *)data_in)[r] != expected_int64[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
             case PIO_UINT64:
                 if (((unsigned long long *)data_in)[r] != expected_uint64[r % (LAT_LEN * LON_LEN)])
-                    ERR(ERR_WRONG);
+                    BAIL(ERR_WRONG);
                 break;
 #endif /* _NETCDF4 */
             default:
-                ERR(ERR_WRONG);
-            }
-        }
-
-        /* Check the 4D vars. */
-        for (int v = 0; v < NUM_4D_VARS; v++)
-        {
-            void *data_in;
-            int xtype;
-            PIO_Offset size;
-
-            /* Get the type of the 4d var. */
-            if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype)))
-                ERR(ret);
-
-            /* Get the size of this type. */
-            if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size)))
-                ERR(ret);
-
-            /* Allocate memory for data. */
-            if (!(data_in = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC)))
-                ERR(PIO_ENOMEM);
-
-            /* Read the data. */
-            if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in)))
-                ERR(ret);
-
-            /* Check each element of data. */
-            for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++)
-            {
-                switch (xtype)
-                {
-                case PIO_INT:
-                    if (((int *)data_in)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)])
-                        ERR(ERR_WRONG);
-                    break;
-                case PIO_FLOAT:
-                    if (((float *)data_in)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)])
-                        ERR(ERR_WRONG);
-                    break;
-                default:
-                    ERR(ERR_WRONG);
-                }
+                BAIL(ERR_WRONG);
             }
-
-            /* Release memory. */
-            free(data_in);
         }
 
+        /* Free memory. */
         free(data_in);
+        data_in = NULL;
         free(norec_data_in);
+        norec_data_in = NULL;
+
+        /* Check the 4D vars. */
+        if ((ret = check_4d_vars(my_rank, ncid, varid_4d)))
+            BAIL(ret);
     }
 
     /* Close the file. */
     if ((ret = PIOc_closefile(ncid)))
         ERR(ret);
 
-    return 0;
+exit:
+    if (data_in)
+        free(data_in);
+    if (norec_data_in)
+        free(norec_data_in);
+    return ret;
 }
 
 /* Run a simple test using darrays with async. */
@@ -620,7 +642,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
 
             /* Free the computation conomponent communicator. */
diff --git a/tests/cunit/test_darray_async_simple.c b/tests/cunit/test_darray_async_simple.c
index c5a8f0f5d16c..1947961472c2 100644
--- a/tests/cunit/test_darray_async_simple.c
+++ b/tests/cunit/test_darray_async_simple.c
@@ -192,7 +192,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
 
             /* Free the computation conomponent communicator. */
diff --git a/tests/cunit/test_darray_fill.c b/tests/cunit/test_darray_fill.c
new file mode 100644
index 000000000000..0dafa22899bd
--- /dev/null
+++ b/tests/cunit/test_darray_fill.c
@@ -0,0 +1,376 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date 4/21/18
+ */
+#include 
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_fill"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 4
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+#define VAR_NAME "PIO_TF_test_var"
+#define DIM_NAME "PIO_TF_test_dim"
+#define FILL_VALUE_NAME "_FillValue"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+#define NDIM1 1
+#define MAPLEN 7
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM1] = {28};
+
+/* Run test for each of the rearrangers. */
+#define NUM_REARRANGERS_TO_TEST 2
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, -1, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int wioid, rioid;
+        int maplen = MAPLEN;
+        MPI_Offset wcompmap[MAPLEN];
+        MPI_Offset rcompmap[MAPLEN];
+        int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+
+        /* Data we will write for each type. */
+        signed char byte_data[MAPLEN];
+        char char_data[MAPLEN];
+        short short_data[MAPLEN];
+        int int_data[MAPLEN];
+        float float_data[MAPLEN];
+        double double_data[MAPLEN];
+#ifdef _NETCDF4
+        unsigned char ubyte_data[MAPLEN];
+        unsigned short ushort_data[MAPLEN];
+        unsigned int uint_data[MAPLEN];
+        long long int64_data[MAPLEN];
+        unsigned long long uint64_data[MAPLEN];
+#endif /* _NETCDF4 */
+
+        /* Expected results for each type. */
+        signed char byte_expected[MAPLEN];
+        char char_expected[MAPLEN];
+        short short_expected[MAPLEN];
+        int int_expected[MAPLEN];
+        float float_expected[MAPLEN];
+        double double_expected[MAPLEN];
+#ifdef _NETCDF4
+        unsigned char ubyte_expected[MAPLEN];
+        unsigned short ushort_expected[MAPLEN];
+        unsigned int uint_expected[MAPLEN];
+        long long int64_expected[MAPLEN];
+        unsigned long long uint64_expected[MAPLEN];
+#endif /* _NETCDF4 */
+
+        /* Custom fill value for each type. */
+        signed char byte_fill = -2;
+        char char_fill = 2;
+        short short_fill = -2;
+        int int_fill = -2;
+        float float_fill = -2;
+        double double_fill = -2;
+#ifdef _NETCDF4
+        unsigned char ubyte_fill = 2;
+        unsigned short ushort_fill = 2;
+        unsigned int uint_fill = 2;
+        long long int64_fill = 2;
+        unsigned long long uint64_fill = 2;
+#endif /* _NETCDF4 */
+
+        /* Default fill value for each type. */
+        signed char byte_default_fill = NC_FILL_BYTE;
+        char char_default_fill = NC_FILL_CHAR;
+        short short_default_fill = NC_FILL_SHORT;
+        int int_default_fill = NC_FILL_INT;
+        float float_default_fill = NC_FILL_FLOAT;
+        double double_default_fill = NC_FILL_DOUBLE;
+#ifdef _NETCDF4
+        unsigned char ubyte_default_fill = NC_FILL_UBYTE;
+        unsigned short ushort_default_fill = NC_FILL_USHORT;
+        unsigned int uint_default_fill = NC_FILL_UINT;
+        long long int64_default_fill = NC_FILL_INT64;
+        unsigned long long uint64_default_fill = NC_FILL_UINT64;
+#endif /* _NETCDF4 */
+
+        int ret;      /* Return code. */
+
+        /* Set up the compmaps. Don't forget these are 1-based
+         * numbers, like in Fortran! */
+        for (int i = 0; i < MAPLEN; i++)
+        {
+            wcompmap[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : 0; /* Even values missing. */
+            rcompmap[i] = my_rank * MAPLEN + i + 1;
+        }
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Test for each rearranger. */
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO_PROCS, ioproc_stride, ioproc_start,
+                                           rearranger[r], &iosysid)))
+                return ret;
+
+            /* Test with and without custom fill values. */
+            for (int fv = 0; fv < NUM_TEST_CASES_FILLVALUE; fv++)
+            {
+#ifndef _NETCDF4
+#define NUM_TYPES 6
+                int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+#else
+#define NUM_TYPES 11
+                int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                            PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
+                
+#endif /* _NETCDF4 */
+
+                /* Determine what data to write. Put value of 42 into
+                 * array elements that will not get written. Due to
+                 * the decomposition, these will be replaced by fill
+                 * values. */
+                for (int i = 0; i < MAPLEN; i++)
+                {
+                    byte_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    char_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    short_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    int_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    float_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    double_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+#ifdef _NETCDF4
+                    ubyte_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    ushort_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    uint_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    int64_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    uint64_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+#endif /* _NETCDF4 */
+                }
+
+                /* Determine what data to expect from the test. For
+                 * even values of i, the fill value will be used, and
+                 * it may be custom or default fill value. */
+                for (int i = 0; i < MAPLEN; i++)
+                {
+                    byte_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? byte_default_fill : byte_fill);
+                    char_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? char_default_fill : char_fill);
+                    short_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? short_default_fill : short_fill);
+                    int_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? int_default_fill : int_fill);
+                    float_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? float_default_fill : float_fill);
+                    double_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? double_default_fill : double_fill);
+#ifdef _NETCDF4
+                    ubyte_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? ubyte_default_fill : ubyte_fill);
+                    ushort_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? ushort_default_fill : ushort_fill);
+                    uint_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? uint_default_fill : uint_fill);
+                    int64_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? int64_default_fill : int64_fill);
+                    uint64_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? uint64_default_fill : uint64_fill);
+#endif /* _NETCDF4 */
+                }
+
+                /* Test for each available type. */
+                for (int t = 0; t < NUM_TYPES; t++)
+                {
+                    void *expected;
+                    void *fill;
+                    void *data;
+                    int ncid, dimid, varid;
+                    char filename[NC_MAX_NAME + 1];
+
+                    switch (test_type[t])
+                    {
+                    case PIO_BYTE:
+                        expected = byte_expected;
+                        fill = fv ? &byte_default_fill : &byte_fill;
+                        data = byte_data;
+                        break;
+                    case PIO_CHAR:
+                        expected = char_expected;
+                        fill = fv ? &char_default_fill : &char_fill;
+                        data = char_data;
+                        break;
+                    case PIO_SHORT:
+                        expected = short_expected;
+                        fill = fv ? &short_default_fill : &short_fill;
+                        data = short_data;
+                        break;
+                    case PIO_INT:
+                        expected = int_expected;
+                        fill = fv ? &int_default_fill : &int_fill;
+                        data = int_data;
+                        break;
+                    case PIO_FLOAT:
+                        expected = float_expected;
+                        fill = fv ? &float_default_fill : &float_fill;
+                        data = float_data;
+                        break;
+                    case PIO_DOUBLE:
+                        expected = double_expected;
+                        fill = fv ? &double_default_fill : &double_fill;
+                        data = double_data;
+                        break;
+#ifdef _NETCDF4
+                    case PIO_UBYTE:
+                        expected = ubyte_expected;
+                        fill = fv ? &ubyte_default_fill : &ubyte_fill;
+                        data = ubyte_data;
+                        break;
+                    case PIO_USHORT:
+                        expected = ushort_expected;
+                        fill = fv ? &ushort_default_fill : &ushort_fill;
+                        data = ushort_data;
+                        break;
+                    case PIO_UINT:
+                        expected = uint_expected;
+                        fill = fv ? &uint_default_fill : &uint_fill;
+                        data = uint_data;
+                        break;
+                    case PIO_INT64:
+                        expected = int64_expected;
+                        fill = fv ? &int64_default_fill : &int64_fill;
+                        data = int64_data;
+                        break;
+                    case PIO_UINT64:
+                        expected = uint64_expected;
+                        fill = fv ? &uint64_default_fill : &uint64_fill;
+                        data = uint64_data;
+                        break;
+#endif /* _NETCDF4 */
+                    default:
+                        return ERR_AWFUL;
+                    }
+
+                    /* Initialize decompositions. */
+                    if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap,
+                                               &wioid, &rearranger[r], NULL, NULL)))
+                        return ret;
+                    if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap,
+                                               &rioid, &rearranger[r], NULL, NULL)))
+                        return ret;
+
+                    /* Create the test file in each of the available iotypes. */
+                    for (int fmt = 0; fmt < num_flavors; fmt++)
+                    {
+                        PIO_Offset type_size;
+                        void *data_in;
+
+                        /* Byte type doesn't work with pnetcdf. */
+                        if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (test_type[t] == PIO_BYTE || test_type[t] == PIO_CHAR))
+                            continue;
+
+                        /* NetCDF-4 types only work with netCDF-4 formats. */
+                        if (test_type[t] > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+                            flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+                            continue;
+
+                        /* Put together filename. */
+                        sprintf(filename, "%s_iotype_%d_rearr_%d_type_%d.nc", TEST_NAME, flavor[fmt],
+                                rearranger[r], test_type[t]);
+
+                        /* Create file. */
+                        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER)))
+                            return ret;
+
+                        /* Define metadata. */
+                        if ((ret = PIOc_def_dim(ncid, DIM_NAME, dim_len[0], &dimid)))
+                            return ret;
+                        if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid)))
+                            return ret;
+                        if ((ret = PIOc_put_att(ncid, varid, FILL_VALUE_NAME, test_type[t],
+                                                1, fill)))
+                            return ret;
+                        if ((ret = PIOc_enddef(ncid)))
+                            return ret;
+
+                        /* Write some data. */
+                        if ((ret = PIOc_write_darray(ncid, varid, wioid, MAPLEN, data, fill)))
+                            return ret;
+                        if ((ret = PIOc_sync(ncid)))
+                            return ret;
+
+                        /* What is size of type? */
+                        if ((ret = PIOc_inq_type(ncid, test_type[t], NULL, &type_size)))
+                            return ret;
+
+                        /* Allocate space to read data into. */
+                        if (!(data_in = malloc(type_size * MAPLEN)))
+                            return PIO_ENOMEM;
+
+                        /* Read the data. */
+                        if ((ret = PIOc_read_darray(ncid, varid, rioid, MAPLEN, data_in)))
+                            return ret;
+
+                        /* Check results. */
+                        if (memcmp(data_in, expected, type_size * MAPLEN))
+                            return ERR_AWFUL;
+
+                        /* Release storage. */
+                        free(data_in);
+
+                        /* Close file. */
+                        if ((ret = PIOc_closefile(ncid)))
+                            return ret;
+                    } /* next iotype */
+
+                    /* Free decompositions. */
+                    if ((ret = PIOc_freedecomp(iosysid, wioid)))
+                        return ret;
+                    if ((ret = PIOc_freedecomp(iosysid, rioid)))
+                        return ret;
+
+                } /* next type */
+            } /* next fill value test case */
+
+            /* Finalize PIO iosysid. */
+            if ((ret = PIOc_free_iosystem(iosysid)))
+                return ret;
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/tests/cunit/test_darray_frame.c b/tests/cunit/test_darray_frame.c
index 38c7e99e8a4d..949e90ebfb4f 100644
--- a/tests/cunit/test_darray_frame.c
+++ b/tests/cunit/test_darray_frame.c
@@ -37,7 +37,7 @@
 #define LON_LEN_SHORT 2
 
 /* The number of timesteps of data to write. */
-#define NUM_TIMESTEPS 2
+#define NUM_TIMESTEPS 3
 
 /* The names of variable in the netCDF output files. */
 #define VAR_NAME "prc"
@@ -120,7 +120,7 @@ int test_frame_simple(int iosysid, int num_iotypes, int *iotype, int my_rank,
             ERR(ret);
 
         /* Write records of data. */
-        for (int r = 0; r < TIME_LEN_SHORT; r++)
+        for (int r = 0; r < NUM_TIMESTEPS; r++)
         {
             int test_data_int[elements_per_pe];
 
@@ -225,7 +225,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
         } /* next rearranger */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_darray_multi.c b/tests/cunit/test_darray_multi.c
index 31fe313ff948..2784ee0c53a6 100644
--- a/tests/cunit/test_darray_multi.c
+++ b/tests/cunit/test_darray_multi.c
@@ -84,7 +84,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
     int ncid2;             /* The ncid of the re-opened netCDF file. */
     int varid[NVAR];       /* The IDs of the netCDF varables. */
     int other_varid;       /* The IDs of a var of different type. */
-    int wrong_varid[NVAR]; /* These will not work. */
     PIO_Offset arraylen = 4; /* Amount of data from each task. */
     void *fillvalue;       /* Pointer to fill value. */
     void *test_data;       /* Pointer to test data we will write. */
@@ -279,14 +278,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
                 int frame[NVAR] = {0, 0, 0};
                 int flushtodisk = test_multi;
 
-                /* This will not work, because we mix var types. */
-                wrong_varid[0] = varid[0];
-                wrong_varid[1] = varid[1];
-                wrong_varid[0] = other_varid;
-                if (PIOc_write_darray_multi(ncid, wrong_varid, ioid, NVAR, arraylen, test_data, frame,
-                                            fillvalue, flushtodisk) != PIO_EINVAL)
-                    ERR(ERR_WRONG);
-
                 /* Write the data with the _multi function. */
                 if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame,
                                                    fillvalue, flushtodisk)))
@@ -401,7 +392,7 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
     int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
 #endif /* _NETCDF4 */
     int ioid;
-    char filename[NC_MAX_NAME + 1];
+    char filename[PIO_MAX_NAME + 1];
     int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
     int ret; /* Return code. */
 
@@ -473,7 +464,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
         } /* next rearranger */
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_darray_multivar.c b/tests/cunit/test_darray_multivar.c
index 0c5ecc616e85..5a38cb292f15 100644
--- a/tests/cunit/test_darray_multivar.c
+++ b/tests/cunit/test_darray_multivar.c
@@ -587,7 +587,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
         }
 
diff --git a/tests/cunit/test_darray_multivar2.c b/tests/cunit/test_darray_multivar2.c
index 0637ed9c66d3..377f77a1b299 100644
--- a/tests/cunit/test_darray_multivar2.c
+++ b/tests/cunit/test_darray_multivar2.c
@@ -275,7 +275,7 @@ int main(int argc, char **argv)
             return ret;
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             return ret;
 
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_darray_multivar3.c b/tests/cunit/test_darray_multivar3.c
index 8ea5681772de..8f2528085229 100644
--- a/tests/cunit/test_darray_multivar3.c
+++ b/tests/cunit/test_darray_multivar3.c
@@ -106,7 +106,7 @@ int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
                 if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
                     ERR(ret);
 
-            /* Var 0 does not have a record dim, varid 1 is a record var. */
+            /* Var 0 does not have a record dim, varid 1 and 2 are record vars. */
             if ((ret = PIOc_def_var(ncid, var_name[0], PIO_INT, NDIM - 1, &dimids[1], &varid[0])))
                 ERR(ret);
             if ((ret = PIOc_def_var(ncid, var_name[1], PIO_INT, NDIM, dimids, &varid[1])))
@@ -115,11 +115,11 @@ int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
                 ERR(ret);
 
             /* Set the custom fill values. */
-            if ((ret = PIOc_def_var_fill(ncid, varid[0], 0, &custom_fillvalue_int))) 
+            if ((ret = PIOc_def_var_fill(ncid, varid[0], 0, &custom_fillvalue_int)))
                 ERR(ret);
-            if ((ret = PIOc_def_var_fill(ncid, varid[1], 0, &custom_fillvalue_int))) 
+            if ((ret = PIOc_def_var_fill(ncid, varid[1], 0, &custom_fillvalue_int)))
                 ERR(ret);
-            if ((ret = PIOc_def_var_fill(ncid, varid[2], 0, &custom_fillvalue_float))) 
+            if ((ret = PIOc_def_var_fill(ncid, varid[2], 0, &custom_fillvalue_float)))
                 ERR(ret);
 
             /* End define mode. */
@@ -133,11 +133,11 @@ int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
                 ERR(ret);
 
             int *fvp_int = NULL;
-            float *fvp_float = NULL;
+            /* float *fvp_float = NULL; */
             if (use_fv)
             {
                 fvp_int = &custom_fillvalue_int;
-                fvp_float = &custom_fillvalue_float;
+                /* fvp_float = &custom_fillvalue_float; */
             }
 
             /* Write the data. */
@@ -148,20 +148,19 @@ int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
                                          fvp_int)))
                 ERR(ret);
 
-            /* This should not work, since the type of the var is
-             * PIO_FLOAT, and the type if the decomposition is
-             * PIO_INT. */
-            if (PIOc_write_darray(ncid, varid[2], ioid, arraylen, test_data_float,
-                                  fvp_float) != PIO_EINVAL)
-                ERR(ERR_WRONG);
+            /* This should not work since we cannot mix record and not
+             * record vars. */
+	    int frame[NUM_VAR] = {0, 0, 0};
 
-            /* This should also fail, because it mixes an int and a
-             * float. */
-            int frame[NUM_VAR] = {0, 0, 0};
             if (PIOc_write_darray_multi(ncid, varid, ioid, NUM_VAR, arraylen * NUM_VAR, test_data_float,
-                                        frame, NULL, 0) != PIO_EINVAL)
+                                        frame, NULL, 0) != PIO_EVARDIMMISMATCH)
                 ERR(ERR_WRONG);
 
+	    /* This should work since int and float are the same size
+             * and both are record vars. */
+            if ((ret = PIOc_write_darray_multi(ncid, varid+1, ioid, NUM_VAR-1, arraylen * (NUM_VAR-1), test_data_float,
+					       frame, NULL, 0)))
+                ERR(ret);
 
             /* Close the netCDF file. */
             if ((ret = PIOc_closefile(ncid)))
@@ -277,11 +276,8 @@ int main(int argc, char **argv)
 {
     int my_rank;
     int ntasks;
-    int num_flavors;         /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
     MPI_Comm test_comm;      /* A communicator for this test. */
     int ioid;
-    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
     int ret;                 /* Return code. */
 
     /* Initialize test. */
@@ -298,7 +294,9 @@ int main(int argc, char **argv)
         int iosysid;              /* The ID for the parallel I/O system. */
         int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
         int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
-        int ret;                  /* Return code. */
+	int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+	int num_flavors;         /* Number of PIO netCDF flavors in this build. */
+	int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
 
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
@@ -325,7 +323,7 @@ int main(int argc, char **argv)
             ERR(ret);
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             return ret;
 
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c
new file mode 100644
index 000000000000..1a94ecdf3010
--- /dev/null
+++ b/tests/cunit/test_darray_vard.c
@@ -0,0 +1,523 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date 6/4/19
+ */
+#include 
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_vard"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The names of variables in the netCDF output files. */
+#define VAR_NAME "Billy-Bob"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+#define NUM_TYPES_TO_TEST 6
+
+/**
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param fmt the index of the IOTYPE to test.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_darray(int iosysid, int ioid, int fmt, int num_flavors,
+                int *flavor, int my_rank, int pio_type)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+    int type_to_use;
+    PIO_Offset arraylen = 4;
+    char test_data_char[arraylen];
+    char test_data_char_in[arraylen];
+    signed char test_data_byte[arraylen];
+    signed char test_data_byte_in[arraylen];
+    short test_data_short[arraylen];
+    short test_data_short_in[arraylen];
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen];
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+    unsigned char test_data_ubyte[arraylen];
+    unsigned char test_data_ubyte_in[arraylen];
+    unsigned short test_data_ushort[arraylen];
+    unsigned short test_data_ushort_in[arraylen];
+    unsigned int test_data_uint[arraylen];
+    unsigned int test_data_uint_in[arraylen];
+    long long int test_data_int64[arraylen];
+    long long int test_data_int64_in[arraylen];
+    unsigned long long int test_data_uint64[arraylen];
+    unsigned long long int test_data_uint64_in[arraylen];
+    int f, d;
+
+    /* Initialize some data. */
+    for (f = 0; f < arraylen; f++)
+    {
+        test_data_char[f] = my_rank;
+        test_data_byte[f] = my_rank - f;
+        test_data_short[f] = my_rank + f;
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+        test_data_ubyte[f] = my_rank + f + 2;
+        test_data_ushort[f] = my_rank + f + 20;
+        test_data_uint[f] = my_rank + f + 200;
+        test_data_int64[f] = my_rank - f - 20000;
+        test_data_uint64[f] = my_rank + f + 20000;
+    }
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    {
+        /* Pnetcdf cannot handle 1-byte types. */
+        if (fmt == 0 && (pio_type == PIO_BYTE || pio_type == PIO_CHAR))
+            return PIO_NOERR;
+
+        {
+            /* Create the filename. */
+            sprintf(filename, "%s_iotype_%d_pio_type_%d.nc",
+                    TEST_NAME, flavor[fmt], pio_type);
+
+            /* Create the netCDF output file. */
+            if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename,
+                                       PIO_CLOBBER)))
+                ERR(ret);
+
+            /* Define netCDF dimensions and variable. */
+            for (d = 0; d < NDIM; d++)
+                if ((ret = PIOc_def_dim(ncid, dim_name[d],
+                                        (PIO_Offset)dim_len[d], &dimids[d])))
+                    ERR(ret);
+
+            /* Define a variable. */
+            type_to_use = (pio_type == NC_NAT) ? PIO_INT : pio_type;
+            if ((ret = PIOc_def_var(ncid, VAR_NAME, type_to_use, NDIM, dimids,
+                                    &varid)))
+                ERR(ret);
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                ERR(ret);
+
+
+            switch (pio_type)
+            {
+            case PIO_CHAR:
+                /* These should not work. */
+                if (PIOc_put_vard_text(ncid + TEST_VAL_42, varid, ioid, 0,
+                                       test_data_char) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_put_vard_text(ncid, varid + TEST_VAL_42, ioid, 0,
+                                       test_data_char) != PIO_ENOTVAR)
+                    ERR(ERR_WRONG);
+                if (PIOc_put_vard_text(ncid, varid, ioid + TEST_VAL_42, 0,
+                                       test_data_char) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+
+                /* This will work. */
+                if ((ret = PIOc_put_vard_text(ncid, varid, ioid, 0,
+                                              test_data_char)))
+                    ERR(ret);
+                break;
+            case PIO_BYTE:
+                if ((ret = PIOc_put_vard_schar(ncid, varid, ioid, 0,
+                                               test_data_byte)))
+                    ERR(ret);
+                break;
+            case PIO_SHORT:
+                if ((ret = PIOc_put_vard_short(ncid, varid, ioid, 0,
+                                               test_data_short)))
+                    ERR(ret);
+                break;
+            case PIO_INT:
+                if ((ret = PIOc_put_vard_int(ncid, varid, ioid, 0,
+                                             test_data_int)))
+                    ERR(ret);
+                break;
+            case PIO_FLOAT:
+                if ((ret = PIOc_put_vard_float(ncid, varid, ioid, 0,
+                                               test_data_float)))
+                    ERR(ret);
+                break;
+            case PIO_DOUBLE:
+                if ((ret = PIOc_put_vard_double(ncid, varid, ioid, 0,
+                                                test_data_double)))
+                    ERR(ret);
+                break;
+            case PIO_UBYTE:
+                if ((ret = PIOc_put_vard_uchar(ncid, varid, ioid, 0,
+                                               test_data_ubyte)))
+                    ERR(ret);
+                break;
+            case PIO_USHORT:
+                if ((ret = PIOc_put_vard_ushort(ncid, varid, ioid, 0,
+                                                test_data_ushort)))
+                    ERR(ret);
+                break;
+            case PIO_UINT:
+                if ((ret = PIOc_put_vard_uint(ncid, varid, ioid, 0,
+                                              test_data_uint)))
+                    ERR(ret);
+                break;
+            case PIO_INT64:
+                if ((ret = PIOc_put_vard_longlong(ncid, varid, ioid, 0,
+                                                  test_data_int64)))
+                    ERR(ret);
+                break;
+            case PIO_UINT64:
+                if ((ret = PIOc_put_vard_ulonglong(ncid, varid, ioid, 0,
+                                                   test_data_uint64)))
+                    ERR(ret);
+                break;
+            case NC_NAT:
+                /* Using NAT to test void * version, using int data. */
+                if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data_int)))
+                    ERR(ret);
+                break;
+            default:
+                ERR(ERR_WRONG);
+            }
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename,
+                                     PIO_NOWRITE)))
+                ERR(ret);
+
+            PIO_Offset dimlen;
+
+            /* check the unlimited dim size - it should be 1 */
+            if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen)))
+                ERR(ret);
+            if (dimlen != 1)
+                ERR(ERR_WRONG);
+
+            /* Read the data. */
+            switch (pio_type)
+            {
+            case PIO_CHAR:
+                /* These should not work. */
+                if ((ret = PIOc_get_vard_text(ncid2 + TEST_VAL_42, varid, ioid, 0,
+                                              test_data_char_in)) != PIO_EBADID)
+                    ERR(ret);
+                if ((ret = PIOc_get_vard_text(ncid2, varid, ioid + TEST_VAL_42, 0,
+                                              test_data_char_in)) != PIO_EBADID)
+                    ERR(ret);
+
+                /* This will work. */
+                if ((ret = PIOc_get_vard_text(ncid2, varid, ioid, 0,
+                                              test_data_char_in)))
+                    ERR(ret);
+                break;
+            case PIO_BYTE:
+                if ((ret = PIOc_get_vard_schar(ncid2, varid, ioid, 0,
+                                               test_data_byte_in)))
+                    ERR(ret);
+                break;
+            case PIO_SHORT:
+                if ((ret = PIOc_get_vard_short(ncid2, varid, ioid, 0,
+                                               test_data_short_in)))
+                    ERR(ret);
+                break;
+            case PIO_INT:
+                if ((ret = PIOc_get_vard_int(ncid2, varid, ioid, 0,
+                                             test_data_int_in)))
+                    ERR(ret);
+                break;
+            case PIO_FLOAT:
+                if ((ret = PIOc_get_vard_float(ncid2, varid, ioid, 0,
+                                               test_data_float_in)))
+                    ERR(ret);
+                break;
+            case PIO_DOUBLE:
+                if ((ret = PIOc_get_vard_double(ncid2, varid, ioid, 0,
+                                                test_data_double_in)))
+                    ERR(ret);
+                break;
+            case PIO_UBYTE:
+                if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid, 0,
+                                               test_data_ubyte_in)))
+                    ERR(ret);
+                break;
+            case PIO_USHORT:
+                if ((ret = PIOc_get_vard_ushort(ncid2, varid, ioid, 0,
+                                                test_data_ushort_in)))
+                    ERR(ret);
+                break;
+            case PIO_UINT:
+                if ((ret = PIOc_get_vard_uint(ncid2, varid, ioid, 0,
+                                              test_data_uint_in)))
+                    ERR(ret);
+                break;
+            case PIO_INT64:
+                if ((ret = PIOc_get_vard_longlong(ncid2, varid, ioid, 0,
+                                                  test_data_int64_in)))
+                    ERR(ret);
+                break;
+            case PIO_UINT64:
+                if ((ret = PIOc_get_vard_ulonglong(ncid2, varid, ioid, 0,
+                                                   test_data_uint64_in)))
+                    ERR(ret);
+                break;
+            case NC_NAT:
+                /* Using NAT to test void * version, using int data. */
+                if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0,
+                                         test_data_int_in)))
+                    ERR(ret);
+                break;
+            default:
+                ERR(ERR_WRONG);
+            }
+
+            /* Check the results. */
+            for (f = 0; f < arraylen; f++)
+            {
+                switch (pio_type)
+                {
+                case PIO_CHAR:
+                    if (test_data_char_in[f] != test_data_char[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_BYTE:
+                    if (test_data_byte_in[f] != test_data_byte[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_SHORT:
+                    if (test_data_short_in[f] != test_data_short[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_INT:
+                    if (test_data_int_in[f] != test_data_int[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_FLOAT:
+                    if (test_data_float_in[f] != test_data_float[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_DOUBLE:
+                    if (test_data_double_in[f] != test_data_double[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_UBYTE:
+                    if (test_data_ubyte_in[f] != test_data_ubyte[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_USHORT:
+                    if (test_data_ushort_in[f] != test_data_ushort[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_UINT:
+                    if (test_data_uint_in[f] != test_data_uint[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_INT64:
+                    if (test_data_int64_in[f] != test_data_int64[f])
+                        return ERR_WRONG;
+                    break;
+                case PIO_UINT64:
+                    if (test_data_uint64_in[f] != test_data_uint64[f])
+                        return ERR_WRONG;
+                    break;
+                case NC_NAT:
+                    /* Using NAT to test void * version, using int data. */
+                    if (test_data_int_in[f] != test_data_int[f])
+                        return ERR_WRONG;
+                    break;
+                default:
+                    ERR(ERR_WRONG);
+                }
+            }
+
+            /* Try to write, but it won't work, because we opened file
+             * read-only. */
+            if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data_char,
+                                  NULL) != PIO_EPERM)
+                ERR(ERR_WRONG);
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        } /* next fillvalue test case */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param fmt index into array of IOTYPEs.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor,
+                    int my_rank, MPI_Comm test_comm)
+{
+    int ioid;
+    char filename[PIO_MAX_NAME + 1];
+    int pio_type[NUM_NETCDF4_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT,
+                                       PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT,
+                                       PIO_UINT, PIO_INT64, PIO_UINT64, NC_NAT};
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int num_types;
+    int t;
+    int ret; /* Return code. */
+
+    /* Based on the IOTYPE, decide how many types to check. */
+    if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        num_types = NUM_NETCDF4_TYPES;
+    else
+        num_types = NUM_CLASSIC_TYPES;
+
+    /* Check each type. */
+    for (t = 0; t < num_types; t++)
+    {
+        int type_to_use;
+
+        /* Using NAT to test generic versions of vard functions, so
+         * substiture PIO_INT. */
+        type_to_use = (pio_type[t] == NC_NAT) ? PIO_INT : pio_type[t];
+
+        /* This will be our file name for writing out decompositions. */
+        sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc",
+                TEST_NAME, my_rank, *flavor, pio_type[t]);
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid,
+                                           dim_len_2d, &ioid, type_to_use)))
+            return ret;
+
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, fmt, num_flavors, flavor,
+                               my_rank, pio_type[t])))
+            return ret;
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, -1, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT,
+                                                PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first I/O task. */
+        int r;
+        int fmt;
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        for (fmt = 0; fmt < num_flavors; fmt++)
+        {
+            for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+            {
+                /* Initialize the PIO IO system. This specifies how
+                 * many and which processors are involved in I/O. */
+                if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS,
+                                               ioproc_stride, ioproc_start,
+                                               rearranger[r], &iosysid)))
+                    return ret;
+
+                /* Run tests. */
+                if ((ret = test_all_darray(iosysid, fmt, num_flavors, flavor,
+                                           my_rank, test_comm)))
+                    return ret;
+
+                /* Finalize PIO system. */
+                if ((ret = PIOc_free_iosystem(iosysid)))
+                    return ret;
+            } /* next rearranger */
+        }
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/tests/cunit/test_decomp_frame.c b/tests/cunit/test_decomp_frame.c
new file mode 100644
index 000000000000..54969a8b4c16
--- /dev/null
+++ b/tests/cunit/test_decomp_frame.c
@@ -0,0 +1,378 @@
+/*
+ * Tests for PIO distributed arrays. This code duplicates the code in
+ * the fortran test pio_decomp_frame_tests.F90.
+ *
+ * @author Ed Hartnett
+ * @date 5/7/18
+ */
+#include 
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_decomp_frame"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 4
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+#define VAR_NAME "PIO_TF_test_var"
+#define DIM_NAME "PIO_TF_test_dim"
+#define FILL_VALUE_NAME "_FillValue"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+#define NDIM1 1
+#define MAPLEN 7
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM1] = {28};
+
+/* Run test for each of the rearrangers. */
+#define NUM_REARRANGERS_TO_TEST 2
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, -1, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int wioid, rioid;
+        int maplen = MAPLEN;
+        MPI_Offset wcompmap[MAPLEN];
+        MPI_Offset rcompmap[MAPLEN];
+        int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+
+        /* Data we will write for each type. */
+        signed char byte_data[MAPLEN];
+        char char_data[MAPLEN];
+        short short_data[MAPLEN];
+        int int_data[MAPLEN];
+        float float_data[MAPLEN];
+        double double_data[MAPLEN];
+#ifdef _NETCDF4
+        unsigned char ubyte_data[MAPLEN];
+        unsigned short ushort_data[MAPLEN];
+        unsigned int uint_data[MAPLEN];
+        long long int64_data[MAPLEN];
+        unsigned long long uint64_data[MAPLEN];
+#endif /* _NETCDF4 */
+
+        /* Expected results for each type. */
+        signed char byte_expected[MAPLEN];
+        char char_expected[MAPLEN];
+        short short_expected[MAPLEN];
+        int int_expected[MAPLEN];
+        float float_expected[MAPLEN];
+        double double_expected[MAPLEN];
+#ifdef _NETCDF4
+        unsigned char ubyte_expected[MAPLEN];
+        unsigned short ushort_expected[MAPLEN];
+        unsigned int uint_expected[MAPLEN];
+        long long int64_expected[MAPLEN];
+        unsigned long long uint64_expected[MAPLEN];
+#endif /* _NETCDF4 */
+
+        /* Custom fill value for each type. */
+        signed char byte_fill = -2;
+        char char_fill = 2;
+        short short_fill = -2;
+        int int_fill = -2;
+        float float_fill = -2;
+        double double_fill = -2;
+#ifdef _NETCDF4
+        unsigned char ubyte_fill = 2;
+        unsigned short ushort_fill = 2;
+        unsigned int uint_fill = 2;
+        long long int64_fill = 2;
+        unsigned long long uint64_fill = 2;
+#endif /* _NETCDF4 */
+
+        /* Default fill value for each type. */
+        signed char byte_default_fill = NC_FILL_BYTE;
+        char char_default_fill = NC_FILL_CHAR;
+        short short_default_fill = NC_FILL_SHORT;
+        int int_default_fill = NC_FILL_INT;
+        float float_default_fill = NC_FILL_FLOAT;
+        double double_default_fill = NC_FILL_DOUBLE;
+#ifdef _NETCDF4
+        unsigned char ubyte_default_fill = NC_FILL_UBYTE;
+        unsigned short ushort_default_fill = NC_FILL_USHORT;
+        unsigned int uint_default_fill = NC_FILL_UINT;
+        long long int64_default_fill = NC_FILL_INT64;
+        unsigned long long uint64_default_fill = NC_FILL_UINT64;
+#endif /* _NETCDF4 */
+
+        int ret;      /* Return code. */
+
+        /* Set up the compmaps. Don't forget these are 1-based
+         * numbers, like in Fortran! */
+        for (int i = 0; i < MAPLEN; i++)
+        {
+            wcompmap[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : 0; /* Even values missing. */
+            rcompmap[i] = my_rank * MAPLEN + i + 1;
+        }
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Test for each rearranger. */
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO_PROCS, ioproc_stride, ioproc_start,
+                                           rearranger[r], &iosysid)))
+                return ret;
+
+            /* Test with and without custom fill values. */
+            for (int fv = 0; fv < NUM_TEST_CASES_FILLVALUE; fv++)
+            {
+#ifndef _NETCDF4
+#define NUM_TYPES 6
+                int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+#else
+#define NUM_TYPES 11
+                int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                            PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
+                
+#endif /* _NETCDF4 */
+
+                /* Determine what data to write. Put value of 42 into
+                 * array elements that will not get written. Due to
+                 * the decomposition, these will be replaced by fill
+                 * values. */
+                for (int i = 0; i < MAPLEN; i++)
+                {
+                    byte_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    char_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    short_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    int_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    float_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    double_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+#ifdef _NETCDF4
+                    ubyte_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    ushort_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    uint_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    int64_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+                    uint64_data[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : TEST_VAL_42;
+#endif /* _NETCDF4 */
+                }
+
+                /* Determine what data to expect from the test. For
+                 * even values of i, the fill value will be used, and
+                 * it may be custom or default fill value. */
+                for (int i = 0; i < MAPLEN; i++)
+                {
+                    byte_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? byte_default_fill : byte_fill);
+                    char_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? char_default_fill : char_fill);
+                    short_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? short_default_fill : short_fill);
+                    int_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? int_default_fill : int_fill);
+                    float_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? float_default_fill : float_fill);
+                    double_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? double_default_fill : double_fill);
+#ifdef _NETCDF4
+                    ubyte_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? ubyte_default_fill : ubyte_fill);
+                    ushort_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? ushort_default_fill : ushort_fill);
+                    uint_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? uint_default_fill : uint_fill);
+                    int64_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? int64_default_fill : int64_fill);
+                    uint64_expected[i] = (i % 2) ? my_rank * MAPLEN + i + 1 : (fv ? uint64_default_fill : uint64_fill);
+#endif /* _NETCDF4 */
+                }
+
+                /* Test for each available type. */
+                for (int t = 0; t < NUM_TYPES; t++)
+                {
+                    void *expected;
+                    void *fill;
+                    void *data;
+                    int ncid, dimid, varid;
+                    char filename[NC_MAX_NAME + 1];
+
+                    switch (test_type[t])
+                    {
+                    case PIO_BYTE:
+                        expected = byte_expected;
+                        fill = fv ? &byte_default_fill : &byte_fill;
+                        data = byte_data;
+                        break;
+                    case PIO_CHAR:
+                        expected = char_expected;
+                        fill = fv ? &char_default_fill : &char_fill;
+                        data = char_data;
+                        break;
+                    case PIO_SHORT:
+                        expected = short_expected;
+                        fill = fv ? &short_default_fill : &short_fill;
+                        data = short_data;
+                        break;
+                    case PIO_INT:
+                        expected = int_expected;
+                        fill = fv ? &int_default_fill : &int_fill;
+                        data = int_data;
+                        break;
+                    case PIO_FLOAT:
+                        expected = float_expected;
+                        fill = fv ? &float_default_fill : &float_fill;
+                        data = float_data;
+                        break;
+                    case PIO_DOUBLE:
+                        expected = double_expected;
+                        fill = fv ? &double_default_fill : &double_fill;
+                        data = double_data;
+                        break;
+#ifdef _NETCDF4
+                    case PIO_UBYTE:
+                        expected = ubyte_expected;
+                        fill = fv ? &ubyte_default_fill : &ubyte_fill;
+                        data = ubyte_data;
+                        break;
+                    case PIO_USHORT:
+                        expected = ushort_expected;
+                        fill = fv ? &ushort_default_fill : &ushort_fill;
+                        data = ushort_data;
+                        break;
+                    case PIO_UINT:
+                        expected = uint_expected;
+                        fill = fv ? &uint_default_fill : &uint_fill;
+                        data = uint_data;
+                        break;
+                    case PIO_INT64:
+                        expected = int64_expected;
+                        fill = fv ? &int64_default_fill : &int64_fill;
+                        data = int64_data;
+                        break;
+                    case PIO_UINT64:
+                        expected = uint64_expected;
+                        fill = fv ? &uint64_default_fill : &uint64_fill;
+                        data = uint64_data;
+                        break;
+#endif /* _NETCDF4 */
+                    default:
+                        return ERR_AWFUL;
+                    }
+
+                    /* Initialize decompositions. */
+                    if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, wcompmap,
+                                               &wioid, &rearranger[r], NULL, NULL)))
+                        return ret;
+                    if ((ret = PIOc_InitDecomp(iosysid, test_type[t], NDIM1, dim_len, maplen, rcompmap,
+                                               &rioid, &rearranger[r], NULL, NULL)))
+                        return ret;
+
+                    /* Create the test file in each of the available iotypes. */
+                    for (int fmt = 0; fmt < num_flavors; fmt++)
+                    {
+                        PIO_Offset type_size;
+                        void *data_in;
+
+                        /* Byte type doesn't work with pnetcdf. */
+                        if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (test_type[t] == PIO_BYTE || test_type[t] == PIO_CHAR))
+                            continue;
+
+                        /* NetCDF-4 types only work with netCDF-4 formats. */
+                        if (test_type[t] > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+                            flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+                            continue;
+
+                        /* Put together filename. */
+                        sprintf(filename, "%s_iotype_%d_rearr_%d_type_%d.nc", TEST_NAME, flavor[fmt],
+                                rearranger[r], test_type[t]);
+
+                        /* Create file. */
+                        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER)))
+                            return ret;
+
+                        /* Define metadata. */
+                        if ((ret = PIOc_def_dim(ncid, DIM_NAME, dim_len[0], &dimid)))
+                            return ret;
+                        if ((ret = PIOc_def_var(ncid, VAR_NAME, test_type[t], NDIM1, &dimid, &varid)))
+                            return ret;
+                        if ((ret = PIOc_put_att(ncid, varid, FILL_VALUE_NAME, test_type[t],
+                                                1, fill)))
+                            return ret;
+                        if ((ret = PIOc_enddef(ncid)))
+                            return ret;
+
+                        /* Write some data. */
+                        if ((ret = PIOc_write_darray(ncid, varid, wioid, MAPLEN, data, fill)))
+                            return ret;
+                        if ((ret = PIOc_sync(ncid)))
+                            return ret;
+
+                        /* What is size of type? */
+                        if ((ret = PIOc_inq_type(ncid, test_type[t], NULL, &type_size)))
+                            return ret;
+
+                        /* Allocate space to read data into. */
+                        if (!(data_in = malloc(type_size * MAPLEN)))
+                            return PIO_ENOMEM;
+
+                        /* Read the data. */
+                        if ((ret = PIOc_read_darray(ncid, varid, rioid, MAPLEN, data_in)))
+                            return ret;
+
+                        /* Check results. */
+                        if (memcmp(data_in, expected, type_size * MAPLEN))
+                            return ERR_AWFUL;
+
+                        /* Release storage. */
+                        free(data_in);
+
+                        /* Close file. */
+                        if ((ret = PIOc_closefile(ncid)))
+                            return ret;
+                    } /* next iotype */
+
+                    /* Free decompositions. */
+                    if ((ret = PIOc_freedecomp(iosysid, wioid)))
+                        return ret;
+                    if ((ret = PIOc_freedecomp(iosysid, rioid)))
+                        return ret;
+
+                } /* next type */
+            } /* next fill value test case */
+        } /* next rearranger */
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_free_iosystem(iosysid)))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/tests/cunit/test_decomp_uneven.c b/tests/cunit/test_decomp_uneven.c
index aee53f55263b..8008c06d2d77 100644
--- a/tests/cunit/test_decomp_uneven.c
+++ b/tests/cunit/test_decomp_uneven.c
@@ -17,7 +17,7 @@
 #define MIN_NTASKS 4
 
 /* The name of this test. */
-#define TEST_NAME "test_darray_uneven"
+#define TEST_NAME "test_decomp_uneven"
 
 /* Number of processors that will do IO. */
 #define NUM_IO_PROCS 1
@@ -122,18 +122,19 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
     char title_in[PIO_MAX_NAME + 1];
     char history_in[PIO_MAX_NAME + 1];
     int fortran_order_in; /* Indicates fortran vs. c order. */
-    int ret;              /* Return code. */
 
     /* Use PIO to create the decomp file in one of the four
      * available ways. */
     for (int fmt = 0; fmt < 1; fmt++)
     {
+        int ret;              /* Return code. */
+      
         /* Create the filename. */
-        sprintf(filename, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc", TEST_NAME, pio_type,
-                dim_len[0], dim_len[1], dim_len[2]);
+        snprintf(filename, PIO_MAX_NAME, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc",
+		 TEST_NAME, pio_type, dim_len[0], dim_len[1], dim_len[2]);
 
         /* Create history string. */
-        strncat(history, filename, NC_MAX_NAME - strlen(TEST_DECOMP_HISTORY));
+        strncat(history, filename, PIO_MAX_NAME - strlen(TEST_DECOMP_HISTORY));
 
         if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, title, history, 0)))
             return ret;
@@ -264,11 +265,8 @@ int main(int argc, char **argv)
 /* #define NUM_TYPES_TO_TEST 3 */
 /*     int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; */
 #define NUM_TYPES_TO_TEST 1
-    int test_type[NUM_TYPES_TO_TEST] = {PIO_INT};
     int my_rank;
     int ntasks;
-    int num_flavors; /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
     MPI_Comm test_comm; /* A communicator for this test. */
     int ret;         /* Return code. */
 
@@ -307,6 +305,8 @@ int main(int argc, char **argv)
                                                                       {3, 2, 2, 2},
                                                                       {2, 2, 1, 1}};
         int *expected_map[NUM_DIM_COMBOS_TO_TEST] = {map_1x4x4, map_2x4x4, map_3x4x4, map_1x3x3, map_1x2x3};
+	int num_flavors; /* Number of PIO netCDF flavors in this build. */
+	int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
         int ret;      /* Return code. */
 
         /* Figure out iotypes. */
@@ -328,6 +328,8 @@ int main(int argc, char **argv)
             {
                 for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++)
                 {
+		    int test_type[NUM_TYPES_TO_TEST] = {PIO_INT};
+		    
                     /* What is length of map for this combo? */
                     int full_maplen = 1;
                     for (int d = 0; d < NDIM3; d++)
@@ -352,7 +354,7 @@ int main(int argc, char **argv)
             }
 
             /* Finalize PIO system. */
-            if ((ret = PIOc_finalize(iosysid)))
+            if ((ret = PIOc_free_iosystem(iosysid)))
                 return ret;
 
         } /* next rearranger */
diff --git a/tests/cunit/test_decomps.c b/tests/cunit/test_decomps.c
index 473f70eef658..0c57eec91047 100644
--- a/tests/cunit/test_decomps.c
+++ b/tests/cunit/test_decomps.c
@@ -102,7 +102,10 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm)
         if (!(iostart = calloc(NDIM2, sizeof(PIO_Offset))))
             return ERR_AWFUL;
         if (!(iocount = calloc(NDIM2, sizeof(PIO_Offset))))
+        {
+            free(iostart);
             return ERR_AWFUL;
+        }
         if (my_rank == 0)
             for (int i = 0; i < NDIM2; i++)
                 iocount[i] = 4;
@@ -111,15 +114,20 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm)
     /* Create the PIO decomposition for this test. */
     if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
                                compdof, &ioid, NULL, iostart, iocount)))
+    {
+        if (iostart)
+            free(iostart);
+        if (iocount)
+            free(iocount);
         return ret;
+    }
 
     /* Free resources. */
     free(compdof);
-    if (use_io)
-    {
+    if (iostart)
         free(iostart);
+    if (iocount)
         free(iocount);
-    }
 
     /* These should not work. */
     if (PIOc_write_decomp(DECOMP_FILE, iosysid + TEST_VAL_42, ioid, test_comm) != PIO_EBADID)
@@ -412,7 +420,7 @@ int main(int argc, char **argv)
                     ERR(ret);
 
                 /* Finalize PIO systems. */
-                if ((ret = PIOc_finalize(iosysid)))
+                if ((ret = PIOc_free_iosystem(iosysid)))
                     ERR(ret);
             } /* next io test */
         } /* next rearranger */
diff --git a/tests/cunit/test_intercomm2.c b/tests/cunit/test_intercomm2.c
index 6131e0ef85f7..eb99e0ff8680 100644
--- a/tests/cunit/test_intercomm2.c
+++ b/tests/cunit/test_intercomm2.c
@@ -9,6 +9,7 @@
  * @author Ed Hartnett
  *
  */
+#include 
 #include 
 #include 
 #include 
@@ -60,14 +61,14 @@ int check_file(int iosysid, int format, char *filename, int my_rank)
     int ndims, nvars, ngatts, unlimdimid;
     int ndims2, nvars2, ngatts2, unlimdimid2;
     int dimid2;
-    char dimname[NC_MAX_NAME + 1];
+    char dimname[PIO_MAX_NAME + 1];
     PIO_Offset dimlen;
-    char dimname2[NC_MAX_NAME + 1];
+    char dimname2[PIO_MAX_NAME + 1];
     PIO_Offset dimlen2;
-    char varname[NC_MAX_NAME + 1];
+    char varname[PIO_MAX_NAME + 1];
     nc_type vartype;
     int varndims, vardimids, varnatts;
-    char varname2[NC_MAX_NAME + 1];
+    char varname2[PIO_MAX_NAME + 1];
     nc_type vartype2;
     int varndims2, vardimids2, varnatts2;
     int varid2;
@@ -204,7 +205,7 @@ int check_file(int iosysid, int format, char *filename, int my_rank)
     /* Check out the global attributes. */
     nc_type atttype;
     PIO_Offset attlen;
-    char myattname[NC_MAX_NAME + 1];
+    char myattname[PIO_MAX_NAME + 1];
     int myid;
     if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen)))
         ERR(ret);
@@ -279,7 +280,7 @@ int main(int argc, char **argv)
     int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
 
     /* Names for the output files. */
-    char filename[NUM_FLAVORS][NC_MAX_NAME + 1];
+    char filename[NUM_FLAVORS][PIO_MAX_NAME + 1];
 
     /* The ID for the parallel I/O system. */
     int iosysid[COMPONENT_COUNT];
@@ -361,7 +362,7 @@ int main(int argc, char **argv)
                     ERR(ERR_AWFUL);
 
                 /* Test the inq_type function for atomic types. */
-                char type_name[NC_MAX_NAME + 1];
+                char type_name[PIO_MAX_NAME + 1];
                 PIO_Offset type_size;
                 nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
                                             NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
@@ -382,7 +383,7 @@ int main(int argc, char **argv)
                 }
 
                 /* Define a dimension. */
-                char dimname2[NC_MAX_NAME + 1];
+                char dimname2[PIO_MAX_NAME + 1];
                 if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME, DIM_LEN, &dimid)))
                     ERR(ret);
                 if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
@@ -401,7 +402,7 @@ int main(int argc, char **argv)
                     ERR(ERR_WRONG);
 
                 /* Define a 1-D variable. */
-                char varname2[NC_MAX_NAME + 1];
+                char varname2[PIO_MAX_NAME + 1];
                 if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME, NC_INT, NDIM, &dimid, &varid)))
                     ERR(ret);
                 if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
@@ -424,7 +425,7 @@ int main(int argc, char **argv)
                 short short_att_data = ATT_VALUE;
                 float float_att_data = ATT_VALUE;
                 double double_att_data = ATT_VALUE;
-                char attname2[NC_MAX_NAME + 1];
+                char attname2[PIO_MAX_NAME + 1];
 
                 /* Write an att and rename it. */
                 if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
@@ -520,7 +521,7 @@ int main(int argc, char **argv)
             } /* next netcdf flavor */
 
             /* Finalize the IO system. Only call this from the computation tasks. */
-            if ((ret = PIOc_finalize(iosysid[my_comp_idx])))
+            if ((ret = PIOc_free_iosystem(iosysid[my_comp_idx])))
                 ERR(ret);
         }
     } /* my_rank < TARGET_NTASKS */
diff --git a/tests/cunit/test_iosystem2.c b/tests/cunit/test_iosystem2.c
index 32324e553dd8..285fa9d156da 100644
--- a/tests/cunit/test_iosystem2.c
+++ b/tests/cunit/test_iosystem2.c
@@ -143,7 +143,7 @@ int main(int argc, char **argv)
             ERR(ret);
 
         /* This should fail. */
-        if (PIOc_finalize(iosysid + TEST_VAL_42) != PIO_EBADID)
+        if (PIOc_free_iosystem(iosysid + TEST_VAL_42) != PIO_EBADID)
             ERR(ERR_WRONG);
 
         /* Initialize another PIO system. */
@@ -190,16 +190,22 @@ int main(int argc, char **argv)
                 ERR(ret);
             if ((ret = PIOc_closefile(ncid2)))
                 ERR(ret);
+
+            /* Wait for everyone to finish. */
+            if ((ret = MPI_Barrier(test_comm)))
+                MPIERR(ret);
+
         } /* next iotype */
+
         if ((ret = MPI_Comm_free(&newcomm)))
             MPIERR(ret);
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             ERR(ret);
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid_world)))
+        if ((ret = PIOc_free_iosystem(iosysid_world)))
             ERR(ret);
     } /* my_rank < TARGET_NTASKS */
 
diff --git a/tests/cunit/test_iosystem2_simple.c b/tests/cunit/test_iosystem2_simple.c
index db94bf0ed41c..863d94faf6e8 100644
--- a/tests/cunit/test_iosystem2_simple.c
+++ b/tests/cunit/test_iosystem2_simple.c
@@ -152,8 +152,8 @@ int main(int argc, char **argv)
         int ncid2;
         for (int i = 0; i < num_flavors; i++)
         {
-            char fn[NUM_FILES][NC_MAX_NAME + 1];
-            char dimname[NUM_FILES][NC_MAX_NAME + 1];
+            char fn[NUM_FILES][PIO_MAX_NAME + 1];
+            char dimname[NUM_FILES][PIO_MAX_NAME + 1];
 
             /* Create the test files. */
             for (int f = 0; f < NUM_FILES; f++)
@@ -193,7 +193,7 @@ int main(int argc, char **argv)
                 return ret;
 
             /* Check the first file. */
-            char dimname_in[NC_MAX_NAME + 1];
+            char dimname_in[PIO_MAX_NAME + 1];
             if ((ret = PIOc_inq_dimname(ncid, 0, dimname_in)))
                 return ret;
             if (strcmp(dimname_in, dimname[0]))
@@ -221,12 +221,12 @@ int main(int argc, char **argv)
             MPIERR(ret);
 
         /* Finalize PIO odd/even intracomm. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             ERR(ret);
 
 
         /* Finalize PIO world intracomm. */
-        if ((ret = PIOc_finalize(iosysid_world)))
+        if ((ret = PIOc_free_iosystem(iosysid_world)))
             ERR(ret);
     }/* my_rank < TARGET_NTASKS */
 
diff --git a/tests/cunit/test_iosystem2_simple2.c b/tests/cunit/test_iosystem2_simple2.c
index 721a4ef5a76e..d4db1540f705 100644
--- a/tests/cunit/test_iosystem2_simple2.c
+++ b/tests/cunit/test_iosystem2_simple2.c
@@ -36,10 +36,8 @@ int main(int argc, char **argv)
     int ntasks; /* Number of processors involved in current execution. */
     int iosysid; /* The ID for the parallel I/O system. */
     int iosysid_world; /* The ID for the parallel I/O system. */
-    int num_flavors; /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
-    int ret; /* Return code. */
     MPI_Comm test_comm;
+    int ret; /* Return code. */
 
     /* Initialize test. */
     if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, TARGET_NTASKS,
@@ -49,13 +47,16 @@ int main(int argc, char **argv)
     /* Only do something on the first TARGET_NTASKS tasks. */
     if (my_rank < TARGET_NTASKS)
     {
+	int num_flavors; /* Number of PIO netCDF flavors in this build. */
+	int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+	
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
             ERR(ret);
 
         /* Split world into odd and even. */
         MPI_Comm newcomm;
-        int even = my_rank % 2 ? 0 : 1;
+        int even = (my_rank % 2) ? 0 : 1;
         if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
             MPIERR(ret);
 
@@ -75,12 +76,12 @@ int main(int argc, char **argv)
 
         for (int flv = 0; flv < num_flavors; flv++)
         {
-            char filename[NUM_SAMPLES][NC_MAX_NAME + 1]; /* Test filename. */
+            char filename[NUM_SAMPLES][PIO_MAX_NAME + 1]; /* Test filename. */
             int sample_ncid[NUM_SAMPLES];
 
             for (int sample = 0; sample < NUM_SAMPLES; sample++)
             {
-                char iotype_name[NC_MAX_NAME + 1];
+                char iotype_name[PIO_MAX_NAME + 1];
 
                 /* Create a filename. */
                 if ((ret = get_iotype_name(flavor[flv], iotype_name)))
@@ -119,11 +120,11 @@ int main(int argc, char **argv)
             MPIERR(ret);
 
         /* Finalize PIO odd/even intracomm. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             ERR(ret);
 
         /* Finalize PIO world intracomm. */
-        if ((ret = PIOc_finalize(iosysid_world)))
+        if ((ret = PIOc_free_iosystem(iosysid_world)))
             ERR(ret);
     }  /* my_rank < TARGET_NTASKS */
 
diff --git a/tests/cunit/test_iosystem3.c b/tests/cunit/test_iosystem3.c
index 61ca0e24e59f..1c91b00c5c49 100644
--- a/tests/cunit/test_iosystem3.c
+++ b/tests/cunit/test_iosystem3.c
@@ -56,7 +56,8 @@ int create_file(MPI_Comm comm, int iosysid, int format, char *filename,
         return ret;
 
     /* Write an attribute. */
-    if ((ret = PIOc_put_att_text(ncid, varid, attname, strlen(filename), filename)))
+    if ((ret = PIOc_put_att_text(ncid, varid, attname, strnlen(filename, PIO_TF_MAX_STR_LEN),
+				 filename)))
         return ret;
 
     /* End define mode. */
@@ -93,11 +94,11 @@ int check_file(MPI_Comm comm, int iosysid, int format, int ncid, char *filename,
 
     /* Check the attribute. Null terminating byte deliberately ignored
      * to match fortran code. */
-    if (!(att_data = malloc(strlen(filename) * sizeof(char))))
+    if (!(att_data = malloc(strnlen(filename, PIO_TF_MAX_STR_LEN) * sizeof(char))))
         return PIO_ENOMEM;
     if ((ret = PIOc_get_att(ncid, varid, attname, att_data)))
         return ret;
-    if (strncmp(att_data, filename, strlen(filename)))
+    if (strncmp(att_data, filename, strnlen(filename, PIO_TF_MAX_STR_LEN)))
         return ERR_WRONG;
     free(att_data);
 
@@ -142,10 +143,7 @@ int main(int argc, char **argv)
     MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
     int even_rank = -1, overlap_rank = -1; /* Tasks rank in communicator. */
     int even_size = 0, overlap_size = 0; /* Size of communicator. */
-    int num_flavors; /* Number of PIO netCDF flavors in this build. */
-    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
     MPI_Comm test_comm;
-    int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
     int ret; /* Return code. */
 
     /* Initialize test. */
@@ -157,6 +155,10 @@ int main(int argc, char **argv)
      * nothing. */
     if (my_rank < TARGET_NTASKS)
     {
+      int num_flavors; /* Number of PIO netCDF flavors in this build. */
+      int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+      int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
             ERR(ret);
@@ -312,17 +314,17 @@ int main(int argc, char **argv)
                     ERR(ret);
 
             } /* next iotype */
-        
+
             /* Finalize PIO systems. */
             if (even_comm != MPI_COMM_NULL)
-                if ((ret = PIOc_finalize(even_iosysid)))
+                if ((ret = PIOc_free_iosystem(even_iosysid)))
                     ERR(ret);
             if (overlap_comm != MPI_COMM_NULL)
             {
-                if ((ret = PIOc_finalize(overlap_iosysid)))
+                if ((ret = PIOc_free_iosystem(overlap_iosysid)))
                     ERR(ret);
             }
-            if ((ret = PIOc_finalize(iosysid_world)))
+            if ((ret = PIOc_free_iosystem(iosysid_world)))
                 ERR(ret);
 
             /* Free MPI resources used by test. */
diff --git a/tests/cunit/test_iosystem3_simple.c b/tests/cunit/test_iosystem3_simple.c
index b78f8b420c1b..a144c206e7a3 100644
--- a/tests/cunit/test_iosystem3_simple.c
+++ b/tests/cunit/test_iosystem3_simple.c
@@ -85,10 +85,10 @@ int main(int argc, char **argv)
 
         /* Finalize PIO system. */
         if (overlap_comm != MPI_COMM_NULL)
-            if ((ret = PIOc_finalize(overlap_iosysid)))
+            if ((ret = PIOc_free_iosystem(overlap_iosysid)))
                 ERR(ret);
 
-        if ((ret = PIOc_finalize(iosysid_world)))
+        if ((ret = PIOc_free_iosystem(iosysid_world)))
             ERR(ret);
 
         /* Free MPI resources used by test. */
diff --git a/tests/cunit/test_iosystem3_simple2.c b/tests/cunit/test_iosystem3_simple2.c
index 7253412557c4..627c2882549d 100644
--- a/tests/cunit/test_iosystem3_simple2.c
+++ b/tests/cunit/test_iosystem3_simple2.c
@@ -29,7 +29,7 @@ int main(int argc, char **argv)
     int my_rank; /* Zero-based rank of processor. */
     int ntasks; /* Number of processors involved in current execution. */
     int iosysid_world; /* The ID for the parallel I/O system. */
-    char fname0[NC_MAX_NAME + 1];
+    char fname0[PIO_MAX_NAME + 1];
     int ncid;
     int num_flavors; /* Number of PIO netCDF flavors in this build. */
     int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
@@ -86,7 +86,7 @@ int main(int argc, char **argv)
         } /* next iotype */
 
         /* Finalize PIO systems. */
-        if ((ret = PIOc_finalize(iosysid_world)))
+        if ((ret = PIOc_free_iosystem(iosysid_world)))
             ERR(ret);
     } /* my_rank < TARGET_NTASKS */
 
diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c
new file mode 100644
index 000000000000..aa651dda5820
--- /dev/null
+++ b/tests/cunit/test_perf2.c
@@ -0,0 +1,585 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date 2/21/17
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+#ifdef USE_MPE
+#include 
+#endif /* USE_MPE */
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 16
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS TARGET_NTASKS
+
+/* The name of this test. */
+#define TEST_NAME "test_perf2"
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 4
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM3 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 128
+#define Y_DIM_LEN 128
+#define Z_DIM_LEN 32
+/* #define X_DIM_LEN 1024 */
+/* #define Y_DIM_LEN 1024 */
+/* #define Z_DIM_LEN 128 */
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 10
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* How many different number of IO tasks to check? */
+#define MAX_IO_TESTS 5
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+
+/* Run test for each of the rearrangers. */
+#define NUM_REARRANGERS_TO_TEST 2
+
+#ifdef USE_MPE
+/* These are for the event numbers array used to log various events in
+ * the program with the MPE library, which produces output for the
+ * Jumpshot program. */
+#define NUM_EVENTS 7
+#define START 0
+#define END 1
+#define INIT 0
+#define DECOMP 1
+#define CREATE 2
+#define DARRAY_WRITE 3
+#define CLOSE 4
+#define CALCULATE 5
+#define INGEST 6
+
+#define ERR_LOGGING 99
+
+/* This array holds even numbers for MPE. */
+int event_num[2][NUM_EVENTS];
+
+/* This will set up the MPE logging event numbers. */
+int
+init_logging(int my_rank, int event_num[][NUM_EVENTS])
+{
+   /* Get a bunch of event numbers. */
+   event_num[START][INIT] = MPE_Log_get_event_number();
+   event_num[END][INIT] = MPE_Log_get_event_number();
+   event_num[START][DECOMP] = MPE_Log_get_event_number();
+   event_num[END][DECOMP] = MPE_Log_get_event_number();
+   event_num[START][INGEST] = MPE_Log_get_event_number();
+   event_num[END][INGEST] = MPE_Log_get_event_number();
+   event_num[START][CLOSE] = MPE_Log_get_event_number();
+   event_num[END][CLOSE] = MPE_Log_get_event_number();
+   event_num[START][CALCULATE] = MPE_Log_get_event_number();
+   event_num[END][CALCULATE] = MPE_Log_get_event_number();
+   event_num[START][CREATE] = MPE_Log_get_event_number();
+   event_num[END][CREATE] = MPE_Log_get_event_number();
+   event_num[START][DARRAY_WRITE] = MPE_Log_get_event_number();
+   event_num[END][DARRAY_WRITE] = MPE_Log_get_event_number();
+
+   /* You should track at least initialization and partitioning, data
+    * ingest, update computation, all communications, any memory
+    * copies (if you do that), any output rendering, and any global
+    * communications. */
+   if (!my_rank)
+   {
+      MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "yellow");
+      MPE_Describe_state(event_num[START][INGEST], event_num[END][INGEST], "ingest", "red");
+      MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green");
+      MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], "calculate", "orange");
+      MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create", "purple");
+      MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue");
+      MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink");
+   }
+   return 0;
+}
+#endif /* USE_MPE */
+
+/* Create the decomposition to divide the 4-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 3 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int
+create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+    int ret;
+
+    /* How many data elements per task? */
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this test. */
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe,
+                                compdof, ioid, 0, NULL, NULL)))
+        ERR(ret);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/**
+ * Test the darray functionality. Create a netCDF file with 4
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param ntasks number of tasks in test_comm.
+ * @param num_io_procs number of IO processors.
+ * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray().
+ * @param rearranger the rearranger in use.
+ * @returns 0 for success, error code otherwise.
+ */
+int
+test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
+            int my_rank, int ntasks, int num_io_procs, int provide_fill,
+            int rearranger)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    PIO_Offset arraylen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks);
+    int int_fillvalue = NC_FILL_INT;
+    void *fillvalue = NULL;
+    int *test_data;
+    int ret;       /* Return code. */
+
+    if (!(test_data = malloc(sizeof(int) * arraylen)))
+        ERR(PIO_ENOMEM);
+
+    /* Are we providing a fill value? */
+    if (provide_fill)
+        fillvalue = &int_fillvalue;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        struct timeval starttime, endtime;
+        long long startt, endt;
+        long long delta;
+        float num_megabytes = 0;
+        float delta_in_sec;
+        float mb_per_sec;
+
+#ifdef USE_MPE
+        if ((ret = MPE_Log_event(event_num[START][CREATE], 0, "start init")))
+            return ERR_MPI;
+#endif /* USE_MPE */
+
+        /* Create the filename. Use the same filename for all, so we
+         * don't waste disk space. */
+        /* sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], */
+        /*         rearranger); */
+        sprintf(filename, "data_%s.nc", TEST_NAME);
+
+        /* Create the netCDF output file. */
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode. */
+        if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        for (int d = 0; d < NDIM; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+#ifdef USE_MPE
+        if ((ret = MPE_Log_event(event_num[END][CREATE], 0, "end init")))
+            MPIERR(ret);
+#endif /* USE_MPE */
+
+        /* Start the clock. */
+        gettimeofday(&starttime, NULL);
+
+        for (int t = 0; t < NUM_TIMESTEPS; t++)
+        {
+            /* Initialize some data. */
+            for (int f = 0; f < arraylen; f++)
+                test_data[f] = (my_rank * 10 + f) + t * 1000;
+
+#ifdef USE_MPE
+            if ((ret = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, "start init")))
+                return ERR_MPI;
+#endif /* USE_MPE */
+
+            /* Set the value of the record dimension. */
+            if ((ret = PIOc_setframe(ncid, varid, t)))
+                ERR(ret);
+
+            /* Write the data. */
+            if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+                ERR(ret);
+
+#ifdef USE_MPE
+            if ((ret = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, "end init")))
+                MPIERR(ret);
+#endif /* USE_MPE */
+
+            num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024);
+        }
+
+#ifdef USE_MPE
+        if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start init")))
+            return ERR_MPI;
+#endif /* USE_MPE */
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+#ifdef USE_MPE
+        if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end init")))
+            MPIERR(ret);
+#endif /* USE_MPE */
+
+
+        /* Stop the clock. */
+        gettimeofday(&endtime, NULL);
+
+        /* Compute the time delta */
+        startt = (1000000 * starttime.tv_sec) + starttime.tv_usec;
+        endt = (1000000 * endtime.tv_sec) + endtime.tv_usec;
+        delta = (endt - startt)/NUM_TIMESTEPS;
+        delta_in_sec = (float)delta / 1000000;
+        mb_per_sec = num_megabytes / delta_in_sec;
+        if (!my_rank)
+            printf("%d\t%d\t%d\t%d\t%d\t%8.3f\t%8.1f\t%8.3f\n", ntasks, num_io_procs,
+                   rearranger, provide_fill, fmt, delta_in_sec, num_megabytes, mb_per_sec);
+    }
+
+    free(test_data);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param ntasks number of tasks in test_comm.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+ */
+int
+test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
+                       int my_rank, int ntasks, int rearranger,
+                       MPI_Comm test_comm)
+{
+
+    /* for (int fmt = 0; fmt < num_flavors; fmt++) */
+    for (int fmt = 0; fmt < 1; fmt++)
+    {
+	int ioid2;             /* ID for decomposition we will create from file. */
+	char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+	char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+	char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+	int fortran_order_in; /* Indicates fortran vs. c order. */
+	int ret;              /* Return code. */
+
+        /* Create the filename. */
+        snprintf(filename, PIO_MAX_NAME, "decomp_%s_iotype_%d.nc", TEST_NAME,
+		 flavor[fmt]);
+
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0)))
+            return ret;
+
+        /* Read the data. */
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+            int expected_maplen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks);
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+            if (iodesc->ioid != ioid2 || iodesc->maplen != expected_maplen || iodesc->ndims != NDIM3 ||
+                iodesc->ndof != expected_maplen)
+                return ERR_WRONG;
+            if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 ||
+                iodesc->needsfill || iodesc->mpitype != MPI_INT)
+                return ERR_WRONG;
+            for (int e = 0; e < iodesc->maplen; e++)
+                if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
+                    return ERR_WRONG;
+            if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN ||
+                iodesc->dimlen[2] != Z_DIM_LEN)
+                return ERR_WRONG;
+            if (rearranger == PIO_REARR_SUBSET)
+            {
+                if (iodesc->nrecvs != 1  || iodesc->num_aiotasks != ntasks)
+                    return ERR_WRONG;
+            }
+            else
+            {
+                /* I haven't figured out yet what these should be for
+                 * the box rearranger. */
+                /* printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, */
+                /*        iodesc->num_aiotasks); */
+            }
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param ntasks number of tasks in test_comm.
+ * @param num_io_procs number of IO procs used.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int
+test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                int ntasks, int num_io_procs, int rearranger,
+                MPI_Comm test_comm)
+{
+    int ioid;
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+#ifdef USE_MPE
+    if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init")))
+        return ERR_MPI;
+#endif /* USE_MPE */
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid)))
+        return ret;
+
+    /* Test decomposition read/write. */
+    if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                      ntasks, rearranger, test_comm)))
+        return ret;
+
+#ifdef USE_MPE
+    if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init")))
+        MPIERR(ret);
+#endif /* USE_MPE */
+
+    /* Test with/without providing a fill value to PIOc_write_darray(). */
+    for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+    {
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank,
+                               ntasks, num_io_procs, provide_fill, rearranger)))
+            return ret;
+    }
+
+#ifdef USE_MPE
+    if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init")))
+        return ERR_MPI;
+#endif /* USE_MPE */
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+#ifdef USE_MPE
+    if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init")))
+        MPIERR(ret);
+#endif /* USE_MPE */
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int
+main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int iosysid;  /* The ID for the parallel I/O system. */
+    int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+    int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int num_io_procs[MAX_IO_TESTS] = {1, 4, 16, 64, 128}; /* Number of processors that will do IO. */
+    int num_io_tests; /* How many different num IO procs to try? */
+    int r, i;
+    int ret;      /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, 1,
+                              0, -1, &test_comm)))
+        ERR(ERR_INIT);
+
+#ifdef USE_MPE
+
+   if ((ret = MPE_Init_log()))
+       return ret;
+   if (init_logging(my_rank, event_num))
+       return ERR_LOGGING;
+#endif /* USE_MPE */
+
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+
+    if (!my_rank)
+        printf("ntasks\tnio\trearr\tfill\tformat\ttime(s)\tdata size (MB)\t"
+               "performance(MB/s)\n");
+
+    /* How many processors for IO? */
+    num_io_tests = 1;
+    if (ntasks >= 32)
+        num_io_tests = 2;
+    if (ntasks >= 64)
+        num_io_tests = 3;
+    if (ntasks >= 128)
+        num_io_tests = 4;
+    if (ntasks >= 512)
+        num_io_tests = 5;
+
+    for (i = 0; i < num_io_tests; i++)
+    {
+        /* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */
+        for (r = 0; r < 1; r++)
+        {
+#ifdef USE_MPE
+            if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init")))
+                return ERR_MPI;
+#endif /* USE_MPE */
+
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, num_io_procs[i], ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+#ifdef USE_MPE
+            if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
+                MPIERR(ret);
+#endif /* USE_MPE */
+
+            /* Run tests. */
+            if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank,
+                                       ntasks, num_io_procs[i], rearranger[r], test_comm)))
+                return ret;
+
+            /* Finalize PIO system. */
+            if ((ret = PIOc_free_iosystem(iosysid)))
+                return ret;
+        } /* next rearranger */
+    } /* next num io procs */
+
+
+#ifdef USE_MPE
+   {
+       /* This causes problems on my MPICH2 library on Linux, but seems to be
+	* required for frost. */
+       char file_name[128];
+       sprintf(file_name, "chart_%d", 1);
+       if ((ret = MPE_Finish_log(file_name)))
+	   MPIERR(ret);
+   }
+
+#endif /* USE_MPE */
+
+
+    if (!my_rank)
+        printf("finalizing io_test!\n");
+
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    /* printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); */
+    return 0;
+}
diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c
index c4cd184a1853..9bdb05603516 100644
--- a/tests/cunit/test_pioc.c
+++ b/tests/cunit/test_pioc.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * Tests for PIO Functions.
  *
  * @author Ed Hartnett
@@ -539,11 +539,10 @@ int test_iotypes(int my_rank)
  */
 int check_strerror_netcdf(int my_rank)
 {
-#define NUM_NETCDF_TRIES 5
-    int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 1, 0, 1, -600};
+#define NUM_NETCDF_TRIES 3
+    int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, 0, 1};
     const char *expected[NUM_NETCDF_TRIES] = {"NetCDF: Not a valid ID",
-                                              "Unknown Error: Unrecognized error code", "No error",
-                                              nc_strerror(1), "Unknown Error: Unrecognized error code"};
+                                              "No error", nc_strerror(1)};
     int ret;
 
     if ((ret = check_error_strings(my_rank, NUM_NETCDF_TRIES, errcode, expected)))
@@ -551,9 +550,7 @@ int check_strerror_netcdf(int my_rank)
 
     /* When called with a code of 0, these functions should do nothing
      * and return 0. */
-    if (check_mpi(NULL, 0, __FILE__, __LINE__))
-        ERR(ERR_WRONG);
-    if (check_mpi2(NULL, NULL, 0, __FILE__, __LINE__))
+    if (check_mpi(NULL, NULL, 0, __FILE__, __LINE__))
         ERR(ERR_WRONG);
     if (pio_err(NULL, NULL, 0, __FILE__, __LINE__))
         ERR(ERR_WRONG);
@@ -562,15 +559,6 @@ int check_strerror_netcdf(int my_rank)
     if (check_netcdf2(NULL, NULL, 0, __FILE__, __LINE__))
         ERR(ERR_WRONG);
 
-    /* When called with other error messages, these functions should
-     * return PIO_EIO. */
-    /* if (check_mpi(NULL, MPI_ERR_OTHER, __FILE__, __LINE__) != PIO_EIO) */
-    /*     ERR(ERR_WRONG); */
-    /* This returns the correct result, but prints a confusing error
-     * message during the test run, so I'll leave it commented out. */
-    /* if (check_mpi(NULL, MPI_ERR_UNKNOWN, __FILE__, __LINE__) != PIO_EIO) */
-    /*     ERR(ERR_WRONG); */
-
     return PIO_NOERR;
 }
 
@@ -627,14 +615,13 @@ int check_strerror_pnetcdf(int my_rank)
  */
 int check_strerror_pio(int my_rank)
 {
-#define NUM_PIO_TRIES 6
+#define NUM_PIO_TRIES 5
     int errcode[NUM_PIO_TRIES] = {PIO_EBADID,
-                                  NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1,
+                                  NC_ENOTNC3, 0, 1,
                                   PIO_EBADIOTYPE};
     const char *expected[NUM_PIO_TRIES] = {"NetCDF: Not a valid ID",
                                            "NetCDF: Attempting netcdf-3 operation on netcdf-4 file",
-                                           "Unknown Error: Unrecognized error code", "No error",
-                                           nc_strerror(1), "Bad IO type"};
+                                           "No error", nc_strerror(1), "Bad IO type"};
     int ret;
 
     if ((ret = check_error_strings(my_rank, NUM_PIO_TRIES, errcode, expected)))
@@ -1905,8 +1892,8 @@ int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len
                          MPI_Comm test_comm, int async)
 {
     int ioid;
-    char filename[NC_MAX_NAME + 1];    /* Test decomp filename. */
-    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    char filename[PIO_MAX_NAME + 1];    /* Test decomp filename. */
+    char nc_filename[PIO_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
     iosystem_desc_t *ios; /* IO system info. */
     int ret;
 
@@ -2094,7 +2081,7 @@ int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len,
                        MPI_Comm test_comm, int async)
 {
     int ioid;
-    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    char nc_filename[PIO_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
     int ret;
 
     /* This will be our file name for writing out decompositions. */
@@ -2239,7 +2226,7 @@ int test_decomp_public_2(int my_test_size, int my_rank, int iosysid, int dim_len
                          MPI_Comm test_comm, int async)
 {
     int ioid;
-    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    char nc_filename[PIO_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
     int ret;
 
     /* This will be our file name for writing out decompositions. */
@@ -2265,7 +2252,7 @@ int test_decomp_2(int my_test_size, int my_rank, int iosysid, int dim_len,
                   MPI_Comm test_comm, int async)
 {
     int ioid;
-    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    char nc_filename[PIO_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
     int ret;
 
     /* This will be our file name for writing out decompositions. */
@@ -2319,8 +2306,8 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te
 {
     int ioid;
     int my_test_size;
-    char filename[NC_MAX_NAME + 1];
-    char nc_filename[NC_MAX_NAME + 1];
+    char filename[PIO_MAX_NAME + 1];
+    char nc_filename[PIO_MAX_NAME + 1];
     int ret; /* Return code. */
 
     if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
diff --git a/tests/cunit/test_pioc_fill.c b/tests/cunit/test_pioc_fill.c
index 52e0571a6ebe..829631420956 100644
--- a/tests/cunit/test_pioc_fill.c
+++ b/tests/cunit/test_pioc_fill.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * More tests for PIO data reading and writing routines.
  *
  * Ed Hartnett
diff --git a/tests/cunit/test_pioc_putget.c b/tests/cunit/test_pioc_putget.c
index 15452eb99c4e..78e55f2e9d68 100644
--- a/tests/cunit/test_pioc_putget.c
+++ b/tests/cunit/test_pioc_putget.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * Tests for PIO data reading and writing routines.
  *
  * @author Ed Hartnett
@@ -153,15 +153,10 @@ int test_att_conv_byte(int ncid, int flavor, char *name, int *expected, long lon
 {
     signed char byte_array_in[ATT_LEN];
     short short_array_in[ATT_LEN];
-    unsigned char ubyte_array_in[ATT_LEN];
     int int_array_in[ATT_LEN];
     long long_array_in[ATT_LEN];
     float float_array_in[ATT_LEN];
     double double_array_in[ATT_LEN];
-    unsigned short ushort_array_in[ATT_LEN];
-    unsigned int uint_array_in[ATT_LEN];
-    long long int64_array_in[ATT_LEN];
-    unsigned long long uint64_array_in[ATT_LEN];
 
     /* Read the att and check results. */
     if (expected[PIO_BYTE] != PIOc_get_att_schar(ncid, NC_GLOBAL, name, byte_array_in))
@@ -214,6 +209,12 @@ int test_att_conv_byte(int ncid, int flavor, char *name, int *expected, long lon
 
     if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
     {
+        unsigned char ubyte_array_in[ATT_LEN];
+        unsigned short ushort_array_in[ATT_LEN];
+        unsigned int uint_array_in[ATT_LEN];
+        long long int64_array_in[ATT_LEN];
+        unsigned long long uint64_array_in[ATT_LEN];
+
         if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
             return ERR_WRONG;
 
@@ -271,10 +272,6 @@ int test_att_conv_int64(int ncid, int flavor, char *name, int *expected, long lo
 {
     float float_array_in[ATT_LEN];
     double double_array_in[ATT_LEN];
-    unsigned char ubyte_array_in[ATT_LEN];
-    unsigned short ushort_array_in[ATT_LEN];
-    unsigned int uint_array_in[ATT_LEN];
-    long long int64_array_in[ATT_LEN];
 
     /* Read the att and check results. */
     if (expected[PIO_FLOAT] != PIOc_get_att_float(ncid, NC_GLOBAL, name, float_array_in))
@@ -285,6 +282,11 @@ int test_att_conv_int64(int ncid, int flavor, char *name, int *expected, long lo
 
     if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
     {
+        unsigned char ubyte_array_in[ATT_LEN];
+        unsigned short ushort_array_in[ATT_LEN];
+        unsigned int uint_array_in[ATT_LEN];
+        long long int64_array_in[ATT_LEN];
+
         if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
             return ERR_WRONG;
         if ((expected[PIO_USHORT] != PIOc_get_att_ushort(ncid, NC_GLOBAL, name, ushort_array_in)))
@@ -775,15 +777,10 @@ int test_read_att(int ncid, int *varid, int flavor)
     char text_in[ATT_LEN];
     signed char byte_array_in[ATT_LEN];
     short short_array_in[ATT_LEN];
-    unsigned char ubyte_array_in[ATT_LEN];
     int int_array_in[ATT_LEN];
     long int long_array_in[ATT_LEN];
     float float_array_in[ATT_LEN];
     double double_array_in[ATT_LEN];
-    unsigned short ushort_array_in[ATT_LEN];
-    unsigned int uint_array_in[ATT_LEN];
-    long long int64_array_in[ATT_LEN];
-    unsigned long long uint64_array_in[ATT_LEN];
     int x;
     int ret;
 
@@ -821,6 +818,12 @@ int test_read_att(int ncid, int *varid, int flavor)
 
     if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
     {
+      unsigned char ubyte_array_in[ATT_LEN];
+      unsigned short ushort_array_in[ATT_LEN];
+      unsigned int uint_array_in[ATT_LEN];
+      long long int64_array_in[ATT_LEN];
+      unsigned long long uint64_array_in[ATT_LEN];
+      
         if ((ret = PIOc_get_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, ubyte_array_in)))
             return ret;
         if ((ret = PIOc_get_att_ushort(ncid, varid[8], USHORT_ATT_NAME, ushort_array_in)))
diff --git a/tests/cunit/test_pioc_unlim.c b/tests/cunit/test_pioc_unlim.c
index 725132cdfce6..88621523e374 100644
--- a/tests/cunit/test_pioc_unlim.c
+++ b/tests/cunit/test_pioc_unlim.c
@@ -1,4 +1,4 @@
-/*
+ /*
  * Tests for PIO Functions. In this test we use a simple 3D variable,
  * with an unlimited dimension. The data will have two timesteps, and
  * 4x4 elements each timestep.
@@ -236,7 +236,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te
     int ncid;
     int varid;
     int my_test_size;
-    char filename[NC_MAX_NAME + 1];
+    char filename[PIO_MAX_NAME + 1];
     int ret; /* Return code. */
 
     if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
diff --git a/tests/cunit/test_rearr.c b/tests/cunit/test_rearr.c
index 02397af6f8ff..3f29022713ae 100644
--- a/tests/cunit/test_rearr.c
+++ b/tests/cunit/test_rearr.c
@@ -479,18 +479,18 @@ int test_expand_region()
 }
 
 /* Test define_iodesc_datatypes() function. */
-int test_define_iodesc_datatypes()
+int test_define_iodesc_datatypes(int my_rank)
 {
 #define NUM_REARRANGERS 2
     int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    io_desc_t iodesc;
     int mpierr;
-    int ret;
+    int ret = PIO_NOERR;
 
     /* Run the functon. */
     for (int r = 0; r < NUM_REARRANGERS; r++)
     {
         iosystem_desc_t ios;
-        io_desc_t iodesc;
 
         /* Set up test for IO task with BOX rearranger to create one type. */
         ios.ioproc = 1; /* this is IO proc. */
@@ -500,15 +500,22 @@ int test_define_iodesc_datatypes()
         iodesc.nrecvs = 1; /* Number of types created. */
         iodesc.mpitype = MPI_INT;
         iodesc.stype = NULL; /* Array of MPI types will be created here. */
+        iodesc.rcount = NULL;
+        iodesc.rfrom = NULL;
+        iodesc.rindex = NULL;
+        iodesc.scount = NULL;
+        iodesc.sindex = NULL;
+        iodesc.rtype = NULL;
+        iodesc.stype = NULL;
 
         /* Allocate space for arrays in iodesc that will be filled in
          * define_iodesc_datatypes(). */
         if (!(iodesc.rcount = malloc(iodesc.nrecvs * sizeof(int))))
-            return PIO_ENOMEM;
+            BAIL(PIO_ENOMEM);
         if (!(iodesc.rfrom = malloc(iodesc.nrecvs * sizeof(int))))
-            return PIO_ENOMEM;
+            BAIL(PIO_ENOMEM);
         if (!(iodesc.rindex = malloc(1 * sizeof(PIO_Offset))))
-            return PIO_ENOMEM;
+            BAIL(PIO_ENOMEM);
         iodesc.rindex[0] = 0;
         iodesc.rcount[0] = 1;
 
@@ -518,9 +525,9 @@ int test_define_iodesc_datatypes()
         int num_send_types = iodesc.rearranger == PIO_REARR_BOX ? ios.num_iotasks : 1;
 
         if (!(iodesc.sindex = malloc(num_send_types * sizeof(PIO_Offset))))
-            return PIO_ENOMEM;
+            BAIL(PIO_ENOMEM);
         if (!(iodesc.scount = malloc(num_send_types * sizeof(int))))
-            return PIO_ENOMEM;
+            BAIL(PIO_ENOMEM);
         for (int st = 0; st < num_send_types; st++)
         {
             iodesc.sindex[st] = 0;
@@ -534,23 +541,67 @@ int test_define_iodesc_datatypes()
         /* We created send types, so free them. */
         for (int st = 0; st < num_send_types; st++)
             if ((mpierr = MPI_Type_free(&iodesc.stype[st])))
-                MPIERR(mpierr);
+                MPIBAIL(mpierr);
 
         /* We created one receive type, so free it. */
         if ((mpierr = MPI_Type_free(&iodesc.rtype[0])))
-            MPIERR(mpierr);
+            MPIBAIL(mpierr);
 
         /* Free resources. */
+        if (iodesc.rtype)
+        {
+            free(iodesc.rtype);
+            iodesc.rtype = NULL;
+        }
+        if (iodesc.sindex)
+        {
+            free(iodesc.sindex);
+            iodesc.sindex = NULL;
+        }
+        if (iodesc.scount)
+        {
+            free(iodesc.scount);
+            iodesc.scount = NULL;
+        }
+        if (iodesc.stype)
+        {
+            free(iodesc.stype);
+            iodesc.stype = NULL;
+        }
+        if (iodesc.rcount)
+        {
+            free(iodesc.rcount);
+            iodesc.rcount = NULL;
+        }
+        if (iodesc.rfrom)
+        {
+            free(iodesc.rfrom);
+            iodesc.rfrom = NULL;
+        }
+        if (iodesc.rindex)
+        {
+            free(iodesc.rindex);
+            iodesc.rindex = NULL;
+        }
+    }
+
+exit:
+    if (iodesc.rtype)
         free(iodesc.rtype);
+    if (iodesc.sindex)
         free(iodesc.sindex);
+    if (iodesc.scount)
         free(iodesc.scount);
+    if (iodesc.stype)
         free(iodesc.stype);
+    if (iodesc.rcount)
         free(iodesc.rcount);
+    if (iodesc.rfrom)
         free(iodesc.rfrom);
+    if (iodesc.rindex)
         free(iodesc.rindex);
-    }
 
-    return 0;
+    return ret;
 }
 
 /* Test the compute_counts() function with the box rearranger. */
@@ -821,7 +872,7 @@ int test_box_rearrange_create_2(MPI_Comm test_comm, int my_rank)
 
     /* Check some results. */
     if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen ||
-        iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill)
+        (iodesc->llen != (my_rank ? 0 : 8)) || !iodesc->needsfill)
         return ERR_WRONG;
 
     for (int i = 0; i < ios->num_iotasks; i++)
@@ -910,11 +961,11 @@ int test_default_subset_partition(MPI_Comm test_comm, int my_rank)
 int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank)
 {
     iosystem_desc_t *ios;
-    io_desc_t *iodesc;
+    io_desc_t *iodesc = NULL;
     void *sbuf = NULL;
     void *rbuf = NULL;
     int nvars = 1;
-    io_region *ior1;
+    io_region *ior1 = NULL;
     int maplen = 2;
     PIO_Offset compmap[2] = {1, 0};
     const int gdimlen[NDIM1] = {8};
@@ -924,17 +975,17 @@ int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank)
 
     /* Allocate some space for data. */
     if (!(sbuf = calloc(4, sizeof(int))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
     if (!(rbuf = calloc(4, sizeof(int))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     /* Allocate IO system info struct for this test. */
     if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     /* Allocate IO desc struct for this test. */
     if (!(iodesc = calloc(1, sizeof(io_desc_t))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     ios->ioproc = 1;
     ios->compproc = 1;
@@ -970,17 +1021,17 @@ int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank)
     ios->union_rank = my_rank;
     ios->num_comptasks = 4;
     if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
-        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        BAIL(PIO_ENOMEM);
     for (int i = 0; i < TARGET_NTASKS; i++)
         ios->ioranks[i] = i;
     if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
-        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        BAIL(PIO_ENOMEM);
     for (int i = 0; i < TARGET_NTASKS; i++)
         ios->compranks[i] = i;
 
     /* This is how we allocate a region. */
     if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
-        return ret;
+        BAIL(ret);
     ior1->next = NULL;
     if (my_rank == 0)
         ior1->count[0] = 8;
@@ -989,56 +1040,73 @@ int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank)
 
     /* Create the box rearranger. */
     if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
-        return ret;
+        BAIL(ret);
 
     /* Run the function to test. */
     if ((ret = rearrange_comp2io(ios, iodesc, sbuf, rbuf, nvars)))
-        return ret;
+        BAIL(ret);
 
     /* We created send types, so free them. */
     for (int st = 0; st < num_send_types; st++)
         if (iodesc->stype[st] != PIO_DATATYPE_NULL)
             if ((mpierr = MPI_Type_free(&iodesc->stype[st])))
-                MPIERR(mpierr);
+                MPIBAIL(mpierr);
 
     /* We created one receive type, so free it. */
     if (iodesc->rtype)
         for (int r = 0; r < iodesc->nrecvs; r++)
             if (iodesc->rtype[r] != PIO_DATATYPE_NULL)
                 if ((mpierr = MPI_Type_free(&iodesc->rtype[r])))
-                    MPIERR(mpierr);
+		    MPIBAIL(mpierr);
 
+exit:
     /* Free resources allocated in library code. */
-    free(iodesc->rtype);
-    free(iodesc->sindex);
-    free(iodesc->scount);
-    free(iodesc->stype);
-    free(iodesc->rcount);
-    free(iodesc->rfrom);
-    free(iodesc->rindex);
+    if (iodesc->rtype)
+	free(iodesc->rtype);
+    if (iodesc->sindex)
+	free(iodesc->sindex);
+    if (iodesc->scount)
+	free(iodesc->scount);
+    if (iodesc->stype)
+	free(iodesc->stype);
+    if (iodesc->rcount)
+	free(iodesc->rcount);
+    if (iodesc->rfrom)
+	free(iodesc->rfrom);
+    if (iodesc->rindex)
+	free(iodesc->rindex);
 
     /* Free resources from test. */
-    free(ior1->start);
-    free(ior1->count);
-    free(ior1);
-    free(ios->ioranks);
-    free(ios->compranks);
-    free(iodesc);
-    free(ios);
-    free(sbuf);
-    free(rbuf);
-
-    return 0;
+    if (ior1)
+    {
+	free(ior1->start);
+	free(ior1->count);
+	free(ior1);
+    }
+    if (ios)
+    {
+	free(ios->ioranks);
+	free(ios->compranks);
+	free(ios);
+    }
+    if (iodesc)
+	free(iodesc);
+    if (sbuf)
+	free(sbuf);
+    if (rbuf)
+	free(rbuf);
+
+    return ret;
 }
 
 /* Test function rearrange_io2comp. */
 int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank)
 {
-    iosystem_desc_t *ios;
-    io_desc_t *iodesc;
+    iosystem_desc_t *ios = NULL;
+    io_desc_t *iodesc = NULL;
     void *sbuf = NULL;
     void *rbuf = NULL;
-    io_region *ior1;
+    io_region *ior1 = NULL;
     int maplen = 2;
     PIO_Offset compmap[2] = {1, 0};
     const int gdimlen[NDIM1] = {8};
@@ -1048,17 +1116,17 @@ int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank)
 
     /* Allocate some space for data. */
     if (!(sbuf = calloc(4, sizeof(int))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
     if (!(rbuf = calloc(4, sizeof(int))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     /* Allocate IO system info struct for this test. */
     if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     /* Allocate IO desc struct for this test. */
     if (!(iodesc = calloc(1, sizeof(io_desc_t))))
-        return PIO_ENOMEM;
+        BAIL(PIO_ENOMEM);
 
     ios->ioproc = 1;
     ios->io_rank = my_rank;
@@ -1098,17 +1166,17 @@ int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank)
     ios->num_comptasks = 4;
     ios->num_uniontasks = 4;
     if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
-        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        BAIL(PIO_ENOMEM);
     for (int i = 0; i < TARGET_NTASKS; i++)
         ios->ioranks[i] = i;
     if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
-        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        BAIL(PIO_ENOMEM);
     for (int i = 0; i < TARGET_NTASKS; i++)
         ios->compranks[i] = i;
 
     /* This is how we allocate a region. */
     if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
-        return ret;
+        BAIL(ret);
     ior1->next = NULL;
     if (my_rank == 0)
         ior1->count[0] = 8;
@@ -1121,42 +1189,55 @@ int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank)
 
     /* Run the function to test. */
     if ((ret = rearrange_io2comp(ios, iodesc, sbuf, rbuf)))
-        return ret;
+        BAIL(ret);
 
     /* We created send types, so free them. */
     for (int st = 0; st < num_send_types; st++)
         if (iodesc->stype[st] != PIO_DATATYPE_NULL)
             if ((mpierr = MPI_Type_free(&iodesc->stype[st])))
-                MPIERR(mpierr);
+                MPIBAIL(mpierr);
 
     /* We created one receive type, so free it. */
     if (iodesc->rtype)
         for (int r = 0; r < iodesc->nrecvs; r++)
             if (iodesc->rtype[r] != PIO_DATATYPE_NULL)
                 if ((mpierr = MPI_Type_free(&iodesc->rtype[r])))
-                    MPIERR(mpierr);
+                    MPIBAIL(mpierr);
 
+exit:
     /* Free resources allocated in library code. */
-    free(iodesc->rtype);
-    free(iodesc->sindex);
-    free(iodesc->scount);
-    free(iodesc->stype);
-    free(iodesc->rcount);
-    free(iodesc->rfrom);
-    free(iodesc->rindex);
+    if (iodesc)
+    {
+        free(iodesc->rtype);
+        free(iodesc->sindex);
+        free(iodesc->scount);
+        free(iodesc->stype);
+        free(iodesc->rcount);
+        free(iodesc->rfrom);
+        free(iodesc->rindex);
+    }
 
     /* Free resources from test. */
-    free(ior1->start);
-    free(ior1->count);
-    free(ior1);
-    free(ios->ioranks);
-    free(ios->compranks);
-    free(iodesc);
-    free(ios);
-    free(sbuf);
-    free(rbuf);
-
-    return 0;
+    if (ior1->start)
+        free(ior1->start);
+    if (ior1->count)
+        free(ior1->count);
+    if (ior1)
+        free(ior1);
+    if (ios->ioranks)
+        free(ios->ioranks);
+    if (ios->compranks)
+        free(ios->compranks);
+    if (iodesc)
+        free(iodesc);
+    if (ios)
+        free(ios);
+    if (sbuf)
+        free(sbuf);
+    if (rbuf)
+        free(rbuf);
+
+    return ret;
 }
 
 /* These tests do not need an iosysid. */
@@ -1188,7 +1269,7 @@ int run_no_iosys_tests(int my_rank, MPI_Comm test_comm)
     if ((ret = test_create_mpi_datatypes()))
         return ret;
 
-    if ((ret = test_define_iodesc_datatypes()))
+    if ((ret = test_define_iodesc_datatypes(my_rank)))
         return ret;
 
     if ((ret = test_compare_offsets()))
@@ -1497,7 +1578,7 @@ int main(int argc, char **argv)
                     return ret;
 
                 /* Finalize PIO system. */
-                if ((ret = PIOc_finalize(iosysid)))
+                if ((ret = PIOc_free_iosystem(iosysid)))
                     return ret;
             } /* next numio */
         } /* next rearranger */
diff --git a/tests/cunit/test_shared.c b/tests/cunit/test_shared.c
index 9d999700994c..cd73089177d3 100644
--- a/tests/cunit/test_shared.c
+++ b/tests/cunit/test_shared.c
@@ -48,7 +48,7 @@ int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm,
                 return ret;
 
             /* Finalize the IO system. Only call this from the computation tasks. */
-            if ((ret = PIOc_finalize(iosysid[c])))
+            if ((ret = PIOc_free_iosystem(iosysid[c])))
                 ERR(ret);
             if ((mpierr = MPI_Comm_free(&comp_comm[c])))
                 MPIERR(mpierr);
@@ -118,7 +118,7 @@ int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm
         return ret;
 
     /* Finalize PIO system. */
-    if ((ret = PIOc_finalize(iosysid)))
+    if ((ret = PIOc_free_iosystem(iosysid)))
         return ret;
 
     return PIO_NOERR;
diff --git a/tests/cunit/test_spmd.c b/tests/cunit/test_spmd.c
index e7e64664f4e9..5db8c3a3b811 100644
--- a/tests/cunit/test_spmd.c
+++ b/tests/cunit/test_spmd.c
@@ -61,7 +61,7 @@ int run_spmd_tests(MPI_Comm test_comm)
 
     /* Get the size of the int type for MPI. (Should always be 4.) */
     if ((mpierr = MPI_Type_size(MPI_INT, &type_size)))
-        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        MPIERR(mpierr);
     assert(type_size == sizeof(int));
 
     /* Initialize the arrays. */
@@ -222,13 +222,13 @@ int test_determine_procs()
 #define TWO_COMPONENTS 2
 #define THREE_PROCS 3
     int ret;
-    
+
     {
         int num_io_procs = 1;
         int component_count = ONE_COMPONENT;
         int num_procs_per_comp[ONE_COMPONENT] = {1};
         int *my_proc_list[ONE_COMPONENT];
-        
+
         if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp, NULL,
                                    my_proc_list)))
             return ret;
@@ -241,17 +241,17 @@ int test_determine_procs()
             free(my_proc_list[c]);
         }
     }
-    
+
     {
         int num_io_procs = 3;
         int component_count = TWO_COMPONENTS;
         int num_procs_per_comp[TWO_COMPONENTS] = {1, 1};
         int *my_proc_list[TWO_COMPONENTS];
-        
+
         if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp, NULL,
                                    my_proc_list)))
             return ret;
-        
+
         /* Check results and free resources. */
         for (int c = 0; c < TWO_COMPONENTS; c++)
         {
@@ -260,17 +260,17 @@ int test_determine_procs()
             free(my_proc_list[c]);
         }
     }
-    
+
     {
         int num_io_procs = 3;
         int component_count = TWO_COMPONENTS;
         int num_procs_per_comp[TWO_COMPONENTS] = {THREE_PROCS, THREE_PROCS};
         int *my_proc_list[TWO_COMPONENTS];
-        
+
         if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp, NULL,
                                    my_proc_list)))
             return ret;
-        
+
         /* Check results and free resources. */
         for (int c = 0; c < TWO_COMPONENTS; c++)
         {
@@ -280,7 +280,7 @@ int test_determine_procs()
             free(my_proc_list[c]);
         }
     }
-    
+
     {
         int num_io_procs = 3;
         int component_count = TWO_COMPONENTS;
@@ -289,11 +289,11 @@ int test_determine_procs()
         int proc_list_2[THREE_PROCS] = {11, 12, 13};
         int *proc_list[TWO_COMPONENTS] = {proc_list_1, proc_list_2};
         int *my_proc_list[TWO_COMPONENTS];
-        
+
         if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp,
                                    (int **)proc_list, my_proc_list)))
             return ret;
-        
+
         /* Check results and free resources. */
         for (int c = 0; c < TWO_COMPONENTS; c++)
         {
@@ -303,7 +303,7 @@ int test_determine_procs()
             free(my_proc_list[c]);
         }
     }
-    
+
     return PIO_NOERR;
 }
 
@@ -483,7 +483,7 @@ int test_varlists3()
         return ERR_WRONG;
     if (get_var_desc(3, &varlist, &var_desc) != PIO_ENOTVAR)
         return ERR_WRONG;
-    
+
     return 0;
 }
 
@@ -664,9 +664,10 @@ int test_CalcStartandCount()
     return 0;
 }
 
-/* Test the GDCblocksize() function. */
-int run_GDCblocksize_tests(MPI_Comm test_comm)
+/* Test the GCDblocksize() function. */
+int run_GCDblocksize_tests(MPI_Comm test_comm)
 {
+
     {
         int arrlen = 1;
         PIO_Offset arr_in[1] = {0};
@@ -703,7 +704,7 @@ int run_GDCblocksize_tests(MPI_Comm test_comm)
         PIO_Offset blocksize;
 
         blocksize = GCDblocksize(arrlen, arr_in);
-        if (blocksize != 1)
+        if (blocksize != 2)
             return ERR_WRONG;
     }
 
@@ -732,7 +733,6 @@ int main(int argc, char **argv)
     if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
                               TARGET_NTASKS, -1, &test_comm)))
         ERR(ERR_INIT);
-
     /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
      * nothing. */
     if (my_rank < TARGET_NTASKS)
@@ -746,7 +746,7 @@ int main(int argc, char **argv)
         if ((ret = run_sc_tests(test_comm)))
             return ret;
 
-        if ((ret = run_GDCblocksize_tests(test_comm)))
+        if ((ret = run_GCDblocksize_tests(test_comm)))
             return ret;
 
         if ((ret = run_spmd_tests(test_comm)))
@@ -780,7 +780,7 @@ int main(int argc, char **argv)
             return ret;
 
         /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
+        if ((ret = PIOc_free_iosystem(iosysid)))
             return ret;
 
     } /* endif my_rank < TARGET_NTASKS */
diff --git a/tests/general/CMakeLists.txt b/tests/general/CMakeLists.txt
index 09358d5b9fd2..4f54c7831782 100644
--- a/tests/general/CMakeLists.txt
+++ b/tests/general/CMakeLists.txt
@@ -45,6 +45,10 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
   #        PRIVATE -mismatch_all)
 endif ()
 
+if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+     set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g")
+endif()
+
 #==============================================================================
 #  DEFINE THE TARGETS AND TESTS
 #==============================================================================
diff --git a/tests/general/Makefile.am b/tests/general/Makefile.am
new file mode 100644
index 000000000000..d7f11687daa8
--- /dev/null
+++ b/tests/general/Makefile.am
@@ -0,0 +1,103 @@
+## This is the automake file for building the Fortran general tests
+## for the PIO library.
+
+# Ed Hartnett 3/25/19
+
+# Parallel builds don't currently work in this directory.
+.NOTPARALLEL:
+
+# Put together AM_CPPFLAGS and AM_LDFLAGS.
+include $(top_srcdir)/set_flags.am
+
+LDADD = libpio_tutil.la	\
+${top_builddir}/src/flib/libpiof.la		\
+${top_builddir}/src/clib/libpio.la
+
+# There is a test utility mod file in this subdir which must be built.
+SUBDIRS = util
+
+# Build these uninstalled convenience libraries.
+noinst_LTLIBRARIES = libpio_tutil.la libpio_rearr_opts.la
+
+# The convenience libraries depends on their source.
+libpio_tutil_la_SOURCES = pio_tutil.F90 # configure copies this from util dir.
+libpio_rearr_opts_la_SOURCES = pio_rearr_opts.F90
+
+# Each mod file depends on the .o file.
+pio_tutil.mod: pio_tutil.$(OBJEXT)
+pio_rearr_opts_tgv.mod: pio_rearr_opts.$(OBJEXT)
+
+# Some mods are dependant on other mods in this dir.
+pio_rearr_opts.$(OBJEXT): pio_tutil.mod
+
+BUILT_SOURCES = pio_tutil.mod pio_rearr_opts_tgv.mod
+
+# Build the test for make check.
+check_PROGRAMS = pio_init_finalize pio_file_simple_tests	\
+pio_file_fail ncdf_simple_tests ncdf_get_put ncdf_fail ncdf_inq	\
+pio_rearr pio_rearr_opts2 pio_decomp_tests	\
+pio_decomp_tests_1d pio_decomp_tests_2d pio_decomp_tests_3d	\
+pio_decomp_frame_tests pio_decomp_fillval pio_iosystem_tests	\
+pio_iosystem_tests2 pio_iosystem_tests3
+
+pio_init_finalize_SOURCES = pio_init_finalize.F90
+pio_file_simple_tests_SOURCES = pio_file_simple_tests.F90
+pio_file_fail_SOURCES = pio_file_fail.F90
+ncdf_simple_tests_SOURCES = ncdf_simple_tests.F90
+ncdf_get_put_SOURCES = ncdf_get_put.F90
+ncdf_fail_SOURCES = ncdf_fail.F90
+ncdf_inq_SOURCES = ncdf_inq.F90
+pio_rearr_SOURCES = pio_rearr.F90
+#pio_rearr_opts_SOURCES = pio_rearr_opts.F90
+pio_rearr_opts2_SOURCES = pio_rearr_opts2.F90
+pio_decomp_tests_SOURCES = pio_decomp_tests.F90
+pio_decomp_tests_1d_SOURCES = pio_decomp_tests_1d.F90
+pio_decomp_tests_2d_SOURCES = pio_decomp_tests_2d.F90
+pio_decomp_tests_3d_SOURCES = pio_decomp_tests_3d.F90
+pio_decomp_frame_tests_SOURCES = pio_decomp_frame_tests.F90
+pio_decomp_fillval_SOURCES = pio_decomp_fillval.F90
+pio_iosystem_tests_SOURCES = pio_iosystem_tests.F90
+pio_iosystem_tests2_SOURCES = pio_iosystem_tests2.F90
+pio_iosystem_tests3_SOURCES = pio_iosystem_tests3.F90
+
+if RUN_TESTS
+# Tests will run from a bash script.
+TESTS = run_tests.sh
+endif # RUN_TESTS
+
+%.F90: %.F90.in
+	util/pio_tf_f90gen.pl --annotate-source --out=$@ $<
+
+ncdf_fail.F90: ncdf_fail.F90.in
+ncdf_get_put.F90: ncdf_get_put.F90.in
+ncdf_inq.F90: ncdf_inq.F90.in
+ncdf_simple_tests.F90:ncdf_simple_tests.F90.in
+pio_decomp_fillval.F90:pio_decomp_fillval.F90.in
+pio_decomp_frame_tests.F90:pio_decomp_frame_tests.F90.in
+pio_decomp_tests_1d.F90:pio_decomp_tests_1d.F90.in
+pio_decomp_tests_2d.F90:pio_decomp_tests_2d.F90.in
+pio_decomp_tests_3d.F90:pio_decomp_tests_3d.F90.in
+pio_decomp_tests.F90:pio_decomp_tests.F90.in
+pio_file_fail.F90:pio_file_fail.F90.in
+pio_file_simple_tests.F90:pio_file_simple_tests.F90.in
+pio_init_finalize.F90:pio_init_finalize.F90.in
+pio_iosystem_tests2.F90:pio_iosystem_tests2.F90.in
+pio_iosystem_tests3.F90:pio_iosystem_tests3.F90.in
+pio_iosystem_tests.F90:pio_iosystem_tests.F90.in
+pio_rearr.F90:pio_rearr.F90.in
+pio_rearr_opts2.F90:pio_rearr_opts2.F90.in
+pio_rearr_opts.F90:pio_rearr_opts.F90.in
+
+# Distribute the test script.
+EXTRA_DIST = CMakeLists.txt run_tests.sh ncdf_fail.F90.in		\
+ncdf_get_put.F90.in ncdf_inq.F90.in ncdf_simple_tests.F90.in		\
+pio_decomp_fillval.F90.in pio_decomp_frame_tests.F90.in			\
+pio_decomp_tests_1d.F90.in pio_decomp_tests_2d.F90.in			\
+pio_decomp_tests_3d.F90.in pio_decomp_tests.F90.in pio_fail.F90.in	\
+pio_file_fail.F90.in pio_file_simple_tests.F90.in			\
+pio_init_finalize.F90.in pio_iosystem_tests2.F90.in			\
+pio_iosystem_tests3.F90.in pio_iosystem_tests.F90.in pio_rearr.F90.in	\
+pio_rearr_opts2.F90.in pio_rearr_opts.F90.in
+
+# Clean up files produced during testing.
+CLEANFILES = *.nc *.log *.mod
diff --git a/tests/general/ncdf_simple_tests.F90.in b/tests/general/ncdf_simple_tests.F90.in
index b1d8133194aa..2af0e21c651f 100644
--- a/tests/general/ncdf_simple_tests.F90.in
+++ b/tests/general/ncdf_simple_tests.F90.in
@@ -115,6 +115,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion
   integer :: pio_dim
   integer :: i, ierr
 
+
   do i=1,VEC_LOCAL_SZ
     compdof_rel_disps(i) = i
   end do
@@ -147,11 +148,14 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion
 
   call PIO_syncfile(pio_file)
 
-  ! Read the variable back (data conversion might occur)
-  call PIO_read_darray(pio_file, pio_var, riodesc, rbuf, ierr)
-  PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(data_fname))
-
-  PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+  if (tgv_iotype .eq. PIO_iotype_pnetcdf) then
+     PIO_TF_LOG(0,*) "WARNING: Data type conversion not supported in pnetcdf vard interface, skipping test"
+  else
+     ! Read the variable back (data conversion might occur)
+     call PIO_read_darray(pio_file, pio_var, riodesc, rbuf, ierr)
+     PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(data_fname))
+     PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+  endif
 
   call PIO_closefile(pio_file)
   call PIO_deletefile(pio_tf_iosystem_, data_fname);
diff --git a/tests/general/pio_decomp_tests_1d.F90.in b/tests/general/pio_decomp_tests_1d.F90.in
index 48d4f95a101f..528ce3de0461 100644
--- a/tests/general/pio_decomp_tests_1d.F90.in
+++ b/tests/general/pio_decomp_tests_1d.F90.in
@@ -122,7 +122,7 @@ END SUBROUTINE
 
 ! Test block cyclic interface
 ! Write with one decomp and read with another
-! Test all combs 
+! Test all combs
 ! - no rearrage read + no rearrange write
 ! - rearrage read + no rearrange write
 ! - no rearrage read + rearrange write
@@ -130,7 +130,7 @@ END SUBROUTINE
 PIO_TF_TEMPLATE
 PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc
   implicit none
-  type(var_desc_t)  :: pio_var
+  type(var_desc_t)  :: pio_var1, pio_var2
   type(file_desc_t) :: pio_file
   character(len=PIO_TF_MAX_STR_LEN) :: filename
   type(io_desc_t) :: wr_iodesc, rd_iodesc
@@ -185,32 +185,41 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc
       filename = "test_pio_decomp_simple_tests.testfile"
       do i=1,num_iotypes
         PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
-        ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+        ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
         PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
 
         ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
         PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
 
-        ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+        ierr = PIO_def_var(pio_file, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var1)
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+        ierr = PIO_def_var(pio_file, 'PIO_TF_test_var2', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var2)
         PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
 
         ierr = PIO_enddef(pio_file)
         PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
 
         ! Write the variable out
-        call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+        call PIO_write_darray(pio_file, pio_var1, wr_iodesc, wbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+        wbuf = wbuf + 200
+
+        ! Write the variable out
+        call PIO_write_darray(pio_file, pio_var2, wr_iodesc, wbuf, ierr)
         PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
 
         call PIO_syncfile(pio_file)
 
         rbuf = 0
-        call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+        call PIO_read_darray(pio_file, pio_var1, rd_iodesc, rbuf, ierr)
         PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
 
         PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
 
         call PIO_closefile(pio_file)
-        
+        wbuf = wbuf - 200
         call PIO_deletefile(pio_tf_iosystem_, filename);
       end do
 
@@ -266,7 +275,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes
   filename = "test_pio_decomp_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
-    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
 
     ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
@@ -290,7 +299,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes
     PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
 
     call PIO_closefile(pio_file)
-    
+
     call PIO_deletefile(pio_tf_iosystem_, filename);
   end do
 
@@ -304,3 +313,116 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes
   deallocate(rbuf)
   deallocate(wbuf)
 PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_with_holes
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_random
+  use mpi, only : MPI_INT
+  implicit none
+  type(var_desc_t)  :: pio_var1, pio_var2
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc
+  integer, dimension(:), allocatable :: compdof, gcompdof
+  integer, dimension(1) :: count
+  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, j, ierr, lsz
+  integer :: tmp
+  real :: u
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - random order same local size
+  count(1) = 4
+  dims(1) = count(1)*pio_tf_world_sz_
+  if(pio_tf_world_rank_ == 0) then
+     allocate(gcompdof(dims(1)))
+     gcompdof = 0
+     do i=1,dims(1)
+        gcompdof(i) = i
+     enddo
+     do i=dims(1),1,-1
+        call random_number(u)
+        j = CEILING(real(i)*u)
+        tmp = gcompdof(j)
+        gcompdof(j) = gcompdof(i)
+        gcompdof(i) = tmp
+     enddo
+  endif
+  allocate(compdof(count(1)))
+  call mpi_scatter(gcompdof, count(1), MPI_INT, compdof, 4, MPI_INT, 0, pio_tf_comm_, ierr)
+  if(allocated(gcompdof)) deallocate(gcompdof)
+  allocate(rbuf(count(1)))
+  allocate(wbuf(count(1)))
+  do i=1,count(1)
+    wbuf(i) = compdof(i)
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var2', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var1, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    wbuf = wbuf + 200
+
+    call PIO_write_darray(pio_file, pio_var2, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+
+    call PIO_read_darray(pio_file, pio_var1, wr_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    wbuf = wbuf - 200
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    wbuf = wbuf + 200
+
+    call PIO_read_darray(pio_file, pio_var2, wr_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+
+    wbuf = wbuf + 200
+
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_random
diff --git a/tests/general/run_tests.sh b/tests/general/run_tests.sh
new file mode 100755
index 000000000000..63d89e560927
--- /dev/null
+++ b/tests/general/run_tests.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# This is a test script for PIO for tests/general directory.
+# Ed Hartnett 3/25/19
+
+# Stop execution of script if error is returned.
+set -e
+
+# Stop loop if ctrl-c is pressed.
+trap exit INT TERM
+
+printf 'running PIO tests...\n'
+
+PIO_TESTS='pio_init_finalize pio_file_simple_tests pio_file_fail '\
+'ncdf_simple_tests ncdf_get_put ncdf_fail ncdf_inq pio_rearr '\
+'pio_decomp_tests pio_decomp_tests_1d '\
+'pio_decomp_tests_2d pio_decomp_tests_3d pio_decomp_frame_tests '\
+'pio_decomp_fillval pio_iosystem_tests pio_iosystem_tests2 '\
+'pio_iosystem_tests3'
+# pio_rearr_opts pio_rearr_opts2
+
+success1=true
+for TEST in $PIO_TESTS
+do
+    success1=false
+    echo "running ${TEST}"
+    mpiexec -n 4 ./${TEST} && success1=true
+    if test $success1 = false; then
+        break
+    fi
+done
+
+# Did we succeed?
+if test x$success1 = xtrue; then
+    exit 0
+fi
+exit 1
diff --git a/tests/general/test_memleak.c b/tests/general/test_memleak.c
index 3c81f477df1e..48abc4a00a2f 100644
--- a/tests/general/test_memleak.c
+++ b/tests/general/test_memleak.c
@@ -66,7 +66,7 @@ char err_buffer[MPI_MAX_ERROR_STRING];
 int resultlen;
 
 /** The dimension names. */
-char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
 
 /** Length of the dimensions in the sample data. */
 int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
@@ -109,10 +109,10 @@ main(int argc, char **argv)
      * (serial4 and parallel4) will be in netCDF-4/HDF5
      * format. All four can be read by the netCDF library, and all
      * will contain the same contents. */
-    char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc",
-							  "test_nc4_classic.nc",
-							  "test_nc4_serial4.nc",
-							  "test_nc4_parallel4.nc"};
+    char filename[NUM_NETCDF_FLAVORS][PIO_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc",
+                                                           "test_nc4_classic.nc",
+                                                           "test_nc4_serial4.nc",
+                                                           "test_nc4_parallel4.nc"};
 
     /** Number of processors that will do IO. In this example we
      * will do IO from all processors. */
@@ -122,9 +122,6 @@ main(int argc, char **argv)
      * example. */
     int ioproc_stride = 1;
 
-    /** Number of the aggregator? Always 0 in this example. */
-    int numAggregator = 0;
-
     /** Zero based rank of first processor to be used for I/O. */
     int ioproc_start = 0;
 
diff --git a/tests/general/util/Makefile.am b/tests/general/util/Makefile.am
new file mode 100644
index 000000000000..208a28294698
--- /dev/null
+++ b/tests/general/util/Makefile.am
@@ -0,0 +1,6 @@
+## This is the automake file for building the Fortran general tests
+## util mod for the PIO library.
+
+# Ed Hartnett 3/29/19
+
+EXTRA_DIST = pio_tf_f90gen.pl pio_tutil.F90
diff --git a/tests/general/util/pio_tf_f90gen.pl b/tests/general/util/pio_tf_f90gen.pl
index f880cda1bebe..7cf2672cda52 100755
--- a/tests/general/util/pio_tf_f90gen.pl
+++ b/tests/general/util/pio_tf_f90gen.pl
@@ -563,7 +563,11 @@ sub parse_and_store_gen_templ_funcs
                         $ifline_num, \$is_transformed);
   }
   if($annotate_source){
-    $out_line = $out_line . "   ! $base_file_name:$ifline_num" . "\n";
+      if($out_line =~ /[^#]/){      
+	  $out_line .= "\n";
+      }else{
+	  $out_line = $out_line . "   ! $base_file_name:$ifline_num" . "\n";
+      }
   }
   if($verbose) { print "Adding \"$out_line\" to ${$ref_templ_funcname}\n"; }
   if(exists $template_funcs{${$ref_templ_funcname}}){
diff --git a/tests/general/util/pio_tutil.F90 b/tests/general/util/pio_tutil.F90
index 43c0b634b4a1..43717f848a72 100644
--- a/tests/general/util/pio_tutil.F90
+++ b/tests/general/util/pio_tutil.F90
@@ -286,10 +286,12 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
 #ifdef _NETCDF4
       ! netcdf, netcdf4p, netcdf4c
       num_iotypes = num_iotypes + 3
-#elif _NETCDF
+#else
+#ifdef _NETCDF
       ! netcdf
       num_iotypes = num_iotypes + 1
 #endif
+#endif
 #ifdef _PNETCDF
       ! pnetcdf
       num_iotypes = num_iotypes + 1
@@ -317,11 +319,13 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
       iotypes(i) = PIO_iotype_netcdf4p
       iotype_descs(i) = "NETCDF4P"
       i = i + 1
-#elif _NETCDF
+#else
+#ifdef _NETCDF
       ! netcdf
       iotypes(i) = PIO_iotype_netcdf
       iotype_descs(i) = "NETCDF"
       i = i + 1
+#endif
 #endif
   END SUBROUTINE
 
@@ -402,10 +406,12 @@ SUBROUTINE PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes)
     num_iotypes = 0
 #ifdef _NETCDF4
       ! netcdf, netcdf4p, netcdf4c
-      num_iotypes = num_iotypes + 3
-#elif _NETCDF
+    num_iotypes = num_iotypes + 3
+#else
+#ifdef _NETCDF
       ! netcdf
-      num_iotypes = num_iotypes + 1
+    num_iotypes = num_iotypes + 1
+#endif
 #endif
 #ifdef _PNETCDF
       ! pnetcdf
@@ -434,11 +440,13 @@ SUBROUTINE PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes)
       iotypes(i) = PIO_iotype_netcdf4p
       iotype_descs(i) = "NETCDF4P"
       i = i + 1
-#elif _NETCDF
+#else
+#ifdef _NETCDF
       ! netcdf
       iotypes(i) = PIO_iotype_netcdf
       iotype_descs(i) = "NETCDF"
       i = i + 1
+#endif
 #endif
   END SUBROUTINE
 
diff --git a/tests/performance/Makefile.am b/tests/performance/Makefile.am
new file mode 100644
index 000000000000..17a14e91ce8b
--- /dev/null
+++ b/tests/performance/Makefile.am
@@ -0,0 +1,39 @@
+## This is the automake file for building the Fortran performance
+## tests for the PIO library.
+
+# Ed Hartnett 4/6/19
+
+# Parallel builds don't currently work in this directory.
+.NOTPARALLEL:
+
+# Put together AM_CPPFLAGS and AM_LDFLAGS.
+include $(top_srcdir)/set_flags.am
+
+# Link to test util library and PIO Fortran and C libs.
+LDADD = $(top_builddir)/src/gptl/libperf_mod.la	\
+$(top_builddir)/src/gptl/libperf_utils.la	\
+${top_builddir}/tests/general/libpio_tutil.la	\
+${top_builddir}/src/flib/libpiof.la		\
+${top_builddir}/src/clib/libpio.la
+
+# Find perf_mod and perf_util.
+AM_CPPFLAGS += -I$(top_builddir)/src/gptl
+
+# Find pio_tutil.mod
+AM_CPPFLAGS += -I$(top_builddir)/tests/general
+
+# Build the test for make check.
+check_PROGRAMS = pioperf
+
+pioperf_SOURCES = pioperformance.F90
+
+if RUN_TESTS
+# Tests will run from a bash script.
+TESTS = run_tests.sh
+endif # RUN_TESTS
+
+EXTRA_DIST = CMakeLists.txt gensimple.pl Pioperformance.md pioperf.nl	\
+run_tests.sh
+
+# Clean up files produced during testing.
+CLEANFILES = *.nc *.log *.mod
diff --git a/tests/performance/Pioperformance.md b/tests/performance/Pioperformance.md
new file mode 100644
index 000000000000..186e489e8d30
--- /dev/null
+++ b/tests/performance/Pioperformance.md
@@ -0,0 +1,62 @@
+# Using pioperf to Measure Performance
+
+To run pioperformance you need a dof input file. I have a whole repo
+of them here:
+
+
+You need an input namelist:
+
+    &pioperf
+    decompfile=   '/gpfs/fs1/work/jedwards/sandboxes/piodecomps/576/piodecomp576tasks03dims01.dat',
+     pio_typenames = 'pnetcdf'
+     rearrangers = 1,2
+     nframes = 1
+     nvars = 1
+     niotasks = 64, 32, 16
+     /
+
+in the namelist all of the inputs are arrays and it will test all
+combinations of the inputs.  You need to run it on the number of tasks
+specified by the input dof There are also some options to use simple
+generated dof's instead of files.
+
+## Testing
+
+For the automated test you can generate a decomp internally by setting
+decompfile="ROUNDROBIN", or decompfile="BLOCK"
+
+They call init_ideal_dof which internally generates a dof as follows:
+
+    if(doftype .eq. 'ROUNDROBIN') then                                          
+       do i=1,varsize                                                           
+          compmap(i) = (i-1)*npe+mype+1                                         
+       enddo                                                                    
+    else if(doftype .eq. 'BLOCK') then                                          
+       do i=1,varsize                                                           
+          compmap(i) =  (i+varsize*mype)                                        
+       enddo                                                                    
+    endif
+    
+The size of the variable is npes*varsize where varsize can be set in
+the namelist. varsize is the variable array size per task. You can add
+variables by changing nvars in the namelist.
+
+When this is run, output like the following will appear:
+
+    mpiexec -n 4 ./pioperf
+    (t_initf) Read in prof_inparm namelist from: pioperf.nl
+     Testing decomp: BLOCK
+    iotype=           1
+    pioperformance.F90         298 Frame:                     1
+    pioperformance.F90         301 var:            1
+    RESULT: write       BOX         1         4         1        0.0319221529
+     RESULT: read       BOX         1         4         1        0.1658564029
+    pioperformance.F90         298 Frame:                     1
+    pioperformance.F90         301 var:            1
+    RESULT: write    SUBSET         1         4         1        0.0438470950
+     RESULT: read    SUBSET         1         4         1        0.1623275432
+
+These are read and write rates in units of MB/s for Box and Subset
+rearrangers - the time measured is from the call to readdof or
+writedof to the completion of the close (since writes are buffered the
+close needs to be included).
\ No newline at end of file
diff --git a/tests/performance/pioperf.nl b/tests/performance/pioperf.nl
new file mode 100644
index 000000000000..f2064e0cee93
--- /dev/null
+++ b/tests/performance/pioperf.nl
@@ -0,0 +1,9 @@
+&pioperf
+decompfile=   'BLOCK',
+ pio_typenames = 'pnetcdf' 'netcdf4p' 'netcdf4c' 'netcdf'
+ rearrangers = 1,2
+ nframes = 1
+ nvars = 10
+ niotasks = 4
+ varsize = 100000
+/
diff --git a/tests/performance/pioperformance.F90 b/tests/performance/pioperformance.F90
index 16fe145a193a..cb2931587408 100644
--- a/tests/performance/pioperformance.F90
+++ b/tests/performance/pioperformance.F90
@@ -1,4 +1,4 @@
-#define VARINT 1 
+#define VARINT 1
 !#define VARREAL 1
 !#define VARDOUBLE 1
 
@@ -8,11 +8,11 @@ program pioperformance
 #endif
   use perf_mod, only : t_initf, t_finalizef
   use pio, only : pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, &
-       pio_iotype_netcdf4c, pio_rearr_subset, pio_rearr_box
+       pio_iotype_netcdf4c, pio_rearr_subset, pio_rearr_box, pio_set_log_level
   implicit none
 #ifdef NO_MPIMOD
 #include 
-#endif  
+#endif
   integer, parameter :: max_io_task_array_size=64, max_decomp_files=64
 
 
@@ -102,8 +102,8 @@ program pioperformance
   if(rearrangers(1)==0) then
     rearrangers(1)=1
     rearrangers(2)=2
-  endif  
-
+  endif
+  i = pio_set_log_level(-1)
   do i=1,max_decomp_files
      if(len_trim(decompfile(i))==0) exit
      if(mype == 0) print *, ' Testing decomp: ',trim(decompfile(i))
@@ -112,7 +112,7 @@ program pioperformance
            do nv=1,max_nvars
               if(nvars(nv)>0) then
                  call pioperformancetest(decompfile(i), piotypes(1:niotypes), mype, npe, &
-                      rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof) 
+                      rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof)
               endif
            enddo
         endif
@@ -133,7 +133,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
     integer, intent(in) :: piotypes(:)
     integer, intent(in) :: rearrangers(:)
     integer, intent(inout) :: niotasks(:)
-    integer, intent(in) :: nframes 
+    integer, intent(in) :: nframes
     integer, intent(in) :: nvars
     integer, intent(in) :: varsize
     logical, intent(in) :: unlimdimindof
@@ -206,7 +206,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 !       if(gmaplen /= product(gdims)) then
 !          print *,__FILE__,__LINE__,gmaplen,gdims
 !       endif
-    
+
        allocate(ifld(maplen,nvars))
        allocate(ifld_in(maplen,nvars,nframes))
 
@@ -240,9 +240,9 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
              print *,'iotype=',piotypes(k)
           endif
 !          if(iotype==PIO_IOTYPE_PNETCDF) then
-!             mode = PIO_64BIT_DATA
+             mode = PIO_64BIT_DATA
 !          else
-             mode = 0
+!             mode = 0
 !          endif
           do rearrtype=1,2
              rearr = rearrangers(rearrtype)
@@ -255,9 +255,9 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                 stride = max(1,npe/ntasks)
 
                 call pio_init(mype, comm, ntasks, 0, stride, PIO_REARR_SUBSET, iosystem)
-                   
+
                 write(fname, '(a,i1,a,i4.4,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc'
-		
+
                 ierr =  PIO_CreateFile(iosystem, File, iotype, trim(fname), mode)
 
                 call WriteMetadata(File, gdims, vari, varr, vard, unlimdimindof)
@@ -295,10 +295,10 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                       call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr)
 #endif
                    endif
-                   if(mype==0) print *,__FILE__,__LINE__,'Frame: ',recnum
+                   !if(mype==0) print *,__FILE__,__LINE__,'Frame: ',recnum
 
-                   do nv=1,nvars   
-                      if(mype==0) print *,__FILE__,__LINE__,'var: ',nv
+                   do nv=1,nvars
+                      !if(mype==0) print *,__FILE__,__LINE__,'var: ',nv
 #ifdef VARINT
                       call PIO_setframe(File, vari(nv), recnum)
                       call pio_write_darray(File, vari(nv), iodesc_i4, ifld(:,nv)    , ierr, fillval= PIO_FILL_INT)
@@ -313,7 +313,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 #endif
                    enddo
                    if(unlimdimindof) then
-#ifdef VARREAL                
+#ifdef VARREAL
                       call PIO_freedecomp(File, iodesc_r4)
 #endif
 #ifdef VARDOUBLE
@@ -321,7 +321,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 #endif
 #ifdef VARINT
                       call PIO_freedecomp(File, iodesc_i4)
-#endif                
+#endif
                    endif
                 enddo
                 call pio_closefile(File)
@@ -344,7 +344,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 #ifdef VARDOUBLE
                    nvarmult = nvarmult+2
 #endif
-                   write(*,'(a15,a9,i10,i10,i10,f20.10)') &	
+                   write(*,'(a15,a9,i10,i10,i10,f20.10)') &
                    'RESULT: write ',rearr_name(rearr), piotypes(k), ntasks, nvars, &
                                      nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2))
 #ifdef BGQTRY
@@ -383,8 +383,8 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 
                 call MPI_Barrier(comm,ierr)
                 call t_stampf(wall(1), usr(1), sys(1))
-                
-                do frame=1,nframes                   
+
+                do frame=1,nframes
                    do nv=1,nvars
 #ifdef VARINT
                       call PIO_setframe(File, vari(nv), frame)
@@ -400,7 +400,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 #endif
                    enddo
                 enddo
-                
+
                 call pio_closefile(File)
                 call MPI_Barrier(comm,ierr)
                 call t_stampf(wall(2), usr(2), sys(2))
@@ -413,7 +413,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                          if(compmap(j)>0) then
 #ifdef VARINT
 #ifdef DEBUG
-                             write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & 
+                             write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') &
 			        ' Int    PE=',mype,'ifld=',ifld(j,nv),' ifld_in=',ifld_in(j,nv,frame),' compmap=',compmap(j)
 #endif
                             if(ifld(j,nv) /= ifld_in(j,nv,frame)) then
@@ -421,7 +421,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                                !   print *,__LINE__,'Int: ',mype,j,nv,ifld(j,nv),ifld_in(j,nv,frame),compmap(j)
                                !endif
                                write(*,*) '***ERROR:Mismatch!***'
-                               write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & 
+                               write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') &
 			         ' Int    PE=',mype,'ifld=',ifld(j,nv),' ifld_in=',ifld_in(j,nv,frame),' compmap=',compmap(j)
 
                                errorcnt = errorcnt+1
@@ -432,7 +432,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                             write(*,'(a11,i2,a9,f11.2,a9,f11.2,a9,i2)') &
 			        ' Real   PE=',mype,'rfld=',rfld(j,nv),' rfld_in=',rfld_in(j,nv,frame),' compmap=',compmap(j)
 #endif
-                            
+
                             if(rfld(j,nv) /= rfld_in(j,nv,frame) ) then
                                !if(errorcnt < 10) then
                                !   print *,__LINE__,'Real:', mype,j,nv,rfld(j,nv),rfld_in(j,nv,frame),compmap(j)
@@ -441,7 +441,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                                write(*,'(a11,i2,a9,f11.2,a9,f11.2,a9,i2)') &
 			         ' Real   PE=',mype,'rfld=',rfld(j,nv),' rfld_in=',rfld_in(j,nv,frame),' compmap=',compmap(j)
 
-                               errorcnt = errorcnt+1                           
+                               errorcnt = errorcnt+1
                             endif
 #endif
 #ifdef VARDOUBLE
@@ -466,7 +466,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                 enddo
                 j = errorcnt
                 call MPI_Reduce(j, errorcnt, 1, MPI_INTEGER, MPI_SUM, 0, comm, ierr)
-                
+
                 if(mype==0) then
                    if(errorcnt > 0) then
                       print *,'ERROR: INPUT/OUTPUT data mismatch ',errorcnt
@@ -484,11 +484,11 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
                    write(*,'(a15,a9,i10,i10,i10,f20.10)') &
                         'RESULT: read ',rearr_name(rearr), piotypes(k), ntasks, nvars, &
 			           nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2))
-#ifdef BGQTRY 
+#ifdef BGQTRY
   call print_memusage()
 #endif
                 end if
-#ifdef VARREAL                
+#ifdef VARREAL
                 call PIO_freedecomp(iosystem, iodesc_r4)
 #endif
 #ifdef VARDOUBLE
@@ -496,7 +496,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
 #endif
 #ifdef VARINT
                 call PIO_freedecomp(iosystem, iodesc_i4)
-#endif                
+#endif
                 call pio_finalize(iosystem, ierr)
              enddo
           enddo
@@ -510,6 +510,8 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, &
        deallocate(rfld_in)
     endif
 
+    call MPI_Comm_free(comm, ierr)
+
   end subroutine pioperformancetest
 
   subroutine init_ideal_dof(doftype, mype, npe, ndims, gdims, compmap, varsize)
@@ -531,7 +533,7 @@ subroutine init_ideal_dof(doftype, mype, npe, ndims, gdims, compmap, varsize)
     allocate(compmap(varsize))
     if(doftype .eq. 'ROUNDROBIN') then
        do i=1,varsize
-          compmap(i) = (i-1)*npe+mype+1 
+          compmap(i) = (i-1)*npe+mype+1
        enddo
     else if(doftype .eq. 'BLOCK') then
        do i=1,varsize
@@ -568,7 +570,7 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof)
 
    do i=1,ndims
 
-      write(dimname,'(a,i6.6)') 'dim',i  
+      write(dimname,'(a,i6.6)') 'dim',i
       iostat = PIO_def_dim(File, trim(dimname), int(gdims(i),pio_offset_kind), dimid(i))
    enddo
    iostat = PIO_def_dim(File, 'time', PIO_UNLIMITED, dimid(ndims+1))
@@ -609,15 +611,15 @@ subroutine CheckMPIreturn(line,errcode)
     implicit none
 #ifdef NO_MPIMOD
 #include 
-#endif  
+#endif
     integer, intent(in) :: errcode
     integer, intent(in) :: line
     character(len=MPI_MAX_ERROR_STRING) :: errorstring
-    
+
     integer :: errorlen
-    
+
     integer :: ierr
-    
+
     if (errcode .ne. MPI_SUCCESS) then
        call MPI_Error_String(errcode,errorstring,errorlen,ierr)
        write(*,*) errorstring(1:errorlen)
diff --git a/tests/performance/run_tests.sh b/tests/performance/run_tests.sh
new file mode 100755
index 000000000000..780ad7b791d1
--- /dev/null
+++ b/tests/performance/run_tests.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+# This is a test script for PIO for tests/performance directory.
+# Ed Hartnett 4/10/19
+
+# Stop execution of script if error is returned.
+set -e
+
+# Stop loop if ctrl-c is pressed.
+trap exit INT TERM
+
+printf 'running PIO performance test...\n'
+
+PIO_TESTS='pioperf '
+
+success1=true
+for TEST in $PIO_TESTS
+do
+    success1=false
+    echo "running ${TEST}"
+    mpiexec -n 4 ./${TEST} && success1=true
+    if test $success1 = false; then
+        break
+    fi
+done
+
+# Did we succeed?
+if test x$success1 = xtrue; then
+    exit 0
+fi
+exit 1
diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am
new file mode 100644
index 000000000000..848f2af471b5
--- /dev/null
+++ b/tests/unit/Makefile.am
@@ -0,0 +1,36 @@
+## This is the automake file for building the Fortran tests for the
+## PIO library.
+
+# Ed Hartnett 3/20/19
+
+# Parallel builds don't currently work in this directory.
+.NOTPARALLEL:
+
+# Put together AM_CPPFLAGS and AM_LDFLAGS.
+include $(top_srcdir)/set_flags.am
+
+# Build the test for make check.
+check_PROGRAMS = pio_unit_test_driver
+pio_unit_test_driver_SOURCES = driver.F90
+pio_unit_test_driver_LDADD = libglobal_vars.la libncdf_tests.la	\
+libbasic_tests.la ${top_builddir}/src/flib/libpiof.la ${top_builddir}/src/clib/libpio.la
+
+# Build these uninstalled convenience libraries.
+noinst_LTLIBRARIES = libglobal_vars.la libncdf_tests.la	\
+libbasic_tests.la
+
+# The convenience libraries depends on their source.
+libglobal_vars_la_SOURCES = global_vars.F90
+libncdf_tests_la_SOURCES = ncdf_tests.F90
+libbasic_tests_la_SOURCES = basic_tests.F90
+
+if RUN_TESTS
+# Tests will run from a bash script.
+TESTS = run_tests.sh
+endif # RUN_TESTS
+
+# Distribute the test script.
+EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl
+
+# Clean up files produced during testing.
+CLEANFILES = *.nc *.log *.mod
diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90
index daad01babca8..e6e2cb9474cc 100644
--- a/tests/unit/basic_tests.F90
+++ b/tests/unit/basic_tests.F90
@@ -6,7 +6,7 @@
 
 module basic_tests
 
-  use pio 
+  use pio
   use global_vars
 
   Implicit None
@@ -53,7 +53,7 @@ Subroutine test_create(test_id, err_msg)
         err_msg = "Could not create " // trim(filename)
         call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
-      
+
       call mpi_barrier(mpi_comm_world,ret_val)
       ! netcdf files need to end define mode before closing
       if (is_netcdf(iotype)) then
@@ -215,6 +215,16 @@ Subroutine test_open(test_id, err_msg)
           call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
+        ret_val = PIO_put_att(pio_file, pio_var, '_FillValue', -1)
+        if (ret_val .ne. PIO_NOERR) then
+          ! Error in PIO_def_var
+          err_msg = "Could not define _FillValue attribute"
+          call PIO_closefile(pio_file)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
+        end if
+
+
+
         ! Leave define mode
         ret_val = PIO_enddef(pio_file)
         if (ret_val .ne. PIO_NOERR) then
@@ -290,11 +300,11 @@ Subroutine test_open(test_id, err_msg)
            call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
         ret_val = PIO_set_log_level(0)
-        
+
         ! Close file
         call PIO_closefile(pio_file)
       end if
-        
+
       call mpi_barrier(MPI_COMM_WORLD,ret_val)
 
       ! Try to open standard binary file as netcdf (if iotype = netcdf)
@@ -304,6 +314,7 @@ Subroutine test_open(test_id, err_msg)
 
         ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, &
                                "not_netcdf.ieee", PIO_nowrite)
+
         if (ret_val.eq.PIO_NOERR) then
           ! Error in PIO_openfile
           err_msg = "Opened a non-netcdf file as netcdf"
diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90
index a955339ee085..266098e4bd16 100644
--- a/tests/unit/driver.F90
+++ b/tests/unit/driver.F90
@@ -15,7 +15,7 @@ Program pio_unit_test_driver
 
   ! local variables
   character(len=str_len) :: err_msg
-  integer :: fail_cnt, test_cnt, ios, test_id, ierr, test_val 
+  integer :: fail_cnt, test_cnt, ios, test_id, ierr, test_val
   logical :: ltest_netcdf, ltest_pnetcdf
   logical :: ltest_netcdf4p, ltest_netcdf4c
   namelist/piotest_nml/  ltest_netcdf,     &
@@ -26,7 +26,7 @@ Program pio_unit_test_driver
   integer ret_val
   character(len=pio_max_name) :: errmsg
   character(len=pio_max_name) :: expected
-  
+
   ! Set up MPI
   call MPI_Init(ierr)
   call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
@@ -63,7 +63,7 @@ Program pio_unit_test_driver
 
      ! Ignore namelist values if PIO not built with correct options
      ! (i.e. don't test pnetcdf if not built with pnetcdf)
-     ret_val = PIO_set_log_level(2)
+
 #ifndef _NETCDF
      if (ltest_netcdf) then
         write(*,"(A,1x,A)") "WARNING: can not test netcdf files because PIO", &
@@ -132,7 +132,7 @@ Program pio_unit_test_driver
      print *, err_msg
      call parse(err_msg, fail_cnt)
   end if
-     
+
   do test_id=1,ntest
      if (ltest(test_id)) then
         ! Make sure i is a valid test number
@@ -154,7 +154,7 @@ Program pio_unit_test_driver
                 write(*,"(A,I0)") "Error, not configured for test #", test_id
            call MPI_Abort(MPI_COMM_WORLD, 0, ierr)
         end select
-        
+
         ! test_create()
         if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..."
         call test_create(test_id, err_msg)
@@ -209,7 +209,7 @@ Program pio_unit_test_driver
   call MPI_Finalize(ierr)
   if(fail_cnt>0) then
      stop 1
-  else	 
+  else
      stop 0
   endif
 Contains
diff --git a/tests/unit/run_tests.sh b/tests/unit/run_tests.sh
new file mode 100755
index 000000000000..0f7c61ee1788
--- /dev/null
+++ b/tests/unit/run_tests.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+# This is a test script for PIO for tests/unit directory.
+# Ed Hartnett 3/25/19
+
+# Stop execution of script if error is returned.
+set -e
+
+# Stop loop if ctrl-c is pressed.
+trap exit INT TERM
+
+printf 'running PIO tests...\n'
+
+PIO_TESTS='pio_unit_test_driver'
+
+success1=true
+for TEST in $PIO_TESTS
+do
+    success1=false
+    echo "running ${TEST}"
+    mpiexec -n 4 ./${TEST} && success1=true
+    if test $success1 = false; then
+        break
+    fi
+done
+
+# Did we succeed?
+if test x$success1 = xtrue; then
+    exit 0
+fi
+exit 1