diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 00000000000..24fd6e268e1
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,58 @@
+language: c
+dist: trusty
+sudo: false
+
+branches:
+  only:
+  - master
+
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - pkg-config netcdf-bin libnetcdf-dev openmpi-bin libopenmpi-dev gfortran doxygen graphviz
+
+before_install:
+  - test -n $CC && unset CC
+  - test -n $FC && unset FC
+  - test -n $CPPFLAGS && unset CPPFLAGS
+  - test -n FCFLAGS && unset FCFLAGS
+
+before_script:
+  - export CC=mpicc
+  - export FC=mpif90
+  - export CPPFLAGS='-I/usr/include'
+  - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz 
+  - tar -xzvf pnetcdf-1.11.0.tar.gz
+  - ls -l
+  - pushd pnetcdf-1.11.0
+  - ./configure --prefix=/usr --enable-shared
+  - make
+  - sudo make install
+  - popd
+env:
+  global:
+    - CC=mpicc
+    - FC=mpif90
+    - CPPFLAGS='-I/usr/include'
+    - CFLAGS='-std=c99'
+    - LDFLAGS='-L/usr/lib'
+  
+script:
+  - ls -l /usr/include
+  - autoreconf -i
+  - export CFLAGS='-std=c99  -fsanitize=address -fno-omit-frame-pointer'
+  - export FFLAGS='-fsanitize=address -fno-omit-frame-pointer'
+  - export FCFLAGS='-fsanitize=address -fno-omit-frame-pointer'
+  - export DISTCHECK_CONFIGURE_FLAGS='--enable-fortran'
+  - ./configure --enable-fortran --enable-developer-docs
+  - make
+  - make -j distcheck
+  - rm -rf build
+  - mkdir build
+  - cd build
+  - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off ..
+  - make VERBOSE=1
+  - make tests VERBOSE=1
+  - make test VERBOSE=1
\ No newline at end of file
diff --git a/COPYRIGHT b/COPYRIGHT
new file mode 100644
index 00000000000..8652d01dda1
--- /dev/null
+++ b/COPYRIGHT
@@ -0,0 +1,16 @@
+/******************************************************************************
+ *
+ * 
+ *
+ * Copyright (C) 2009-2019
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation under the terms of the GNU General Public License is hereby 
+ * granted. No representations are made about the suitability of this software 
+ * for any purpose. It is provided "as is" without express or implied warranty.
+ * See the GNU General Public License for more details.
+ *
+ * Documents produced by Doxygen are derivative works derived from the
+ * input used in their production; they are not affected by this license.
+ *
+ */ 
\ No newline at end of file
diff --git a/Makefile.am b/Makefile.am
new file mode 100644
index 00000000000..b114d46e041
--- /dev/null
+++ b/Makefile.am
@@ -0,0 +1,16 @@
+# This is part of PIO. It creates the main Makefile.
+
+# Ed Hartnett
+
+# Look in the m4 directory for autotools stuff.
+ACLOCAL_AMFLAGS= -I m4
+
+# Does the user want to build fortran?
+if BUILD_DOCS
+DOC = doc
+endif
+
+SUBDIRS = src tests examples ${DOC} scripts
+
+EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT
+
diff --git a/configure.ac b/configure.ac
new file mode 100644
index 00000000000..d18bf60bb58
--- /dev/null
+++ b/configure.ac
@@ -0,0 +1,278 @@
+## This is the autoconf file for the PIO library.
+## Ed Hartnett 8/16/17
+
+
+# Initialize autoconf and automake.
+AC_INIT(pio, 2.4.4-development)
+AC_CONFIG_SRCDIR(src/clib/pio_darray.c)
+AM_INIT_AUTOMAKE([foreign serial-tests])
+
+# The PIO version, again.
+AC_DEFINE([PIO_VERSION_MAJOR], [2], [PIO major version])
+AC_DEFINE([PIO_VERSION_MINOR], [4], [PIO minor version])
+AC_DEFINE([PIO_VERSION_PATCH], [4], [PIO patch version])
+
+# Once more for the documentation.
+AC_SUBST([VERSION_MAJOR], [2])
+AC_SUBST([VERSION_MINOR], [4])
+AC_SUBST([VERSION_PATCH], [4])
+
+# The m4 directory holds macros for autoconf.
+AC_CONFIG_MACRO_DIR([m4])
+
+# Libtool initialisation.
+LT_INIT
+
+# Find and learn about the C compiler.
+AC_PROG_CC
+
+# Find and learn about the Fortran compiler.
+AC_PROG_FC
+
+# Always use malloc in autotools builds.
+AC_DEFINE([PIO_USE_MALLOC], [1], [use malloc for memory])
+
+AC_MSG_CHECKING([whether a PIO_BUFFER_SIZE was specified])
+AC_ARG_WITH([piobuffersize],
+              [AS_HELP_STRING([--with-piobuffersize=<integer>],
+                              [Specify buffer size for PIO.])],
+            [PIO_BUFFER_SIZE=$with_piobuffersize], [PIO_BUFFER_SIZE=134217728])
+AC_MSG_RESULT([$PIO_BUFFER_SIZE])
+AC_DEFINE_UNQUOTED([PIO_BUFFER_SIZE], [$PIO_BUFFER_SIZE], [buffer size for darray data.])
+
+# Does the user want to enable logging?
+AC_MSG_CHECKING([whether debug logging is enabled])
+AC_ARG_ENABLE([logging],
+              [AS_HELP_STRING([--enable-logging],
+                              [enable debug logging capability (will negatively impact performance). \
+			      This debugging feature is probably only of interest to PIO developers.])])
+test "x$enable_logging" = xyes || enable_logging=no
+AC_MSG_RESULT([$enable_logging])
+if test "x$enable_logging" = xyes; then
+   AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.])
+fi
+
+# Does the user want to use MPE library?
+AC_MSG_CHECKING([whether use of MPE library is enabled])
+AC_ARG_ENABLE([mpe],
+              [AS_HELP_STRING([--enable-mpe],
+                              [enable use of MPE library for timing and diagnostic info (may negatively impact performance).])])
+test "x$enable_mpe" = xyes || enable_mpe=no
+AC_MSG_RESULT([$enable_mpe])
+if test "x$enable_mpe" = xyes; then
+   AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [-lpthread -lm])
+   AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no])
+   if test "x$HAVE_LIBMPE" = xno -o "x$HAVE_MPE" = xno; then
+      AC_MSG_ERROR([MPE not found but --enable-mpe used.])
+   fi
+   AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.])
+fi
+
+# Does the user want to enable timing?
+AC_MSG_CHECKING([whether GPTL timing library is used])
+AC_ARG_ENABLE([timing],
+              [AS_HELP_STRING([--enable-timing],
+                              [enable use of the GPTL timing library.])])
+test "x$enable_timing" = xyes || enable_timing=no
+AC_MSG_RESULT([$enable_timing])
+if test "x$enable_timing" = xyes; then
+   AC_DEFINE([TIMING], 1, [If true, use GPTL timing library.])
+fi
+AM_CONDITIONAL(USE_GPTL, [test "x$enable_timing" = xyes])
+
+# Does the user want to disable papi?
+AC_MSG_CHECKING([whether PAPI should be enabled (if enable-timing is used)])
+AC_ARG_ENABLE([papi], [AS_HELP_STRING([--disable-papi],
+              [disable PAPI library use])])
+test "x$enable_papi" = xno || enable_papi=yes
+AC_MSG_RESULT($enable_papi)
+
+# Does the user want to disable test runs?
+AC_MSG_CHECKING([whether test runs should be enabled for make check])
+AC_ARG_ENABLE([test-runs], [AS_HELP_STRING([--disable-test-runs],
+              [disable running run_test.sh test scripts for make check. Tests will still be built.])])
+test "x$enable_test_runs" = xno || enable_test_runs=yes
+AC_MSG_RESULT($enable_test_runs)
+AM_CONDITIONAL(RUN_TESTS, [test "x$enable_test_runs" = xyes])
+
+# Does the user want to enable Fortran library?
+AC_MSG_CHECKING([whether Fortran library should be built])
+AC_ARG_ENABLE([fortran],
+              [AS_HELP_STRING([--enable-fortran],
+                              [build the PIO Fortran library.])])
+test "x$enable_fortran" = xyes || enable_fortran=no
+AC_MSG_RESULT([$enable_fortran])
+AM_CONDITIONAL(BUILD_FORTRAN, [test "x$enable_fortran" = xyes])
+
+# Does the user want to disable pnetcdf?
+AC_MSG_CHECKING([whether pnetcdf is to be used])
+AC_ARG_ENABLE([pnetcdf],
+              [AS_HELP_STRING([--disable-pnetcdf],
+                              [Disable pnetcdf use.])])
+test "x$enable_pnetcdf" = xno || enable_pnetcdf=yes
+AC_MSG_RESULT([$enable_pnetcdf])
+AM_CONDITIONAL(BUILD_PNETCDF, [test "x$enable_pnetcdf" = xyes])
+
+# Does the user want to build documentation?
+AC_MSG_CHECKING([whether documentation should be build (requires doxygen)])
+AC_ARG_ENABLE([docs],
+              [AS_HELP_STRING([--enable-docs],
+                              [enable building of documentation with doxygen.])])
+test "x$enable_docs" = xyes || enable_docs=no
+AC_MSG_RESULT([$enable_docs])
+
+# Does the user want to developer documentation?
+AC_MSG_CHECKING([whether PIO developer documentation should be build (only for PIO developers)])
+AC_ARG_ENABLE([developer-docs],
+              [AS_HELP_STRING([--enable-developer-docs],
+                              [enable building of PIO developer documentation with doxygen.])])
+test "x$enable_developer_docs" = xyes || enable_developer_docs=no
+AC_MSG_RESULT([$enable_developer_docs])
+
+# Developer docs enables docs.
+if test "x$enable_developer_docs" = xyes; then
+   enable_docs=yes
+fi
+AM_CONDITIONAL(BUILD_DOCS, [test "x$enable_docs" = xyes])
+
+# Is doxygen installed?
+AC_CHECK_PROGS([DOXYGEN], [doxygen])
+if test -z "$DOXYGEN" -a "x$enable_docs" = xyes; then
+   AC_MSG_ERROR([Doxygen not found but --enable-docs used.])
+fi
+
+# If building docs, process Doxyfile.in into Doxyfile.
+if test "x$enable_docs" = xyes; then
+   AC_SUBST([CMAKE_CURRENT_SOURCE_DIR], ["."])
+   AC_SUBST([CMAKE_BINARY_DIR], [".."])
+   if test "x$enable_fortran" = xno; then
+      AC_MSG_ERROR([--enable-fortran is required for documentation builds.])
+   fi
+   AC_SUBST([FORTRAN_SRC_FILES], ["../src/flib/piodarray.f90  ../src/flib/pio.F90  ../src/flib/pio_kinds.F90  ../src/flib/piolib_mod.f90  ../src/flib/pionfatt_mod_2.f90  ../src/flib/pio_nf.F90  ../src/flib/pionfget_mod_2.f90  ../src/flib/pionfput_mod.f90  ../src/flib/pio_support.F90  ../src/flib/pio_types.F90"])
+   if test "x$enable_developer_docs" = xyes; then
+      AC_SUBST([C_SRC_FILES], ["../src/clib"])
+   else
+      AC_SUBST([C_SRC_FILES], ["../src/clib/pio_nc.c ../src/clib/pio_nc4.c ../src/clib/pio_darray.c ../src/clib/pio_get_nc.c ../src/clib/pio_put_nc.c ../src/clib/pioc_support.c ../src/clib/pioc.c ../src/clib/pio_file.c ../src/clib/pio.h ../src/clib/pio_get_vard.c ../src/clib/pio_put_vard.c"])
+   fi
+   AC_CONFIG_FILES([doc/Doxyfile])
+fi
+
+# NetCDF (at least classic) is required for PIO to build.
+AC_DEFINE([_NETCDF], [1], [netCDF classic library available])
+
+# ????
+AC_DEFINE([CPRGNU], [1], [defined by CMake build])
+
+# We must have MPI to build PIO.
+AC_DEFINE([HAVE_MPI], [1], [defined by CMake build])
+
+# ???
+AC_DEFINE([INCLUDE_CMAKE_FCI], [1], [defined by CMake build])
+
+# All builds are on LINUX.
+AC_DEFINE([LINUX], [1], [defined by CMake build])
+
+# Define to solve intel compiler warning.
+AC_DEFINE([_GNU_SOURCE], [1], [solve strnlen declared implicitly warning on intel compiler])
+
+# Check for netCDF library.
+AC_CHECK_LIB([netcdf], [nc_create], [], [AC_MSG_ERROR([Can't find or link to the netcdf library.])])
+
+# Check for pnetcdf library.
+AC_CHECK_LIB([pnetcdf], [ncmpi_create], [], [])
+if test "x$ac_cv_lib_pnetcdf_ncmpi_create" = xno -a $enable_pnetcdf = yes; then
+   AC_MSG_ERROR([Pnetcdf not found. Set CPPFLAGS/LDFLAGS or use --disable-pnetcdf.])
+fi
+
+# If we have parallel-netcdf, then set these as well.
+if test x$ac_cv_lib_pnetcdf_ncmpi_create = xyes; then
+   AC_DEFINE([_PNETCDF], [1], [parallel-netcdf library available])
+   AC_DEFINE([USE_PNETCDF_VARN], [1], [defined by CMake build])
+   AC_DEFINE([USE_PNETCDF_VARN_ON_READ], [1], [defined by CMake build])
+fi
+
+# Do we have a parallel build of netCDF-4?
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"],
+[[#if !NC_HAS_PARALLEL
+# error
+#endif]
+])], [have_netcdf_par=yes], [have_netcdf_par=no])
+
+AC_MSG_CHECKING([whether netCDF provides parallel IO])
+AC_MSG_RESULT([${have_netcdf_par}])
+if test x$have_netcdf_par = xyes; then
+  AC_DEFINE([_NETCDF4],[1],[Does netCDF library provide netCDF-4 with parallel access])
+fi
+AM_CONDITIONAL(BUILD_NETCDF4, [test "x$have_netcdf_par" = xyes])
+
+# Not working for some reason, so I will just set it...
+AC_CHECK_TYPE([MPI_Offset], [], [], [#include <mpi.h>])
+if test "x${ac_cv_type_MPI_Offset}" = xyes; then
+   AC_CHECK_SIZEOF([MPI_Offset], [], [#include <mpi.h>])
+else
+   AC_MSG_ERROR([Unable to find type MPI_Offset in mpi.h])
+fi
+
+#AC_CHECK_SIZEOF([MPI_Offset], [], [[#include <mpi.h>]])
+#AC_DEFINE([SIZEOF_MPI_OFFSET], [8], [netCDF classic library available])
+
+# If we want the timing library, we must find it.
+if test "x$enable_timing" = xyes; then
+   AC_CHECK_HEADERS([gptl.h])
+   AC_CHECK_LIB([gptl], [GPTLinitialize], [],
+                        [AC_MSG_ERROR([Can't find or link to the GPTL library.])])
+   if test "x$enable_fortran" = xyes; then
+      AC_LANG_PUSH([Fortran])
+#      AC_CHECK_HEADERS([gptl.inc])
+      AC_CHECK_LIB([gptlf], [gptlstart], [],
+                            [AC_MSG_ERROR([Can't find or link to the GPTL Fortran library.])])
+      AC_LANG_POP([Fortran])
+   fi
+
+   # Check for papi library.
+   AC_CHECK_LIB([papi], [PAPI_library_init])
+   AC_MSG_CHECKING([whether system can support PAPI])
+   have_papi=no
+   if test $enable_papi = yes; then
+      if test "x$ac_cv_lib_papi_PAPI_library_init" = xyes; then
+         # If we have PAPI library, check /proc/sys/kernel/perf_event_paranoid
+         # to see if we have permissions.
+         if test -f /proc/sys/kernel/perf_event_paranoid; then
+            if test `cat /proc/sys/kernel/perf_event_paranoid` != 1; then
+               AC_MSG_ERROR([PAPI library found, but /proc/sys/kernel/perf_event_paranoid != 1
+                                                  try sudo sh -c 'echo 1 >/proc/sys/kernel/perf_event_paranoid'])
+            fi
+         fi
+         AC_DEFINE([HAVE_PAPI], [1], [PAPI library is present and usable])
+         have_papi=yes
+      fi
+   fi
+   AC_MSG_RESULT($have_papi)
+fi
+AM_CONDITIONAL([HAVE_PAPI], [test "x$have_papi" = xyes])
+
+AC_CONFIG_FILES([tests/general/pio_tutil.F90:tests/general/util/pio_tutil.F90])
+
+AC_CONFIG_LINKS([tests/unit/input.nl:tests/unit/input.nl])
+
+# Create the config.h file.
+AC_CONFIG_HEADERS([config.h])
+
+# Create the makefiles.
+AC_OUTPUT(Makefile
+          src/Makefile
+          src/clib/Makefile
+          src/flib/Makefile
+          src/gptl/Makefile
+          tests/Makefile
+          tests/cunit/Makefile
+          tests/unit/Makefile
+          tests/general/Makefile
+          tests/general/util/Makefile
+          tests/performance/Makefile
+          doc/Makefile
+          doc/source/Makefile
+          doc/images/Makefile
+          examples/Makefile
+          examples/c/Makefile
+          scripts/Makefile)
diff --git a/doc/Makefile.am b/doc/Makefile.am
new file mode 100644
index 00000000000..83ea41382d9
--- /dev/null
+++ b/doc/Makefile.am
@@ -0,0 +1,20 @@
+# This is part of PIO. It creates the doc Makefile.
+
+# Ed Hartnett 4/1/19
+
+# Run doxygen, then confirm warning log file is empty.
+all:
+	doxygen Doxyfile
+	cat doxywarn.log
+	[ ! -s doxywarn.log ]
+
+check: all
+
+# Include these subdirs to include the documention files in the
+# distribution.
+SUBDIRS = source images
+
+CLEANFILES = *.log
+
+EXTRA_DIST = CMakeLists.txt customdoxygen.css Doxyfile.in	\
+DoxygenLayout.xml doxygen.sty
diff --git a/doc/source/Decomp.txt b/doc/source/Decomp.txt
new file mode 100644
index 00000000000..f47c8bf18e2
--- /dev/null
+++ b/doc/source/Decomp.txt
@@ -0,0 +1,86 @@
+/** @page decomp Describing decompositions
+
+One of the biggest challenges to working with PIO is setting up the
+decomposition of the data (Fortran users see @ref PIO_initdecomp, C
+users @ref PIO_initdecomp_c). The user must properly describe how the
+data within each MPI tasks memory should be placed or retrieved from
+disk.
+
+@section The Compmap
+
+When initializing a new decomposition, each task calling
+PIOc_init_decomp() or PIO_initdecomp().
+
+@image html PIO_Decomposition.png
+
+@section Rearrangers
+
+PIO provides two methods to rearrange data from compute tasks to
+IO tasks.
+
+@subsection BOXREARR Box rearrangement
+
+In this method data is rearranged from compute to IO tasks such that
+the arrangement of data on the IO tasks optimizes the call from the IO
+tasks to the underlying (NetCDF) IO library. In this case each compute
+task will transfer data to one or more IO tasks.
+
+@subsection SUBSETREARR Subset rearrangement
+
+In this method each IO task is associated with a unique subset of
+compute tasks so that each compute task will transfer data to one and
+only one IO task. Since this technique does not guarantee that data on
+the IO node represents a contiguous block of data on the file it may
+require multiple calls to the underlying (NetCDF) IO library.
+
+As an example suppose we have a global two dimensional grid of size
+4x5 decomposed over 5 tasks. We represent the two dimensional grid in
+terms of offset from the initial element ie
+
+<pre>
+     0  1  2  3 
+     4  5  6  7 
+     8  9 10 11
+    12 13 14 15
+    16 17 18 19 
+</pre>
+
+Now suppose this data is distributed over the compute tasks as follows:
+
+<pre>
+0: {   0  4 8 12  } 
+1: {  16 1 5 9  } 
+2: {  13 17 2 6  } 
+3: {  10 14 18 3  } 
+4: {   7 11 15 19  } 
+</pre>
+
+If we have 2 io tasks the Box rearranger would give:
+
+<pre>
+0: { 0  1  2  3  4  5  6  7  8  9  }
+1: { 10 11 12 13 14 15 16 17 18 19 }
+</pre>
+
+While the subset rearranger would give:
+
+<pre>
+0: { 0  1  4  5  8  9  12 16 }
+1: { 2  3  6  7  10 11 13 14 15 17 18 19 }
+</pre>
+
+Note that while the box rearranger gives a data layout which is well
+balanced and well suited for the underlying io library, it had to
+communicate with every compute task to do so. On the other hand the
+subset rearranger communicated with only a portion of the compute
+tasks but requires more work on the part of the underlying io library
+to complete the operation.
+
+Also note if every task is an IO task then the box rearranger will
+need to do an alltoall communication, while the subset rearranger does
+none. In fact using the subset rearranger with every compute task an
+IO task provides a measure of what you might expect the performance of
+the underlying IO library to be if it were used without PIO.
+
+*/
+
diff --git a/doc/source/Introduction.txt b/doc/source/Introduction.txt
new file mode 100644
index 00000000000..888c44a84f8
--- /dev/null
+++ b/doc/source/Introduction.txt
@@ -0,0 +1,89 @@
+@tableofcontents
+
+/** @page intro Introduction
+
+Performing I/O is straightforward when a small number of processors
+are being used.
+
+@image html I_O_on_Few.png "I/O on One or a Few Processors"
+
+Parallel I/O does not scale to thousands of processors, because the
+parallel disk systems do not support the bandwitdh to allow thousands
+of processors to access the disk at the same time. As a result, most
+of the processors will have to wait to do I/O.
+
+An obvious solution is to designate a small number of processors to do
+I/O, and use the rest for computation.
+
+@image html I_O_on_Many_Intracomm.png "I/O on Many Processors Intracomm"
+
+PIO provides a netCDF-like API which provides this service. User code
+is written as if parallel I/O is being used from every processor, but,
+under the hood, PIO uses the I/O processors to do all data access.
+
+With Intracomm Mode, the I/O processors are a subset of the
+computational processors, and only one computational unit is
+supported. In Async Mode, the I/O processors are dedicated to I/O, and
+do not perform computation. Also, more than one computational unit may
+be designated.
+
+@image html I_O_on_Many_Async.png "I/O on Many Processors Async"
+
+The user initializes the PIO IO System, designating some processors
+for I/O, others for computation.
+
+PIO decompositions and distributed arrays allow the code to be written
+in terms of the local, distributed sub-array (see @ref decomp). PIO
+handles the stitching of all data into the correct global space in a
+netCDF variable.
+
+PIO also allows for the creation of multiple computational units. Each
+computational unit consists of many processors. I/O for all
+computational units is accomplished through one set of dedicated I/O
+processors (see @ref iosystem).
+
+PIO uses <a
+href=http://www.unidata.ucar.edu/software/netcdf/docs/html_guide/index.html#user_guide>
+netcdf </a> and <a
+href=http://trac.mcs.anl.gov/projects/parallel-netcdf> pnetcdf</a> to
+read and write the netCDF files (see @ref install).
+
+## Basic description of how to optimize IO in a parallel environment:
+
+PIO calls are collective.  A MPI communicator is set in a call to @ref
+PIO_init and all tasks associated with that communicator must
+participate in all subsequent calls to PIO.  An application can make
+multiple calls to @ref PIO_init in order to support multiple MPI
+communicators.
+
+Begin by getting and unpacking the most recent release of PIO from
+[gitHub](https://github.com/PARALLELIO/ParallelIO/releases) and
+installing on your system as per the instructions in the
+[Installation](@ref install) document. Take a look at examples of PIO
+usage in both complex and simple test programs in the [Examples](@ref
+examp) document. Finally, read through the [FAQ](@ref faq) to see if
+any remaining questions can be answered.
+
+### Using PIO has three basic steps. ###
+
+1. Your program should call the @ref PIO_init function, and provide
+the MPI communicator (and the rank within that communicator) of the
+calling task. This call initializes an IO system type structure that
+will be used in subsequent file and decomposition functions.
+
+2. You can open a file for reading or writing with a call to @ref
+PIO_createfile or @ref PIO_openfile. In this call you will specify the
+file type: pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4c
+or pio_iotype_netcdf4p; along with the file name and optionally the
+netcdf mode.
+
+3. Finally, you can read or write decomposed data to the output
+file. You must describe the mapping between the organization of data
+in the file and that same data in the application space.  This is done
+in a call to @ref PIO_initdecomp. In the simplest call to this
+function, a one dimensional integer array is passed from each task,
+the values in the array represent the offset from the beginning of the
+array on file.
+
+
+*/
diff --git a/doc/source/Makefile.am b/doc/source/Makefile.am
new file mode 100644
index 00000000000..97f8e16dc3f
--- /dev/null
+++ b/doc/source/Makefile.am
@@ -0,0 +1,8 @@
+# This is part of PIO. It creates the doc/source Makefile.
+
+# Ed Hartnett 5/20/19
+
+EXTRA_DIST = api.txt CAMexample.txt Decomp.txt faq.txt Installing.txt	\
+Testing.txt base.txt c_api.txt contributing_code.txt Error.txt		\
+Examples.txt Introduction.txt mach_walkthrough.txt			\
+testpio_example.txt users_guide.txt iosystem.txt
diff --git a/doc/source/c_api.txt b/doc/source/c_api.txt
new file mode 100644
index 00000000000..e9d3ed3f62e
--- /dev/null
+++ b/doc/source/c_api.txt
@@ -0,0 +1,50 @@
+ /*!  \page c_api PIO C Interface
+  This is a list of all user interface routines:
+
+  \section api_system_c PIO Startup and Shutdown
+     - \ref PIO_init_c
+     - \ref PIO_finalize_c
+  \section api_fileops_c PIO File Operations
+     - \ref PIO_open_file_c
+     - \ref PIO_create_file_c
+     - \ref PIO_sync_file_c
+     - \ref PIO_close_file_c
+  \section api_decomp_c PIO Decompositions
+    - \ref PIO_initdecomp_c
+    - \ref PIO_freedecomp_c
+  \section readwrite_c Reading and Writing Distributed Arrays
+    - \ref PIO_read_darray_c
+    - \ref PIO_write_darray_c
+    - \ref PIO_setframe_c
+  \section utility_c Utility 
+    - \ref PIO_set_hint_c
+    - \ref PIO_error_method_c
+    - \ref PIO_get_local_array_size_c
+    - \ref PIO_getnumiotasks_c
+    - \ref PIO_set_blocksize_c
+  \section netcdf_c NetCDF-Like Functions
+     Also see: http://www.unidata.ucar.edu/software/netcdf/docs/
+     \subsection utilnc_c File Operations
+       - \ref PIO_enddef_c
+       - \ref PIO_redef_c
+     \subsection write_metadata_c Writing Metadata   
+       - \ref PIO_def_dim_c
+       - \ref PIO_def_var_c
+       - \ref PIO_put_att_c
+     \subsection putget_c Reading/Writing Data
+       - \ref PIO_get_vara_c
+       - \ref PIO_get_var_c
+       - \ref PIO_get_var1_c
+       - \ref PIO_get_vars_c
+       - \ref PIO_put_vara_c
+       - \ref PIO_put_var_c
+       - \ref PIO_put_var1_c
+       - \ref PIO_put_vars_c
+     \subsection inqnc_c Learn about Files and Metadata
+       - \ref PIO_inq_c
+       - \ref PIO_get_att_c
+       - \ref PIO_inq_att_c
+       - \ref PIO_inq_var_c
+       - \ref PIO_inq_dim_c
+
+*/
diff --git a/doc/source/contributing_code.txt b/doc/source/contributing_code.txt
new file mode 100644
index 00000000000..cf7df469c19
--- /dev/null
+++ b/doc/source/contributing_code.txt
@@ -0,0 +1,137 @@
+/*! @page contributing_code Guide for Contributors
+
+# Introduction #
+
+This document describes the code style to be used when contributing to
+the PIO library.
+
+Consistency of style allows bugs to be spotted more quickly and
+contributes to code quality.
+
+# C Code #
+
+## Warnings ##
+
+<p>The C library compiles under GNU gcc without warnings. No code will
+be merged with the C library which causes warnings during compile.
+
+## Backward Compatibility ##
+
+<p>We cannot remove or change a function in the public C API. We can
+add to it, carefully.
+
+## Brevity ##
+
+<p>Consider: C is a terse language.
+
+<ul>
+<li>Use the fact that NULL == 0, and 0 == false, to shorten code.
+<li>Assign and check return values in the same if statement.
+</ul>
+
+## Indentation and Spacing##
+
+<ul>
+<li>Use spaces not tabs.
+<li>4 spaces is the unit of intendation.
+<li>Indentation as defined by the "linux" style in emacs (see below).
+<li>Use spaces around most operators (= + - * /) not pointer or prefix/postfile (* ++ --)
+<li>Spaces after most keywords (if, for, while, etc.)
+<li>No spaces after function name.
+</ul>
+
+## Braces ##
+
+<p>Put braces on their own line, avoiding their use if possible.
+
+## Documentation ##
+
+<ul>
+<li>Every function must be documented using doxygen.
+<li>Keep internal functions in separate code files, so that Doxygen
+can easily build user and development builds of the documentation.
+<li>Use the doxygen \@ingroup to put public functions in the correct
+group.
+<li>Code must be reasonably documented as to intention.
+<li>Documentation quality and quantity are part of code review
+process.
+<li>Document in complete sentences.
+<li>Use C (not C++) comment delimiters.
+<li>Use the author tag to indicate which programmers have worked on
+each function. When adding or changing a function in a non-trivial
+way, programmers should add their name to the end of the list of
+authors for that function.
+</ul>
+
+## Emacs ##
+
+<p>Put this in your .emacs file:
+
+<pre>
+(setq c-default-style "linux"
+          c-basic-offset 4)
+</pre>
+
+<p>The tab key (used anywhere on the line) will indent a line
+correctly. M-x indent-region will indent a selected region of code.
+
+<p>M-x untabify will convert all the tabs in a file to spaces.
+
+## Code Review ##
+
+<ul>
+
+<li>All code is subject to review.
+
+<li>Pull requests will be focused on one issue.
+
+<li>Pull requests may not be submitted until all tests pass.
+
+<li>All non-trivial pull requests are associated with a github
+issue. The issue is where discussion of requirements and
+implementation details can be worked out.
+
+<li>Pull requests will be left up on github for about a day. Request
+more time if you need it and are actively reviewing the code. (Note
+that pull request can also be reviewed after they are merged, if you
+miss one).
+
+<li>Jim will identify key stakeholders in changed code and ensure they
+accept code changes.
+
+<li>Reviewers are open-minded and ready to accept improvements to the
+library.
+
+<li>Reviewers will make comments on the pull request. All comments
+must be resolved.
+
+<li>If chages are dictated, they happen on the branch, so code
+reviewers can see the updated code.
+
+<li>The pull request is only merged when all programmers agree that
+all issues have been resolved.
+
+</ul>
+
+## Merge Proceedure
+
+<ul>
+
+<li>Programmers begin work on a feature or fix by branching from
+master.
+
+<li>When a branch is ready, it is submitted to code review via pull
+request. Travis CI testing is automatically run on the PR.
+
+<li>When code review is complete, and the changes are approved, the PR
+is merged into the master branch.
+
+<li>The master branch is tested periodically by CDash (every ~6
+hours). Any test failures and the merge to master may be rolled back.
+
+<li>The branch is then deleted by whomever merged it to master.
+
+</ul>
+
+
+*/
diff --git a/doc/source/iosystem.txt b/doc/source/iosystem.txt
new file mode 100644
index 00000000000..5acce7fc928
--- /dev/null
+++ b/doc/source/iosystem.txt
@@ -0,0 +1,24 @@
+/** @page iosystem Initializing the IO System
+
+Using PIO begins with initializing the IO System. This sets up the MPI
+communicators with the computational and I/O processors.
+
+When the IO System is created, an IOSystem ID is returned and must be
+used in future PIO calls. The IOSystem ID is returned by C functions
+PIOc_Init_Intracomm() and PIOc_init_async(). Fortran users see @ref
+PIO_init.
+
+When the user program is complete, the IOSystem should be released by
+calling C function PIOc_finalize() or Fortran function piolib_mod::finalize()
+for each open IOSystem.
+
+@section intracomm_mode Intracomm Mode
+
+@image html PIO_Intracomm1.png "PIO Intracomm Mode"
+
+@section async_mode Async Mode
+
+@image html PIO_Async.png "PIO Async Mode"
+
+*/
+
diff --git a/doc/source/users_guide.txt b/doc/source/users_guide.txt
new file mode 100644
index 00000000000..f97123ac50a
--- /dev/null
+++ b/doc/source/users_guide.txt
@@ -0,0 +1,20 @@
+/*! 
+
+@page users_guide PIO User's Guide
+
+This user's guide provides information about the PIO library and
+examples on how it can be used. Please watch the PIO GitHub site
+[https://github.com/NCAR/ParallelIO] for announcements and new
+releases.
+
+ - @ref intro
+ - @ref iosystem
+ - @ref decomp
+ - @ref error
+ - @ref examp
+ - @ref faq
+ - @ref api
+ - @ref c_api
+ - @ref contributing_code
+
+*/
diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c
new file mode 100644
index 00000000000..ce50003a29a
--- /dev/null
+++ b/examples/c/darray_no_async.c
@@ -0,0 +1,382 @@
+/*
+ * @file
+ * @brief A simple C example for the ParallelIO Library.
+ *
+ * This example creates a netCDF output file with three dimensions
+ * (one unlimited) and one variable. It first writes, then reads the
+ * sample file using distributed arrays.
+ *
+ * This example can be run in parallel for 16 processors.
+ */
+
+#include "config.h"
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <mpi.h>
+#include <pio.h>
+#include <pio_internal.h>
+#ifdef TIMING
+#include <gptl.h>
+#endif
+
+/* The number of possible output netCDF output flavors available to
+ * the ParallelIO library. */
+#define NUM_NETCDF_FLAVORS 4
+
+/* The number of dimensions in the example data. */
+#define NDIM3 3
+
+/* The number of timesteps of data. */
+#define NUM_TIMESTEPS 2
+
+/* The length of our sample data in X dimension.*/
+#define DIM_LEN_X 8
+
+/* The length of our sample data in Y dimension.*/
+#define DIM_LEN_Y 8
+
+/* The name of the variable in the netCDF output file. */
+#define VAR_NAME "foo"
+
+/* Return code when netCDF output file does not match
+ * expectations. */
+#define ERR_BAD 1001
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Number of tasks this example runs on. */
+#define TARGET_NTASKS 16
+
+/* Logging level. */
+#define LOG_LEVEL -1
+
+/* Lengths of dimensions. */
+int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+
+/* Names of dimensions. */
+char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
+
+/* These are used when writing the decomposition file. */
+#define DECOMP_FILENAME "darray_no_async_decomp.nc"
+#define DECOMP_TITLE "Example Decomposition from darray_no_async.c"
+#define DECOMP_HISTORY "This file is created by the program darray_no_async in the PIO C library"
+
+/* Handle MPI errors. This should only be used with MPI library
+ * function calls. */
+#define MPIERR(e) do {                                                  \
+	MPI_Error_string(e, err_buffer, &resultlen);			\
+	printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+	MPI_Finalize();							\
+	return 2;							\
+    } while (0)
+
+/* Handle non-MPI errors by finalizing the MPI library and exiting
+ * with an exit code. */
+#define ERR(e) do {				\
+	MPI_Finalize();				\
+	return e;				\
+    } while (0)
+
+/* Global err buffer for MPI. When there is an MPI error, this buffer
+ * is used to store the error message that is associated with the MPI
+ * error. */
+char err_buffer[MPI_MAX_ERROR_STRING];
+
+/* This is the length of the most recent MPI error message, stored
+ * int the global error string. */
+int resultlen;
+
+/* @brief Check the output file.
+ *
+ *  Use netCDF to check that the output is as expected.
+ *
+ * @param ntasks The number of processors running the example.
+ * @param filename The name of the example file to check.
+ *
+ * @return 0 if example file is correct, non-zero otherwise. */
+int check_file(int iosysid, int ntasks, char *filename, int iotype,
+               int elements_per_pe, int my_rank, int ioid)
+{
+
+    int ncid;         /* File ID from netCDF. */
+    int ndims;        /* Number of dimensions. */
+    int nvars;        /* Number of variables. */
+    int ngatts;       /* Number of global attributes. */
+    int unlimdimid;   /* ID of unlimited dimension. */
+    int natts;        /* Number of variable attributes. */
+    nc_type xtype;    /* NetCDF data type of this variable. */
+    int ret;          /* Return code for function calls. */
+    int dimids[NDIM3]; /* Dimension ids for this variable. */
+    char var_name[PIO_MAX_NAME];   /* Name of the variable. */
+    /* size_t start[NDIM3];           /\* Zero-based index to start read. *\/ */
+    /* size_t count[NDIM3];           /\* Number of elements to read. *\/ */
+    /* int buffer[DIM_LEN_X];          /\* Buffer to read in data. *\/ */
+    /* int expected[DIM_LEN_X];        /\* Data values we expect to find. *\/ */
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0)))
+        return ret;
+    printf("opened file %s ncid = %d\n", filename, ncid);
+
+    /* Check the metadata. */
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+
+    /* Check the dimensions. */
+    if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0)
+        return ERR_BAD;
+    for (int d = 0; d < NDIM3; d++)
+    {
+        char my_dim_name[PIO_MAX_NAME];
+        PIO_Offset dimlen;
+
+        if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen)))
+            return ret;
+        if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d]))
+            return ERR_BAD;
+    }
+
+    /* Check the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts)))
+        return ret;
+    if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 ||
+        dimids[2] != 2 || natts != 0)
+        return ERR_BAD;
+
+    /* Allocate storage for sample data. */
+    int buffer[elements_per_pe];
+    int buffer_in[elements_per_pe];
+
+    /* Check each timestep. */
+    for (int t = 0; t < NUM_TIMESTEPS; t++)
+    {
+        int varid = 0; /* There's only one var in sample file. */
+
+        /* This is the data we expect for this timestep. */
+        for (int i = 0; i < elements_per_pe; i++)
+            buffer[i] = 100 * t + START_DATA_VAL + my_rank;
+
+        /* Read one record. */
+        if ((ret = PIOc_setframe(ncid, varid, t)))
+            ERR(ret);
+        if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in)))
+            return ret;
+
+        /* Check the results. */
+        for (int i = 0; i < elements_per_pe; i++)
+            if (buffer_in[i] != buffer[i])
+                return ERR_BAD;
+    }
+
+    /* Close the file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+
+    /* Everything looks good! */
+    return 0;
+}
+
+/* Write, then read, a simple example with darrays.
+
+   The sample file created by this program is a small netCDF file. It
+   has the following contents (as shown by ncdump):
+
+   <pre>
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 8 ;
+	y = 8 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42, 43, 43, 43, 43,
+  44, 44, 44, 44, 45, 45, 45, 45,
+  46, 46, 46, 46, 47, 47, 47, 47,
+  48, 48, 48, 48, 49, 49, 49, 49,
+  50, 50, 50, 50, 51, 51, 51, 51,
+  52, 52, 52, 52, 53, 53, 53, 53,
+  54, 54, 54, 54, 55, 55, 55, 55,
+  56, 56, 56, 56, 57, 57, 57, 57,
+  142, 142, 142, 142, 143, 143, 143, 143,
+  144, 144, 144, 144, 145, 145, 145, 145,
+  146, 146, 146, 146, 147, 147, 147, 147,
+  148, 148, 148, 148, 149, 149, 149, 149,
+  150, 150, 150, 150, 151, 151, 151, 151,
+  152, 152, 152, 152, 153, 153, 153, 153,
+  154, 154, 154, 154, 155, 155, 155, 155,
+  156, 156, 156, 156, 157, 157, 157, 157 ;
+}
+   </pre>
+
+*/
+int main(int argc, char* argv[])
+{
+    int my_rank;  /* Zero-based rank of processor. */
+    int ntasks;   /* Number of processors involved in current execution. */
+    int ioproc_stride = 1;	    /* Stride in the mpi rank between io tasks. */
+    int ioproc_start = 0; 	    /* Rank of first task to be used for I/O. */
+    PIO_Offset elements_per_pe; /* Array elements per processing unit. */
+    int iosysid;  /* The ID for the parallel I/O system. */
+    int ncid;     /* The ncid of the netCDF file. */
+    int dimid[NDIM3];    /* The dimension ID. */
+    int varid;    /* The ID of the netCDF varable. */
+    int ioid;     /* The I/O description ID. */
+    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+    int num_flavors = 0;            /* Number of iotypes available in this build. */
+    int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */
+    int ret;                        /* Return value. */
+
+#ifdef TIMING
+    /* Initialize the GPTL timing library. */
+    if ((ret = GPTLinitialize ()))
+        return ret;
+#endif
+
+    /* Initialize MPI. */
+    if ((ret = MPI_Init(&argc, &argv)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+        MPIERR(ret);
+
+    /* Learn my rank and the total number of processors. */
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+        MPIERR(ret);
+
+    /* Check that a valid number of processors was specified. */
+    if (ntasks != TARGET_NTASKS)
+        fprintf(stderr, "Number of processors must be 16!\n");
+    printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
+           my_rank, ntasks);
+
+    /* Turn on logging. */
+    if ((ret = PIOc_set_log_level(LOG_LEVEL)))
+        return ret;
+
+    /* /\* Change error handling so we can test inval parameters. *\/ */
+    /* if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) */
+    /*     return ret; */
+
+    /* Initialize the PIO IO system. This specifies how many and
+     * which processors are involved in I/O. */
+    if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 4, ioproc_stride,
+                                   ioproc_start, PIO_REARR_BOX, &iosysid)))
+        ERR(ret);
+
+    /* Describe the decomposition. */
+    elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS;
+
+    /* Allocate and initialize array of decomposition mapping. */
+    PIO_Offset compdof[elements_per_pe];
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this example. Since this
+     * is a variable with an unlimited dimension, we want to
+     * create a 2-D composition which represents one record. */
+    printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank,
+           elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
+                                compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL)))
+        ERR(ret);
+
+    /* Write the decomposition file. */
+    if ((ret = PIOc_write_nc_decomp(iosysid, DECOMP_FILENAME, NC_CLOBBER,
+                                    ioid, DECOMP_TITLE, DECOMP_HISTORY, 0)))
+        ERR(ret);
+
+    /* The number of favors may change with the build parameters. */
+#ifdef _PNETCDF
+    format[num_flavors++] = PIO_IOTYPE_PNETCDF;
+#endif
+    format[num_flavors++] = PIO_IOTYPE_NETCDF;
+#ifdef _NETCDF4
+    format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
+    format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
+#endif
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create a filename. */
+        sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n",
+               my_rank, filename, format[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimension and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM3; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
+                ERR(ret);
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid)))
+            ERR(ret);
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Allocate storage for sample data. */
+        int buffer[elements_per_pe];
+
+        /* Write each timestep. */
+        for (int t = 0; t < NUM_TIMESTEPS; t++)
+        {
+            /* Create some data for this timestep. */
+            for (int i = 0; i < elements_per_pe; i++)
+                buffer[i] = 100 * t + START_DATA_VAL + my_rank;
+
+            /* Write data to the file. */
+            printf("rank: %d Writing sample data...\n", my_rank);
+            if ((ret = PIOc_setframe(ncid, varid, t)))
+                ERR(ret);
+            if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
+                ERR(ret);
+        }
+
+        /* THis will cause all data to be written to disk. */
+        if ((ret = PIOc_sync(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check the output file. */
+        /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */
+        /*                       my_rank, ioid))) */
+        /*     ERR(ret); */
+    }
+
+    /* Free the PIO decomposition. */
+    printf("rank: %d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    /* Finalize the IO system. */
+    printf("rank: %d Freeing PIO resources...\n", my_rank);
+    if ((ret = PIOc_free_iosystem(iosysid)))
+        ERR(ret);
+
+    /* Finalize the MPI library. */
+    MPI_Finalize();
+
+#ifdef TIMING
+    /* Finalize the GPTL timing library. */
+    if ((ret = GPTLfinalize ()))
+        return ret;
+#endif
+
+    printf("rank: %d SUCCESS!\n", my_rank);
+    return 0;
+}
diff --git a/scripts/Makefile.am b/scripts/Makefile.am
new file mode 100644
index 00000000000..9e65f382902
--- /dev/null
+++ b/scripts/Makefile.am
@@ -0,0 +1,6 @@
+# This is part of PIO. It handles the scripts directory which has some
+# useful scripts.
+
+# Ed Hartnett 5/17/19
+
+EXTRA_DIST = prune_decomps.pl genf90.pl
diff --git a/scripts/genf90.pl b/scripts/genf90.pl
new file mode 100755
index 00000000000..ec7d0dba28b
--- /dev/null
+++ b/scripts/genf90.pl
@@ -0,0 +1,387 @@
+#!/usr/bin/env perl
+use strict;
+my $outfile;
+# Beginning with F90, Fortran has strict typing of variables based on "TKR"
+# (type, kind, and rank). In many cases we want to write subroutines that
+# provide the same functionality for different variable types and ranks. In
+# order to do this without cut-and-paste duplication of code, we create a
+# template file with the extension ".F90.in", which can be parsed by this script
+# to generate F90 code for all of the desired specific types.
+#
+# Keywords are delimited by curly brackets: {}
+# 
+# {TYPE} and {DIMS} are used to generate the specific subroutine names from the 
+#            generic template
+# {TYPE} : Variable type name; implemented types are character, 4 or 8 byte real,
+#          and 4 or 8 byte integer.
+#                allowed values: text, real, double, int, long, logical
+#                default values:  text, real, double, int
+# {VTYPE} : Used to generate variable declarations to match the specific type.
+#                if {TYPE}=double then {VTYPE} is "real(r8)"
+# {ITYPE}, {ITYPENAME} : Used to generate CPP statements for the specific type.
+# {MPITYPE} : Used to generate MPI types corresponding to the specific type.
+# 
+# {DIMS} : Rank of arrays, "0" for scalar.
+#                allowed values: 0-7
+#                default values : 0-5
+# {DIMSTR} : Generates the parenthesis and colons used for a variable
+#            declaration of {DIMS} dimensions.
+#                if {DIMS}=3 then {DIMSTR} is (:,:,:)
+# {REPEAT} : Repeats an expression for each number from 1 to {DIMS}, with each
+#            iteration separated by commas.
+#                {REPEAT: foo(#, bar)}
+#                expands to this:
+#                foo(1, bar), foo(2, bar), foo(3, bar), ...
+
+# defaults 
+my @types = qw(text real double int);
+my $vtype = {'text' => 'character(len=*)', 
+	     'real' => 'real(r4)', 
+	     'double' => 'real(r8)',
+	     'int'    => 'integer(i4)',
+	     'long'   => 'integer(i8)',
+             'logical' => 'logical' };
+my $itype = {'text' => 100, 
+	     'real' => 101, 
+	     'double' => 102,
+	     'int'    => 103,
+	     'long'   => 104,
+             'logical' => 105};
+my $itypename = {'text' => 'TYPETEXT', 
+	     'real' =>  'TYPEREAL', 
+	     'double' => 'TYPEDOUBLE',
+	     'int'    => 'TYPEINT',
+	     'long'   =>  'TYPELONG',
+             'logical' => 'TYPELOGICAL'};
+my $mpitype = {'text' => 'MPI_CHARACTER',
+	       'real' => 'MPI_REAL4',
+	       'double' => 'MPI_REAL8',
+	       'int' => 'MPI_INTEGER'};
+# Netcdf C datatypes
+my $nctype = {'text' => 'text',
+	      'real' => 'float',
+	      'double' => 'double',
+	      'int' => 'int'};
+# C interoperability types
+my $ctype = {'text' => 'character(C_CHAR)',
+	     'real' => 'real(C_FLOAT)',
+	     'double' => 'real(C_DOUBLE)',
+	     'int' => 'integer(C_INT)'};
+
+
+
+my @dims =(0..5);
+
+my $write_dtypes = "no";
+# begin
+
+foreach(@ARGV){
+    my $infile = $_;
+    usage() unless($infile =~ /(.*.F90).in/);
+    $outfile = $1;
+    open(F,"$infile") || die "$0 Could not open $infile to read";
+    my @parsetext;
+    my $cnt=0;
+    foreach(<F>){
+	$cnt++;
+	if(/^\s*contains/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^\s*interface/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^[^!]*subroutine/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+	if(/^[^!]*function/i){
+	    push(@parsetext,"# $cnt \"$infile\"\n");
+	}
+
+	push(@parsetext,$_);
+    }
+
+    close(F);
+
+    my $end;
+    my $contains=0;
+    my $in_type_block=0;
+    my @unit;
+    my $unitcnt=0;
+    my $date = localtime();
+    my $preamble = 
+"!===================================================
+! DO NOT EDIT THIS FILE, it was generated using $0 
+! Any changes you make to this file may be lost
+!===================================================\n";
+    my @output ;
+    push(@output,$preamble);
+
+    my $line;
+    my $dimmodifier;
+    my $typemodifier;
+    my $itypeflag;
+    my $block;
+    my $block_type;
+    my $cppunit;
+    foreach $line (@parsetext){
+# skip parser comments
+	next if($line =~ /\s*!pl/);
+
+	$itypeflag=1 if($line =~ /{ITYPE}/);
+	$itypeflag=1 if($line =~ /TYPETEXT/);
+	$itypeflag=1 if($line =~ /TYPEREAL/);
+	$itypeflag=1 if($line =~ /TYPEDOUBLE/);
+	$itypeflag=1 if($line =~ /TYPEINT/);
+	$itypeflag=1 if($line =~ /TYPELONG/);
+
+	
+        if($contains==0){
+	    if($line=~/\s*!\s*DIMS\s+[\d,]+!*/){
+		$dimmodifier=$line;
+		next;
+	    }
+	    if($line=~/\s*!\s*TYPE\s+[^!]+!*$/){
+		$typemodifier=$line;
+		next;
+	    }
+            if ((defined $typemodifier or defined $dimmodifier)
+                and not defined $block and $line=~/^\s*#[^{]*$/) {
+                push(@output, $line);
+                next;
+            }
+            # Figure out the bounds of a type statement.
+            # Type blocks start with "type," "type foo" or "type::" but not
+            # "type(".
+            $in_type_block=1 if($line=~/^\s*type\s*[,:[:alpha:]]/i);
+            $in_type_block=0 if($line=~/^\s*end\s*type/i);
+	    if(not defined $block) {
+                if ($line=~/^\s*type[^[:alnum:]_].*(\{TYPE\}|\{DIMS\})/i or
+                    $line=~/^[^!]*(function|subroutine).*(\{TYPE\}|\{DIMS\})/i) {
+                    $block=$line;
+                    next;
+                }
+                if ($line=~/^\s*interface.*(\{TYPE\}|\{DIMS\})/i) {
+                    $block_type="interface";
+                    $block=$line;
+                    next;
+                }
+	    }
+	    if(not defined $block_type and
+               ($line=~/^\s*end\s+type\s+.*(\{TYPE\}|\{DIMS\})/i or
+                $line=~/^\s*end\s+(function|subroutine)\s+.*(\{TYPE\}|\{DIMS\})/i)){
+
+		$line = $block.$line;
+		undef $block;
+	    }
+            if ($line=~/^\s*end\s*interface/i and
+                defined $block) {
+                $line = $block.$line;
+		undef $block;
+		undef $block_type;
+            }
+	    if(defined $block){
+		$block = $block.$line;
+		next;
+	    }
+	    if(defined $dimmodifier){
+		$line = $dimmodifier.$line;
+		undef $dimmodifier;
+	    } 
+	    if(defined $typemodifier){
+		$line = $typemodifier.$line;
+		undef $typemodifier;
+	    } 
+	    
+	    push(@output, buildout($line));
+            if(($line =~ /^\s*contains\s*!*/i && ! $in_type_block) or
+               ($line =~ /^\s*!\s*Not a module/i)){
+                $contains=1;
+                next;
+            }
+	}
+	if($line=~/^\s*end module\s*/){
+	    $end = $line;
+	    last;
+	}
+
+	if($contains==1){
+	    # first parse into functions or subroutines
+            if($cppunit || !(defined($unit[$unitcnt]))){
+		# Make cpp lines and blanks between routines units.
+		if($line =~ /^\s*\#(?!\s[[:digit:]]+)/ || $line =~/^\s*$/ || $line=~/^\s*!(?!\s*(TYPE|DIMS))/){
+		    push(@{$unit[$unitcnt]},$line);
+		    $cppunit=1;
+		    next;
+		} else {
+                    $cppunit=0;
+                    $unitcnt++;
+                }
+	    }
+
+	       
+	    push(@{$unit[$unitcnt]},$line);
+	    if ($line=~/^\s*interface/i) {
+		$block_type="interface";
+		$block=$line;
+	    }
+	    if ($line=~/^\s*end\s*interface/i) {
+		undef $block_type;
+		undef $block;
+	    }
+	    unless(defined $block){
+		if($line =~ /\s*end function/i or $line =~ /\s*end subroutine/i){
+		    $unitcnt++;
+		}
+	    }
+	}
+    }
+    my $i;
+
+
+    for($i=0;$i<$unitcnt;$i++){
+	if(defined($unit[$i])){
+		my $func = join('',@{$unit[$i]});
+		push(@output, buildout($func));
+	}	
+    }
+    push(@output,@{$unit[$#unit]}) if($unitcnt==$#unit);
+    push(@output, $end);	
+    if($itypeflag==1){
+	my $str;
+	$str.="#include \"dtypes.h\"\n";
+	$write_dtypes = "yes";
+	print $str;
+    }
+    print @output;
+    writedtypes() if(!(-e "dtypes.h") && $write_dtypes == "yes");
+
+
+}
+
+
+sub usage{
+    die("$0 Expected input filename of the form .*.F90.in");
+}
+
+sub build_repeatstr{
+    my($dims) = @_;
+   # Create regex to repeat expression DIMS times.
+    my $repeatstr;
+    for(my $i=1;$i<=$dims;$i++){
+	$repeatstr .="\$\{1\}$i\$\{2\},&\n";
+    }
+    if(defined $repeatstr){
+	$repeatstr="\"$repeatstr";
+	chop $repeatstr;
+	chop $repeatstr;
+	chop $repeatstr;
+	$repeatstr.="\"";
+    }else{
+	$repeatstr='';
+    }
+}
+
+sub writedtypes{
+    open(F,">dtypes.h");
+    print F 
+"#define TYPETEXT 100
+#define TYPEREAL 101	
+#define TYPEDOUBLE 102
+#define TYPEINT 103
+#define TYPELONG 104
+#define TYPELOGICAL 105
+";
+    close(F);
+}
+
+sub buildout{
+    my ($func) = @_;
+    
+    my $outstr;
+    my(@ldims, @ltypes);
+
+    if($func=~/\s*!\s*DIMS\s+([\d,]+)\s*/){
+	@ldims = split(/,/,$1);
+    }else{
+	@ldims = @dims;
+    }
+    if($func=~/\s*!\s*TYPE\s+([^!\s]+)\s*/){
+	@ltypes = split(/,/,$1);	
+#	print ">$func<>@ltypes<\n";
+    }else{
+	@ltypes = @types;
+    }
+    
+
+    if(($func =~ /{TYPE}/ && $func =~ /{DIMS}/) ){
+	my ($type, $dims);
+	foreach $type (@ltypes){
+	    foreach $dims (@ldims){
+		my $dimstr;
+		for(my $i=1;$i<=$dims;$i++){
+		    $dimstr .=':,';
+		}
+		if(defined $dimstr){
+		    $dimstr="($dimstr";
+		    chop $dimstr;
+		    $dimstr.=')';
+		}else{
+		    $dimstr='';
+		}
+
+		my $repeatstr = build_repeatstr($dims);
+		
+		my $str = $func;
+		$str =~ s/{TYPE}/$type/g;
+		$str =~ s/{VTYPE}/$vtype->{$type}/g;
+		$str =~ s/{ITYPE}/$itype->{$type}/g;
+		$str =~ s/{MPITYPE}/$mpitype->{$type}/g;
+		$str =~ s/{NCTYPE}/$nctype->{$type}/g;
+		$str =~ s/{CTYPE}/$ctype->{$type}/g;
+		$str =~ s/{DIMS}/$dims/g;
+		$str =~ s/{DIMSTR}/$dimstr/g;
+                $str =~ s/{REPEAT:([^#}]*)#([^#}]*)}/$repeatstr/eeg;
+		$outstr .= $str;
+	    }
+	}
+    }elsif($func =~ /{DIMS}/){
+        my $dims;
+	foreach $dims (@ldims){
+	    my $dimstr;
+	    for(my $i=1;$i<=$dims;$i++){
+		$dimstr .=':,';
+	    }
+	    if(defined $dimstr){
+		$dimstr="($dimstr";
+		chop $dimstr;
+		$dimstr.=')';
+	    }else{
+		$dimstr='';
+	    }
+		
+	    my $repeatstr = build_repeatstr($dims);
+		
+	    my $str = $func;
+	    $str =~ s/{DIMS}/$dims/g;
+	    $str =~ s/{DIMSTR}/$dimstr/g;
+            $str =~ s/{REPEAT:([^#}]*)#([^#}]*)}/$repeatstr/eeg;
+	    $outstr .= $str;
+	}
+    }elsif($func =~ /{TYPE}/){
+	my ($type);
+	foreach $type (@ltypes){
+	    my $str = $func;
+	    $str =~ s/{TYPE}/$type/g;
+	    $str =~ s/{VTYPE}/$vtype->{$type}/g;
+	    $str =~ s/{ITYPE}/$itype->{$type}/g;
+	    $str =~ s/{MPITYPE}/$mpitype->{$type}/g;
+	    $str =~ s/{NCTYPE}/$nctype->{$type}/g;
+	    $str =~ s/{CTYPE}/$ctype->{$type}/g;
+	    $outstr.=$str;
+	}
+    }else{
+	$outstr=$func;
+    }
+
+    return $outstr;
+}
diff --git a/set_flags.am b/set_flags.am
new file mode 100644
index 00000000000..438deee610d
--- /dev/null
+++ b/set_flags.am
@@ -0,0 +1,18 @@
+# This is part of the PIO package.
+#
+# Assemble the CPPFLAGS and for PIO Fortran tests.
+#
+# Ed Hartnett 3/26/19
+
+# Set the CPPFLAGS.
+AM_CPPFLAGS = -I$(top_srcdir)/src/flib -D_NETCDF
+
+# Is the user building with pnetcdf?
+if BUILD_PNETCDF
+AM_CPPFLAGS += -D_PETCDF
+endif
+
+# Is the user building with netCDF-4 parallel I/O?
+if BUILD_NETCDF4
+AM_CPPFLAGS += -D_NETCDF4
+endif
diff --git a/src/Makefile.am b/src/Makefile.am
new file mode 100644
index 00000000000..7cd8d61f77b
--- /dev/null
+++ b/src/Makefile.am
@@ -0,0 +1,16 @@
+# This is part of PIO. It creates the Makefile for the src directory.
+
+# Ed Hartnett
+
+# Does the user want to build fortran?
+if BUILD_FORTRAN
+FLIB = flib
+endif
+
+if USE_GPTL
+GPTL = gptl
+endif
+
+SUBDIRS = clib ${GPTL} $(FLIB)
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am
new file mode 100644
index 00000000000..eba8240a83e
--- /dev/null
+++ b/src/clib/Makefile.am
@@ -0,0 +1,22 @@
+## This is the automake file to build the PIO C library.
+# Ed Hartnett 8/19/17
+
+# The library we are building.
+lib_LTLIBRARIES = libpio.la
+
+# These linker flags specify libtool version info.
+# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
+# for information regarding incrementing `-version-info`.
+libpio_la_LDFLAGS = -version-info 2:0:1
+
+# The header file.
+include_HEADERS = pio.h
+
+# The library soure files.
+libpio_la_SOURCES = bget.c pioc_sc.c pio_darray.c pio_file.c		\
+pio_getput_int.c pio_msg.c pio_nc.c pio_rearrange.c pioc.c		\
+pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c	\
+pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_internal.h	\
+bget.h uthash.h
+
+EXTRA_DIST = CMakeLists.txt
diff --git a/src/externals/pio1/doc/Error.txt b/src/externals/pio1/doc/Error.txt
index 774e11039d7..c41d45bec95 100644
--- a/src/externals/pio1/doc/Error.txt
+++ b/src/externals/pio1/doc/Error.txt
@@ -17,14 +17,18 @@
 \page error Error Handling
 
 By default, PIO handles errors internally by printing a string
-describing the error and then calling mpi_abort.  Application
+describing the error and then calling mpi_abort. Application
 developers can change this behaivior with a call to
-\ref PIO_seterrorhandling
+\ref PIO_seterrorhandling or PIOc_set_iosystem_error_handling().
 
 For example, if a developer wanted
 to see if an input netcdf file contained the variable 'U' they might do the following:
 
 \verbinclude errorhandle
 
-\copydoc PIO_error_method
+1 - ::PIO_INTERNAL_ERROR abort on error from any task.
+
+2 - ::PIO_BCAST_ERROR broadcast error to all tasks on IO communicator
+
+3 - ::PIO_RETURN_ERROR return error and do nothing else
 */
diff --git a/src/externals/pio1/doc/api.txt b/src/externals/pio1/doc/api.txt
index caba3bb377b..4f2eba6c2f7 100644
--- a/src/externals/pio1/doc/api.txt
+++ b/src/externals/pio1/doc/api.txt
@@ -1,4 +1,4 @@
- /*!  \page api PIO user interface
+ /*!  \page api PIO Fortran Interface
   This is a list of all user interface routines:
 
   \section api_fileops PIO file Operations
diff --git a/src/externals/pio1/doc/base.txt b/src/externals/pio1/doc/base.txt
index 21cfdaf7bf5..60c42fec30d 100644
--- a/src/externals/pio1/doc/base.txt
+++ b/src/externals/pio1/doc/base.txt
@@ -16,7 +16,7 @@
  */
 /*!
 
-\mainpage Parallel I/O library (PIO)
+@mainpage Parallel I/O Libraries (PIO)
 
 \author John M. Dennis
 \author Jim Edwards
diff --git a/src/externals/pio1/doc/faq.txt b/src/externals/pio1/doc/faq.txt
index 0f37956ea19..bce9faad6ba 100644
--- a/src/externals/pio1/doc/faq.txt
+++ b/src/externals/pio1/doc/faq.txt
@@ -15,14 +15,19 @@
  *
  */ /*! \page faq Frequently Asked Questions
 
-   Here is a list of frequently asked questions and their answers.
+Here is a list of frequently asked questions and their answers.
+   
 <dl>
 <dt>How do I specify which tasks perform IO?  </dt>
     <dd>This is done in the call to \ref PIO_init which has two interfaces: init_intracom and init_intercom.
   <ul><li> In the init_intracom interface, use the num_iotasks and stride variables to specify the total number of io tasks and the stride between them with respect to the mpi communicator, comp_comm, which is provided.  You can also use the optional base argument to shift the first IO task away from the first computational task, this is often desirable because the applications first computational task often has higher memory requirements than other tasks.   IO tasks are a subset of the tasks defined in comp_comm.
 <li> In the init_intercom interface, IO tasks are a disjoint set of tasks from those in the computational communicator.  <b>This interface is still experimental and not recommended for production use at this time. </b>
 </ul>
-Note that num_iotasks is the maximum number of IO tasks to use for an IO operation.   The size of the field being read or written along with the tunable blocksize parameter, \ref PIO_set_blocksize, determines the actual number of tasks used for a given IO operation.
+
+Note that num_iotasks is the maximum number of IO tasks to use for an
+IO operation.  The size of the field being read or written along with
+the tunable blocksize parameter, \ref PIO_set_blocksize, determines
+the actual number of tasks used for a given IO operation.
 </dd>
 <dt>How do I test if PIO is installed and working correctly? </dt>
     <dd>The PIO Library distribution contains a testpio subdirectory with a number of programs to test the PIO library. Please see the \ref examp page for details. </dd>
diff --git a/src/externals/pio1/doc/images/I_O_on_Few.png b/src/externals/pio1/doc/images/I_O_on_Few.png
new file mode 100644
index 00000000000..8909c4e1b6a
Binary files /dev/null and b/src/externals/pio1/doc/images/I_O_on_Few.png differ
diff --git a/src/externals/pio1/doc/images/I_O_on_Many_Async.png b/src/externals/pio1/doc/images/I_O_on_Many_Async.png
new file mode 100644
index 00000000000..aa32ad7863e
Binary files /dev/null and b/src/externals/pio1/doc/images/I_O_on_Many_Async.png differ
diff --git a/src/externals/pio1/doc/images/I_O_on_Many_Intracomm.png b/src/externals/pio1/doc/images/I_O_on_Many_Intracomm.png
new file mode 100644
index 00000000000..e48efad2c25
Binary files /dev/null and b/src/externals/pio1/doc/images/I_O_on_Many_Intracomm.png differ
diff --git a/src/externals/pio1/doc/images/Makefile.am b/src/externals/pio1/doc/images/Makefile.am
new file mode 100644
index 00000000000..1ab0bd06f21
--- /dev/null
+++ b/src/externals/pio1/doc/images/Makefile.am
@@ -0,0 +1,9 @@
+# This is part of PIO. It creates the doc/images Makefile.
+
+# Ed Hartnett 5/25/19
+
+# These are the images used in the documentation.
+EXTRA_DIST = block-cyclic.png block-cyclic-rearr.png dof.png	\
+dof-rearr.png PIO_Intracomm1.png PIO_Library_Architecture1.jpg	\
+PIO_Decomposition.png I_O_on_Few.png I_O_on_Many_Intracomm.png	\
+I_O_on_Many_Async.png PIO_Async.png
diff --git a/src/externals/pio1/doc/images/PIO_Async.png b/src/externals/pio1/doc/images/PIO_Async.png
new file mode 100644
index 00000000000..60711639a1c
Binary files /dev/null and b/src/externals/pio1/doc/images/PIO_Async.png differ
diff --git a/src/externals/pio1/doc/images/PIO_Decomposition.png b/src/externals/pio1/doc/images/PIO_Decomposition.png
new file mode 100644
index 00000000000..cfa0de11e56
Binary files /dev/null and b/src/externals/pio1/doc/images/PIO_Decomposition.png differ
diff --git a/src/externals/pio1/doc/images/PIO_Intracomm1.png b/src/externals/pio1/doc/images/PIO_Intracomm1.png
new file mode 100644
index 00000000000..127b2ffe8f2
Binary files /dev/null and b/src/externals/pio1/doc/images/PIO_Intracomm1.png differ
diff --git a/src/externals/pio1/doc/images/PIO_Library_Architecture1.jpg b/src/externals/pio1/doc/images/PIO_Library_Architecture1.jpg
new file mode 100644
index 00000000000..d8058dfb184
Binary files /dev/null and b/src/externals/pio1/doc/images/PIO_Library_Architecture1.jpg differ
diff --git a/src/externals/pio1/pio/pio_kinds.F90 b/src/externals/pio1/pio/pio_kinds.F90
index 1655a4fd5ec..8f1b11cc4d5 100644
--- a/src/externals/pio1/pio/pio_kinds.F90
+++ b/src/externals/pio1/pio/pio_kinds.F90
@@ -7,18 +7,6 @@
 !<
  module pio_kinds
 
-!BOP
-! !MODULE: pio_kinds
-!
-! !DESCRIPTION:
-!  This module defines default numerical data types for all common data
-!  types like integer, character, logical, real4 and real8.
-!
-! !REVISION HISTORY:
-!  CVS:$Id: pio_kinds.F90,v 1.1.1.1 2006/07/31 16:15:30 dennis Exp $
-!  CVS:$Name:  $
-
-! !USES:
 !  uses mpi if available
 #ifndef NO_MPIMOD
    use mpi, only : MPI_OFFSET_KIND ! _EXTERNAL
@@ -43,11 +31,4 @@ module pio_kinds
    integer, parameter, public ::          &
       PIO_OFFSET = MPI_OFFSET_KIND
 
-!EOP
-!BOC
-!EOC
-!***********************************************************************
-
  end module pio_kinds
-
-!|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
diff --git a/src/externals/pio2/src/clib/pio_darray.c b/src/externals/pio2/src/clib/pio_darray.c
index e9f6c28a526..5d16d314271 100644
--- a/src/externals/pio2/src/clib/pio_darray.c
+++ b/src/externals/pio2/src/clib/pio_darray.c
@@ -316,6 +316,16 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     {
 	tmparray = array;
     }
+    if (iodesc->needssort)
+    {
+        if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
+    }
+    else
+    {
+        tmparray = array;
+    }
 
     /* Move data from compute to IO tasks. */
     if ((ierr = rearrange_comp2io(ios, iodesc, tmparray, file->iobuf, nvars)))
@@ -644,6 +654,112 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
     return PIO_NOERR;
 }
 
+/**
+ * Find the fill value that would be used for a variable, if fill mode
+ * was turned on.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param pio_type Type of the variable.
+ * @param type_size Size of one element of this type.
+ * @param fillvalue Pointer that will get the fill value.
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray_c
+ * @author Ed Hartnett
+ */
+static int
+pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_size,
+                          void *fillvalue)
+{
+    signed char byte_fill_value = NC_FILL_BYTE;
+    char char_fill_value = NC_FILL_CHAR;
+    short short_fill_value = NC_FILL_SHORT;
+    int int_fill_value = NC_FILL_INT;
+    float float_fill_value = NC_FILL_FLOAT;
+    double double_fill_value = NC_FILL_DOUBLE;
+    unsigned char ubyte_fill_value = NC_FILL_UBYTE;
+    unsigned short ushort_fill_value = NC_FILL_USHORT;
+    unsigned int uint_fill_value = NC_FILL_UINT;
+    long long int64_fill_value = NC_FILL_INT64;
+    unsigned long long uint64_fill_value = NC_FILL_UINT64;
+    char *string_fill_value = "";
+    int ret;
+
+    /* Check inputs. */
+    assert(fillvalue);
+
+    LOG((2, "pio_inq_var_fill_expected ncid %d varid %d pio_type %d type_size %d",
+         ncid, varid, pio_type, type_size));
+
+    /* Is there a _FillValue attribute? */
+    ret = PIOc_inq_att_eh(ncid, varid, "_FillValue", 0, NULL, NULL);
+
+    LOG((3, "pio_inq_var_fill_expected ret %d", ret));
+
+    /* If there is a fill value, get it. */
+    if (!ret)
+    {
+        if ((ret = PIOc_get_att(ncid, varid, "_FillValue", fillvalue)))
+            return ret;
+    }
+    else /* If no _FillValue at was found we still have work to do. */
+    {
+        /* Did we get some other error? */
+        if (ret != PIO_ENOTATT)
+            return ret;
+
+        /* What is the default fill value for this type? */
+        switch (pio_type)
+        {
+        case PIO_BYTE:
+            memcpy(fillvalue, &byte_fill_value, type_size);
+            break;
+        case PIO_CHAR:
+            memcpy(fillvalue, &char_fill_value, type_size);
+            break;
+        case PIO_SHORT:
+            memcpy(fillvalue, &short_fill_value, type_size);
+            break;
+        case PIO_INT:
+            memcpy(fillvalue, &int_fill_value, type_size);
+            break;
+        case PIO_FLOAT:
+            memcpy(fillvalue, &float_fill_value, type_size);
+            break;
+        case PIO_DOUBLE:
+            memcpy(fillvalue, &double_fill_value, type_size);
+            break;
+#if defined(_NETCDF4) || defined(_PNETCDF)
+        case PIO_UBYTE:
+            memcpy(fillvalue, &ubyte_fill_value, type_size);
+            break;
+        case PIO_USHORT:
+            memcpy(fillvalue, &ushort_fill_value, type_size);
+            break;
+        case PIO_UINT:
+            memcpy(fillvalue, &uint_fill_value, type_size);
+            break;
+        case PIO_INT64:
+            memcpy(fillvalue, &int64_fill_value, type_size);
+            break;
+        case PIO_UINT64:
+            memcpy(fillvalue, &uint64_fill_value, type_size);
+            break;
+#ifdef _NETCDF4
+        case PIO_STRING:
+            memcpy(fillvalue, string_fill_value, type_size);
+            break;
+#endif /* _NETCDF4 */
+#endif/* _NETCDF4 || _PNETCDF */
+        default:
+            return PIO_EBADTYPE;
+        }
+    }
+
+    return PIO_NOERR;
+}
+
 /**
  * Find the fillvalue that should be used for a variable.
  *
@@ -704,6 +820,13 @@ find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
                                               vdesc->fillvalue)))
             return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
 
+    /* Get the fill value one would expect, if NOFILL were not turned
+     * on. */
+    if (!vdesc->use_fill)
+        if ((ierr = pio_inq_var_fill_expected(file->pio_ncid, varid, pio_type, type_size,
+                                              vdesc->fillvalue)))
+            return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
     return PIO_NOERR;
 }
 
diff --git a/src/externals/pio2/src/clib/pio_nc.c b/src/externals/pio2/src/clib/pio_nc.c
index 36c086a9ef2..5af2160b986 100644
--- a/src/externals/pio2/src/clib/pio_nc.c
+++ b/src/externals/pio2/src/clib/pio_nc.c
@@ -2166,7 +2166,7 @@ PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
         /* Get the MPI type corresponding with the PIO type. */
         if ((ierr = find_mpi_type(xtype, &mpi_type, NULL)))
             return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
-        
+
         /* Get the size of the MPI type. */
         if(mpi_type == MPI_DATATYPE_NULL)
             mpi_type_size = 0;
diff --git a/src/externals/pio2/src/clib/pioc.c b/src/externals/pio2/src/clib/pioc.c
index b0e703a6a12..3c727b14f1e 100644
--- a/src/externals/pio2/src/clib/pioc.c
+++ b/src/externals/pio2/src/clib/pioc.c
@@ -23,6 +23,43 @@ extern int event_num[2][NUM_EVENTS];
 extern int ncint_initialized;
 #endif /* NETCDF_INTEGRATION */
 
+/**
+ * @defgroup PIO_init_c Initialize the IO System
+ * Initialize the IOSystem, including specifying number of IO and
+ * computation tasks in C.
+ *
+ * @defgroup PIO_finalize_c Shut Down the IO System
+ * Shut down an IOSystem, freeing all associated resources in C.
+ *
+ * @defgroup PIO_initdecomp_c Initialize a Decomposition
+ * Intiailize a decomposition of data into distributed arrays in C.
+ *
+ * @defgroup PIO_freedecomp_c Free a Decomposition
+ * Free a decomposition, and associated resources in C.
+ *
+ * @defgroup PIO_setframe_c Set the Record Number
+ * Set the record number for a future call to PIOc_write_darray() or
+ * PIOc_read_darray() in C.
+ *
+ * @defgroup PIO_set_hint_c Set a Hint
+ * Set an MPI Hint in C.
+ *
+ * @defgroup PIO_error_method_c Set Error Handling
+ * Set the error handling method in case error is encountered in C.
+ *
+ * @defgroup PIO_get_local_array_size_c Get the Local Size
+ * Get the local size of a distributed array in C.
+ *
+ * @defgroup PIO_iosystem_is_active_c Check IOSystem
+ * Is the IO system active (in C)?
+ *
+ * @defgroup PIO_getnumiotasks_c Get Number IO Tasks
+ * Get the Number of IO Tasks in C.
+ *
+ * @defgroup PIO_set_blocksize_c Set Blocksize
+ * Set the Blocksize in C.
+ */
+
 /**
  * @defgroup PIO_init_c Initialize the IO System
  * Initialize the IOSystem, including specifying number of IO and
@@ -466,6 +503,27 @@ void pio_map_sort(const PIO_Offset *map, int *remap, int maplen)
 }
 
 
+/**
+ * Compare.
+ *
+ * @param a pointer to a
+ * @param b pointer to b
+ * @return -1 if a.map < b.map, 1 if a.map > b.map, 0 if equal
+ * @author Jim Edwards
+ */
+int
+compare( const void* a, const void* b)
+{
+    struct sort_map l_a = * ( (struct sort_map *) a );
+    struct sort_map l_b = * ( (struct sort_map *) b );
+
+    if ( l_a.map < l_b.map )
+        return -1;
+    else if ( l_a.map > l_b.map )
+        return 1;
+    return 0;
+}
+
 /**
  * Compare.
  *
@@ -668,6 +726,33 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma
         }
     }
 
+        if (!(tmpsort = malloc(sizeof(struct sort_map) * maplen)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if (!(iodesc->remap = malloc(sizeof(int) * maplen)))
+        {
+            free(tmpsort);
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+        for (int m=0; m < maplen; m++)
+        {
+            tmpsort[m].remap = m;
+            tmpsort[m].map = compmap[m];
+        }
+        qsort( tmpsort, maplen, sizeof(struct sort_map), compare );
+        for (int m=0; m < maplen; m++)
+        {
+            iodesc->map[m] = compmap[tmpsort[m].remap];
+            iodesc->remap[m] = tmpsort[m].remap;
+        }
+        free(tmpsort);
+    }
+    else
+    {
+        for (int m=0; m < maplen; m++)
+        {
+            iodesc->map[m] = compmap[m];
+        }
+    }
     /* Remember the dim sizes. */
     if (!(iodesc->dimlen = malloc(sizeof(int) * ndims)))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
diff --git a/src/flib/pio_types.F90 b/src/flib/pio_types.F90
new file mode 100644
index 00000000000..aeba6349b7a
--- /dev/null
+++ b/src/flib/pio_types.F90
@@ -0,0 +1,213 @@
+!>
+!! @file
+!! Derived datatypes and constants for PIO Fortran API.
+!! @author Jim Edwards
+!<
+!>
+!! @private
+!! @defgroup iodesc_generate Creating Decompositions
+!! Create a decomposition of data from a variable to multiple
+!! computation tasks.
+!!
+!! @public
+!! @defgroup PIO_iotype PIO_iotype
+!! An integer parameter which controls the iotype.
+!!   - PIO_iotype_pnetcdf : parallel read/write of pNetCDF files (netcdf3)
+!!   - PIO_iotype_netcdf : serial read/write of NetCDF files using 'base_node' (netcdf3)
+!!   - PIO_iotype_netcdf4c : parallel read/serial write of NetCDF4 (HDF5) files with data compression
+!!   - PIO_iotype_netcdf4p : parallel read/write of NETCDF4 (HDF5) files
+!!
+!! @defgroup PIO_rearr_method Rearranger Methods
+!! Rearranger methods.
+!!  - PIO_rearr_none : Do not use any form of rearrangement
+!!  - PIO_rearr_box : Use a PIO internal box rearrangement
+!!  - PIO_rearr_subset : Use a PIO internal subsetting rearrangement
+!!
+!! @defgroup PIO_error_method Error Handling Methods
+!! The error handling setting controls what happens if errors are
+!! encountered by PIO. The three types of error handling methods are:
+!!  - PIO_INTERNAL_ERROR  : abort on error from any task
+!!  - PIO_BCAST_ERROR     : broadcast an error from io_rank 0 to all tasks in comm
+!!  - PIO_RETURN_ERROR    : do nothing - allow the user to handle it
+!>
+!! @defgroup error_return Error Return Codes
+!! The error return code (see @ref PIO_seterrorhandling).
+!!
+!! @defgroup PIO_kinds PIO Fortran Type Kinds
+!! PIO supports different kinds of Fortran types.
+!!  - PIO_double : 8-byte reals or double precision
+!!  - PIO_real : 4-byte reals
+!!  - PIO_int :  4-byte integers
+!!  - PIO_char : character
+
+module pio_types
+  use pio_kinds
+  use iso_c_binding
+  implicit none
+  private
+  type, public :: DecompMap_t !< data structure to describe decomposition.
+#ifdef SEQUENCE
+     sequence
+#endif
+     integer(i4) :: start  !< start
+     integer(i4) :: length !< length
+  end type DecompMap_t
+
+  !>
+  !! @struct iosystem_desc_t
+  !! A defined PIO system descriptor created by @ref PIO_init (see
+  !! pio_types).
+  type, public :: IOSystem_desc_t
+     integer(kind=c_int) :: iosysid = -1 !< iosysid
+  end type IOSystem_desc_t
+
+  !>
+  !! @public
+  !! @struct file_desc_t
+  !! File descriptor returned by \ref PIO_openfile or \ref
+  !! PIO_createfile (see pio_types).
+  type, public :: File_desc_t
+     integer(kind=c_int) :: fh !< file handle
+     type(iosystem_desc_t), pointer :: iosystem => null() !< iosystem
+  end type File_desc_t
+
+  !>
+  !! @public
+  !! @struct io_desc_t
+  !! An decomposition handle that is generated in @ref PIO_initdecomp.
+  !! (see pio_types)
+  type, public :: io_desc_t
+#ifdef SEQUENCE
+     sequence
+#endif
+     integer(i4)         :: ioid !< decomposition id
+  end type io_desc_t
+
+  !>
+  !! @public
+  !! @struct var_desc_t
+  !! A variable descriptor returned from @ref PIO_def_var (see
+  !! pio_types).
+  type, public :: Var_desc_t
+#ifdef SEQUENCE
+     sequence
+#endif
+     integer(i4) :: varID !< variable id
+     integer(i4) :: ncid  !< file id
+  end type Var_desc_t
+
+  integer(i4), public, parameter ::  &
+       PIO_iotype_pnetcdf = 1, &   !< parallel read/write of pNetCDF files
+       PIO_iotype_netcdf  = 2, &   !< serial read/write of NetCDF file using 'base_node'
+       PIO_iotype_netcdf4c = 3, &  !< netcdf4 (hdf5 format) file opened for compression (serial write access only)
+       PIO_iotype_netcdf4p = 4     !< netcdf4 (hdf5 format) file opened in parallel
+
+
+  ! These are for backward compatability and should not be used or expanded upon
+  integer(i4), public, parameter ::                       &
+       iotype_pnetcdf = PIO_iotype_pnetcdf,                & !< pnetcdf iotype
+       iotype_netcdf  = PIO_iotype_netcdf                    !< netcdf iotype
+
+
+  integer(i4), public, parameter :: PIO_rearr_box =  1    !< box rearranger
+  integer(i4), public, parameter :: PIO_rearr_subset =  2 !< subset rearranger
+
+  integer(i4), public, parameter :: PIO_INTERNAL_ERROR = -51 !< abort on error from any task
+  integer(i4), public, parameter :: PIO_BCAST_ERROR = -52    !< broadcast an error
+  integer(i4), public, parameter :: PIO_RETURN_ERROR = -53   !< do nothing
+
+  integer(i4), public, parameter :: PIO_DEFAULT = -1 !< default error handler
+
+  !>
+  !! @struct use_PIO_kinds
+  !! The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+
+#ifdef _PNETCDF
+#include <pnetcdf.inc>
+  integer, public, parameter :: PIO_64BIT_DATA = nf_64bit_data            !< CDF5 format
+#else
+#include <netcdf.inc>
+  integer, public, parameter :: PIO_64BIT_DATA = 0            !< CDF5 format
+#endif
+  integer, public, parameter :: PIO_num_OST =  16 !< num ost
+  integer, public, parameter :: PIO_global = nf_global       !< global atts
+  integer, public, parameter :: PIO_unlimited = nf_unlimited !< unlimited dimension
+  integer, public, parameter :: PIO_double = nf_double       !< double type
+  integer, public, parameter :: PIO_real   = nf_real         !< real type
+  integer, public, parameter :: PIO_int    = nf_int          !< int type
+  integer, public, parameter :: PIO_char   = nf_char         !< char type
+  integer, public, parameter :: PIO_noerr  = nf_noerr        !< no error
+  integer, public, parameter :: PIO_WRITE  = nf_write        !< read-write
+  integer, public, parameter :: PIO_nowrite  = nf_nowrite    !< read-only
+  integer, public, parameter :: PIO_CLOBBER = nf_clobber     !< clobber existing file
+  integer, public, parameter :: PIO_NOCLOBBER = nf_NOclobber !< do not clobber existing file
+  integer, public, parameter :: PIO_NOFILL = nf_nofill       !< do not use fill values
+  integer, public, parameter :: PIO_MAX_NAME = nf_max_name   !< max name len
+  integer, public, parameter :: PIO_MAX_VAR_DIMS = min(6,nf_max_var_dims) !< max dims for a var
+  integer, public, parameter :: PIO_64BIT_OFFSET = nf_64bit_offset        !< 64bit offset format
+  integer, public, parameter :: PIO_FILL_INT = nf_fill_int;               !< int fill value
+  real, public, parameter :: PIO_FILL_FLOAT = nf_fill_float;              !< float fill value
+  double precision, public, parameter :: PIO_FILL_DOUBLE = nf_fill_double; !< double fill value
+
+  enum, bind(c)
+     enumerator :: PIO_rearr_comm_p2p = 0
+     enumerator :: PIO_rearr_comm_coll
+  end enum
+
+  !>
+  !! @defgroup PIO_rearr_comm_t Rearranger Communication
+  !! @public
+  !! There are two choices for rearranger communication.
+  !!  - PIO_rearr_comm_p2p : Point to point
+  !!  - PIO_rearr_comm_coll : Collective
+  !>
+  !>
+  !! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
+  !! @public
+  !! There are four choices for rearranger communication direction.
+  !!  - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
+  !!  - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
+  !!  - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
+  !!  - PIO_rearr_comm_fc_2d_disable: Disable flow control
+  !!
+  !! @defgroup PIO_rearr_comm_fc_options Rearranger Flow Control Options
+  !! Type that defines the PIO rearranger options.
+  !!  - enable_hs : Enable handshake (true/false)
+  !!  - enable_isend : Enable Isends (true/false)
+  !!  - max_pend_req : Maximum pending requests (To indicated unlimited
+  !!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+  !!
+  !! @defgroup PIO_rearr_options Rearranger Options
+  !! Type that defines the PIO rearranger options.
+  !!
+  !!  - comm_type : @copydoc PIO_rearr_comm_t
+  !!  - fcd : @copydoc PIO_rearr_comm_dir
+  !!  - comm_fc_opts_comp2io : @copydoc PIO_rearr_comm_fc_options
+  !!  - comm_fc_opts_io2comp : @copydoc PIO_rearr_comm_fc_options
+  enum, bind(c)
+     enumerator :: PIO_rearr_comm_fc_2d_enable = 0 !< COMM procs to IO procs and vice versa.
+     enumerator :: PIO_rearr_comm_fc_1d_comp2io !< COMM procs to IO procs only.
+     enumerator :: PIO_rearr_comm_fc_1d_io2comp !< IO procs to COMM procs only.
+     enumerator :: PIO_rearr_comm_fc_2d_disable !< Disable flow control.
+  end enum
+
+  type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+     logical(c_bool) :: enable_hs            !< Enable handshake.
+     logical(c_bool) :: enable_isend         !< Enable isends.
+     integer(c_int) :: max_pend_req          !< Maximum pending requests (PIO_REARR_COMM_UNLIMITED_PEND_REQ for unlimited).
+  end type PIO_rearr_comm_fc_opt_t
+
+  integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1 !< unlimited requests
+  type, bind(c), public :: PIO_rearr_opt_t
+     integer(c_int)                         :: comm_type !< Rearranger communication.
+     integer(c_int)                         :: fcd       !< Communication direction.
+     type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io !< The comp2io options.
+     type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp !< The io2comp options.
+  end type PIO_rearr_opt_t
+
+  public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
+       PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
+       PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
+
+end module pio_types
diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90
new file mode 100644
index 00000000000..1f6900d3b08
--- /dev/null
+++ b/src/flib/piolib_mod.F90
@@ -0,0 +1,1678 @@
+#define __PIO_FILE__ "piolib_mod.f90"
+!>
+!! @file
+!! Initialization Routines for PIO.
+!!
+!<
+
+!>
+!! @defgroup PIO_openfile Open a File
+!! Open an existing netCDF file with PIO in Fortran.
+!!
+!! @defgroup PIO_syncfile Sync File
+!! Sync a file to disk, flushing all buffers in Fortran.
+!!
+!! @defgroup PIO_createfile Create a File
+!! Create a new netCDF file in Fortran.
+!!
+!! @defgroup PIO_setframe Set Record Number
+!! Set the record number for distributed array reads/writes in
+!! Fortran.
+!!
+!! @defgroup PIO_closefile Close a File
+!! Close a netCDF file in Fortran.
+!!
+!! @defgroup PIO_freedecomp Free a Decomposition
+!! Free a decomposition, releasing all resources in Fortran.
+!!
+!! @defgroup PIO_init Initialize an IOSystem
+!! Create a new IO System, designating numbers of I/O and computation
+!! tasks in Fortran.
+!!
+!! @defgroup PIO_finalize Free an IOSystem
+!! Free an IO System, releasing all resources in Fortran.
+!!
+!! @defgroup PIO_initdecomp Define a Decomposition
+!! Define a new decomposition of variables to distributed arrays in
+!! Fortran.
+!!
+!! @defgroup PIO_getnumiotasks Get Number IO Tasks
+!! Get the number of IO tasks in Fortran.
+!!
+!! @defgroup PIO_setdebuglevel Internal Debug Settings for Fortran
+!! Set the debug level in Fortran.
+!!
+!! @defgroup PIO_seterrorhandling Error Handling for Fortran
+!! Set the behavior if an error is encountered in Fortran.
+!!
+!! @defgroup PIO_get_local_array_size Get Local Array Size
+!! Get the local size of the distributed array in a decomposition in
+!! Fortran.
+!!
+!! @defgroup PIO_set_hint Set MPI Hint
+!! Set the MPI hint in Fortran.
+
+module piolib_mod
+  use iso_c_binding
+  !--------------
+  use pio_kinds
+  !--------------
+  use pio_types, only : file_desc_t, iosystem_desc_t, var_desc_t, io_desc_t, &
+       pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
+       pio_noerr, pio_rearr_subset, pio_rearr_opt_t
+  !--------------
+  use pio_support, only : piodie, debug, debugio, debugasync, checkmpireturn
+  use pio_nf, only : pio_set_log_level
+  !
+
+#ifdef TIMING
+  use perf_mod, only : t_startf, t_stopf     ! _EXTERNAL
+#endif
+#ifndef NO_MPIMOD
+  use mpi    ! _EXTERNAL
+#endif
+  implicit none
+  private
+#ifdef NO_MPIMOD
+  include 'mpif.h'    ! _EXTERNAL
+#endif
+  ! !public member functions:
+
+  public :: PIO_init,     &
+       PIO_finalize,      &
+       PIO_initdecomp,    &
+       PIO_openfile,      &
+       PIO_syncfile,      &
+       PIO_createfile,    &
+       PIO_closefile,     &
+       PIO_setframe,      &
+       PIO_advanceframe,  &
+       PIO_setdebuglevel, &
+       PIO_seterrorhandling, &
+       PIO_get_local_array_size, &
+       PIO_freedecomp,     &
+       PIO_getnumiotasks, &
+       PIO_set_hint,      &
+       PIO_FILE_IS_OPEN, &
+       PIO_deletefile, &
+       PIO_get_numiotasks, &
+       PIO_iotype_available, &
+       PIO_set_rearr_opts
+
+  !-----------------------------------------------------------------------
+  !
+  !  module variables
+  !
+  !-----------------------------------------------------------------------
+  !>
+  !! Open an existing netCDF file.
+  !<
+  interface PIO_openfile
+     module procedure PIO_openfile
+  end interface PIO_openfile
+
+  !>
+  !! Sync the file to disk, flushing all buffers.
+  !<
+  interface PIO_syncfile
+     module procedure syncfile
+  end interface PIO_syncfile
+
+  !>
+  !! Create a new netCDF file with PIO.
+  !<
+  interface PIO_createfile
+     module procedure createfile
+  end interface PIO_createfile
+
+  !>
+  !! Sets the record number for a future read/write of distributed
+  !! arrays (see @ref PIO_write_darray, @ref PIO_read_darray).
+  !<
+  interface PIO_setframe
+     module procedure setframe
+  end interface PIO_setframe
+
+  !>
+  !! Increment the record number for a future read/write of distributed
+  !! arrays (see @ref PIO_write_darray, @ref PIO_read_darray).
+  !<
+  interface PIO_advanceframe
+     module procedure advanceframe
+  end interface PIO_advanceframe
+
+  !>
+  !! Close an open file.
+  !<
+  interface PIO_closefile
+     module procedure closefile
+  end interface PIO_closefile
+
+  !>
+  !! Free memory associated with a decomposition.
+  !<
+  interface PIO_freedecomp
+     module procedure freedecomp_ios
+     module procedure freedecomp_file
+  end interface PIO_freedecomp
+
+  !>
+  !! Initializes the PIO subsystem, creating a new IOSystem.
+  !<
+  interface PIO_init
+     module procedure init_intracom
+     module procedure init_intercom
+
+  end interface PIO_init
+
+  !>
+  !! Shuts down an IOSystem and associated resources.
+  !<
+  interface PIO_finalize
+     module procedure finalize
+  end interface PIO_finalize
+
+  !>
+  !! PIO_initdecomp is an overload interface the models decomposition to pio.
+  !! initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4,
+  !! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards
+  !! compatibility.
+  !<
+  interface PIO_initdecomp
+     module procedure PIO_initdecomp_dof_i4  ! previous name: initdecomop_1dof_nf_box
+     module procedure PIO_initdecomp_dof_i8  ! previous name: initdecomop_1dof_nf_box
+     module procedure initdecomp_1dof_nf_i4
+     module procedure initdecomp_1dof_nf_i8
+     module procedure initdecomp_1dof_bin_i4
+     module procedure initdecomp_1dof_bin_i8
+     module procedure initdecomp_2dof_nf_i4
+     module procedure initdecomp_2dof_nf_i8
+     module procedure initdecomp_2dof_bin_i4
+     module procedure initdecomp_2dof_bin_i8
+     module procedure PIO_initdecomp_bc
+     !     module procedure PIO_initdecomp_dof_dof
+  end interface PIO_initdecomp
+
+  !>
+  !! Return the actual number of IO-tasks used. PIO will reset the
+  !! total number of IO-tasks if certain conditions are meet.
+  !<
+  interface PIO_get_numiotasks
+     module procedure getnumiotasks
+  end interface PIO_get_numiotasks
+  interface PIO_getnumiotasks
+     module procedure getnumiotasks
+  end interface PIO_getnumiotasks
+
+  !>
+  !! Set the level of debug information that PIO will generate.
+  !<
+  interface PIO_setdebuglevel
+     module procedure setdebuglevel
+  end interface PIO_setdebuglevel
+
+  !>
+  !! Set the form of error handling for PIO.
+  !!
+  !! By default pio handles errors internally by printing a string
+  !! describing the error and calling mpi_abort. Application
+  !! developers can change this behavior for calls to the underlying
+  !! netcdf libraries with a call to PIO_seterrorhandling. For example
+  !! if a developer wanted to see if an input netcdf format file
+  !! contained the variable 'u' they might write the following
+  !! @verbinclude errorhandle
+  !<
+  interface PIO_seterrorhandling
+     module procedure seterrorhandlingfile
+     module procedure seterrorhandlingiosystem
+     module procedure seterrorhandlingiosysid
+  end interface PIO_seterrorhandling
+
+  !>
+  !! Get the local size of a distributed array.
+  !<
+
+contains
+
+!!$#ifdef __GFORTRAN__
+!!$    pure function fptr ( inArr ) result ( ptr )
+!!$        integer (PIO_OFFSET_KIND), dimension(:), target, intent(in) :: inArr
+!!$        integer (PIO_OFFSET_KIND), target :: ptr
+!!$        ptr = inArr(1)
+!!$    end function fptr
+!!$#elif CPRNAG
+!!$! no-op -- nothing here for nag.
+!!$#else
+#define fptr(arg) arg
+!!$#endif
+
+  !>
+  !! @ingroup PIO_file_is_open
+  !! This logical function indicates if a file is open.
+  !! @param File @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
+  logical function PIO_FILE_IS_OPEN(File)
+    type(file_desc_t), intent(in) :: file
+    interface
+       integer(C_INT) function PIOc_File_is_Open(ncid) &
+            bind(C,NAME="PIOc_File_is_Open")
+         use iso_c_binding
+         implicit none
+         integer(c_int), value :: ncid
+       end function PIOc_File_is_Open
+    end interface
+    PIO_FILE_IS_OPEN = .false.
+    if(associated(file%iosystem)) then
+       if(PIOc_File_is_Open(file%fh)==1) then
+          PIO_FILE_IS_OPEN = .true.
+       endif
+    endif
+
+  end function PIO_FILE_IS_OPEN
+
+  !>
+  !! @public
+  !! @ingroup PIO_get_local_array_size
+  !! Return the expected local size of an array associated with a
+  !! decomposition.
+  !! @param iodesc the decomposition.
+  !! @copydoc io_desc_t
+  !! @author Jim Edwards
+  !<
+  integer function PIO_get_local_array_size(iodesc)
+    type(io_desc_t), intent(in) :: iodesc
+    interface
+       integer(C_INT) function PIOc_get_local_array_size(ioid) &
+            bind(C,NAME="PIOc_get_local_array_size")
+         use iso_c_binding
+         implicit none
+         integer(C_INT), value :: ioid
+       end function PIOc_get_local_array_size
+    end interface
+    PIO_get_local_array_size = PIOc_get_local_array_size(iodesc%ioid)
+  end function PIO_get_local_array_size
+
+  !>
+  !! @public
+  !! @ingroup PIO_setframe
+  !! Advance the record dimension of a variable in a netcdf format
+  !! file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copybrief var_desc_t
+  !! @author Jim Edwards
+  !<
+  subroutine advanceframe(file, vardesc)
+    type(file_desc_t), intent(in) :: file
+    type(var_desc_t), intent(inout) :: vardesc
+    integer ierr;
+    interface
+       integer(C_INT) function PIOc_advanceframe(fileid, varid) &
+            bind(C,NAME="PIOc_advanceframe")
+         use iso_c_binding
+         implicit none
+         integer(C_INT), value :: fileid
+         integer(C_INT), value :: varid
+       end function PIOc_advanceframe
+    end interface
+    ierr = PIOc_advanceframe(file%fh, vardesc%varid-1)
+  end subroutine advanceframe
+
+  !>
+  !! @public
+  !! @ingroup PIO_setframe
+  !! Set the record dimension of a variable in a netcdf format file
+  !! or the block address in a binary file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param frame record number
+  !! @author Jim Edwards
+  !<
+  subroutine setframe(file, vardesc,frame)
+    type(file_desc_t) :: file
+    type(var_desc_t), intent(inout) :: vardesc
+    integer(PIO_OFFSET_KIND), intent(in) :: frame
+    integer :: ierr, iframe
+    interface
+       integer(C_INT) function PIOc_setframe(ncid, varid, frame) &
+            bind(C,NAME="PIOc_setframe")
+         use iso_c_binding
+         implicit none
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT), value :: frame
+       end function PIOc_setframe
+    end interface
+    iframe = frame-1
+    ierr = PIOc_setframe(file%fh, vardesc%varid-1, iframe)
+  end subroutine setframe
+
+  !>
+  !! @public
+  !! @public
+  !! @ingroup PIO_setdebuglevel
+  !! Set the level of debug information output to stdout by PIO.
+  !!
+  !! @param level default value is 0, allowed values 0-6
+  !! @author Jim Edwards
+  !<
+  subroutine setdebuglevel(level)
+    integer(i4), intent(in) :: level
+    integer :: ierr
+    if(level.eq.0) then
+       debug=.false.
+       debugio=.false.
+       debugasync=.false.
+    else if(level.eq.1) then
+       debug=.true.
+       debugio=.false.
+       debugasync=.false.
+    else if(level.eq.2) then
+       debug=.false.
+       debugio=.true.
+       debugasync=.false.
+    else if(level.eq.3) then
+       debug=.true.
+       debugio=.true.
+       debugasync=.false.
+    else if(level.eq.4) then
+       debug=.false.
+       debugio=.false.
+       debugasync=.true.
+    else if(level.eq.5) then
+       debug=.true.
+       debugio=.false.
+       debugasync=.true.
+    else if(level.ge.6) then
+       debug=.true.
+       debugio=.true.
+       debugasync=.true.
+    end if
+    ierr = PIO_set_log_level(level)
+    if(ierr /= PIO_NOERR) then
+       ! This is not a fatal error
+       print *, __PIO_FILE__, __LINE__, "Setting log level failed, ierr =",ierr
+    end if
+  end subroutine setdebuglevel
+
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a file.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @param method error handling method
+  !! @param oldmethod old error handling method
+  !! @copydoc PIO_error_method
+  !! @author Jim Edwards
+  !<
+  subroutine seterrorhandlingfile(file, method, oldmethod)
+    type(file_desc_t), intent(inout) :: file
+    integer, intent(in) :: method
+    integer, intent(out), optional :: oldmethod
+    call seterrorhandlingiosysid(file%iosystem%iosysid, method, oldmethod)
+  end subroutine seterrorhandlingfile
+
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a pio system.
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param method
+  !! @copydoc PIO_error_method
+  !! @param oldmethod old error handling method
+  !! @author Jim Edwards
+  !<
+  subroutine seterrorhandlingiosystem(iosystem, method, oldmethod)
+    type(iosystem_desc_t), intent(inout) :: iosystem
+    integer, intent(in) :: method
+    integer, intent(out), optional :: oldmethod
+    call seterrorhandlingiosysid(iosystem%iosysid, method, oldmethod)
+  end subroutine seterrorhandlingiosystem
+
+  !>
+  !! @public
+  !! @ingroup PIO_seterrorhandling
+  !! Set the pio error handling method for a pio system or globally.
+  !!
+  !! @param iosysid a pio system ID (pass PIO_DEFAULT to change the
+  !! global default error handling)
+  !! @param method
+  !! @copydoc PIO_error_method
+  !! @param oldmethod old error handling method
+  !! @author Jim Edwards
+  !<
+  subroutine seterrorhandlingiosysid(iosysid, method, oldmethod)
+    integer, intent(in) :: iosysid
+    integer, intent(in) :: method
+    integer, intent(out), optional :: oldmethod
+
+    interface
+       integer(c_int) function PIOc_Set_IOSystem_Error_Handling(iosysid, method) &
+            bind(C,name="PIOc_Set_IOSystem_Error_Handling")
+         use iso_c_binding
+         integer(c_int), value :: iosysid
+         integer(c_int), value :: method
+       end function PIOc_Set_IOSystem_Error_Handling
+    end interface
+    integer(c_int) ::  loldmethod
+
+    loldmethod = PIOc_Set_IOSystem_Error_Handling(iosysid, method)
+    if(present(oldmethod)) oldmethod = loldmethod
+
+  end subroutine seterrorhandlingiosysid
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! Implements the block-cyclic decomposition for PIO_initdecomp.
+  !! This provides the ability to describe a computational
+  !! decomposition in PIO that has a block-cyclic form. That is
+  !! something that can be described using start and count arrays.
+  !! Optional parameters for this subroutine allows for the
+  !! specification of io decomposition using iostart and iocount
+  !! arrays. If iostart and iocount arrays are not specified by the
+  !! user, and rearrangement is turned on then PIO will calculate a
+  !! suitable IO decomposition
+  !!
+  !! @param iosystem @copydoc iosystem_desc_t
+  !! @param basepiotype @copydoc use_PIO_kinds
+  !! @param dims An array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param compstart The start index into the block-cyclic
+  !! computational decomposition
+  !! @param compcount The count for the block-cyclic computational
+  !! decomposition
+  !! @param iodesc @copydoc iodesc_generate
+  !! @author Jim Edwards
+  !<
+  subroutine PIO_initdecomp_bc(iosystem,basepiotype,dims,compstart,compcount,iodesc)
+    type (iosystem_desc_t), intent(inout) :: iosystem
+    integer(i4), intent(in)               :: basepiotype
+    integer(i4), intent(in)               :: dims(:)
+    integer (kind=PIO_OFFSET_KIND)             :: compstart(:)
+    integer (kind=PIO_OFFSET_KIND)             :: compcount(:)
+    type (IO_desc_t), intent(out)         :: iodesc
+
+    interface
+       integer(C_INT) function PIOc_InitDecomp_bc(iosysid, basetype, ndims, dims, compstart, compcount, ioidp) &
+            bind(C,name="PIOc_InitDecomp_bc")
+         use iso_c_binding
+         integer(C_INT), value :: iosysid
+         integer(C_INT), value :: basetype
+         integer(C_INT), value :: ndims
+         integer(C_INT) :: dims(*)
+         integer(C_INT) :: ioidp
+         integer(C_SIZE_T) :: compstart(*)
+         integer(C_SIZE_T) :: compcount(*)
+       end function PIOc_InitDecomp_bc
+    end interface
+    integer :: i, ndims
+    integer, allocatable ::  cdims(:)
+    integer(PIO_Offset_kind), allocatable :: cstart(:), ccount(:)
+    integer :: ierr
+
+    ndims = size(dims)
+
+    allocate(cstart(ndims), ccount(ndims), cdims(ndims))
+
+    do i=1,ndims
+       cdims(i) = dims(ndims-i+1)
+       cstart(i)  = compstart(ndims-i+1)-1
+       cstart(i)  = compcount(ndims-i+1)
+    end do
+
+    ierr = PIOc_InitDecomp_bc(iosystem%iosysid, basepiotype, ndims, cdims, &
+         cstart, ccount, iodesc%ioid)
+
+
+    deallocate(cstart, ccount, cdims)
+
+
+  end subroutine PIO_initdecomp_bc
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_2dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4)                       :: basetype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in)          :: lenblocks
+    integer (i4), intent(in)          :: compdof(:)   !> global degrees of freedom for computational decomposition
+    integer (i4), intent(in)          :: iodofr(:)     !> global degrees of freedom for io decomposition
+    integer (i4), intent(in)          :: iodofw(:)     !> global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+
+
+    call initdecomp_2dof_bin_i8(iosystem,basepiotype,dims,lenblocks,int(compdof,PIO_OFFSET_KIND),int(iodofr,PIO_OFFSET_KIND), &
+         int(iodofw,PIO_OFFSET_KIND),iodesc)
+
+  end subroutine initdecomp_2dof_bin_i4
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_2dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,iodesc)
+    !    use calcdisplace_mod, only : calcdisplace
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in)          :: lenblocks
+    integer (PIO_OFFSET_KIND), intent(in)          :: compdof(:)   !> global degrees of freedom for computational decomposition
+    integer (PIO_OFFSET_KIND), intent(in)          :: iodofr(:)     !> global degrees of freedom for io decomposition
+    integer (PIO_OFFSET_KIND), intent(in)          :: iodofw(:)     !> global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+
+  end subroutine initdecomp_2dof_bin_i8
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_1dof_bin_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer(i4), intent(in)          :: lenblocks
+    integer(PIO_OFFSET_KIND), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer(PIO_OFFSET_KIND), intent(in)          :: iodofr(:)     ! global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+
+    integer(PIO_OFFSET_KIND) :: start(1), count(1)
+    ! these are not used in the binary interface
+
+    start(1)=-1
+    count(1)=-1
+    call initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,start, count, iodesc)
+  end subroutine initdecomp_1dof_bin_i8
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_1dof_bin_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in)          :: lenblocks
+    integer (i4), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer (i4), intent(in)          :: iodofr(:)     ! global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+
+    integer(PIO_OFFSET_KIND) :: start(1), count(1)
+    ! these are not used in the binary interface
+
+    start(1)=-1
+    count(1)=-1
+    call initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks, &
+         int(compdof,PIO_OFFSET_KIND),int(iodofr,PIO_OFFSET_KIND),start, count, iodesc)
+  end subroutine initdecomp_1dof_bin_i4
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims: an array of the global length of each dimesion of
+  !! the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param start used with count to give a block description of the
+  !! shape of the data
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_2dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,start, count, iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in)          :: lenblocks
+    integer (i4), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer (i4), intent(in)          :: iodofr(:)     ! global degrees of freedom for io decomposition
+    integer (i4), intent(in)          :: iodofw(:)     ! global degrees of freedom for io decomposition
+
+    type (io_desc_t), intent(inout)     :: iodesc
+
+    integer(PIO_OFFSET_KIND), intent(in) :: start(:), count(:)
+    type (io_desc_t) :: tmp
+
+
+    call pio_initdecomp(iosystem, basepiotype,dims,lenblocks,int(compdof,PIO_OFFSET_KIND),int(iodofr,PIO_OFFSET_KIND), &
+         int(iodofw,PIO_OFFSET_KIND),start,count,iodesc)
+
+  end subroutine initdecomp_2dof_nf_i4
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param basepiotype the type of variable(s) associated with this
+  !! iodesc.  @copydoc PIO_kinds
+  !! @param dims: an array of the global length of each dimesion of
+  !! the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to
+  !! its memory order
+  !! @param iodofr
+  !! @param iodofw
+  !! @param start used with count to give a block description of the
+  !! shape of the data
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_2dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodofr,iodofw,start, count, iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in)          :: lenblocks
+    integer (PIO_OFFSET_KIND), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer (PIO_OFFSET_KIND), intent(in)          :: iodofr(:)     ! global degrees of freedom for io decomposition
+    integer (PIO_OFFSET_KIND), intent(in)          :: iodofw(:)     ! global degrees of freedom for io decomposition
+
+    type (io_desc_t), intent(inout)     :: iodesc
+
+    integer(PIO_OFFSET_KIND), intent(in) :: start(:), count(:)
+    type (io_desc_t) :: tmp
+    integer :: ierr
+
+    call initdecomp_1dof_nf_i8(iosystem, basepiotype, dims, lenblocks, compdof, iodofr, start, count, iodesc)
+
+    call initdecomp_1dof_nf_i8(iosystem, basepiotype, dims, lenblocks, compdof, iodofw, start, count, tmp)
+    call mpi_abort(mpi_comm_world, 0, ierr)
+
+  end subroutine initdecomp_2dof_nf_i8
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined PIO system descriptor, see pio_types
+  !! @param basepiotype The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodof
+  !! @param start
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_1dof_nf_i4(iosystem,basepiotype,dims,lenblocks,compdof,iodof,start, count, iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in) :: lenblocks
+    integer (i4), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer (i4), intent(in)          :: iodof(:)     ! global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+    integer :: piotype
+    integer(PIO_OFFSET_KIND), intent(in) :: start(:), count(:)
+
+    call initdecomp_1dof_nf_i8(iosystem, basepiotype,dims,lenblocks,int(compdof,PIO_OFFSET_KIND),int(iodof,PIO_OFFSET_KIND),&
+         start,count,iodesc)
+
+  end subroutine initdecomp_1dof_nf_i4
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! A deprecated interface to the PIO_initdecomp method.
+  !!
+  !! @param iosystem a defined PIO system descriptor, see pio_types
+  !! @param basepiotype The type of variable(s) associated with this iodesc.
+  !! @copydoc PIO_kinds
+  !! @param dims an array of the global length of each dimesion of the variable(s)
+  !! @param lenblocks
+  !! @param compdof mapping of the storage order of the variable to its memory order
+  !! @param iodof
+  !! @param start
+  !! @param count
+  !! @param iodesc @copydoc iodesc_generate
+  !! @deprecated
+  !! @author Jim Edwards
+  !<
+  subroutine initdecomp_1dof_nf_i8(iosystem,basepiotype,dims,lenblocks,compdof,iodof,start, count, iodesc)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (i4), intent(in) :: lenblocks
+    integer (PIO_OFFSET_KIND), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer (PIO_OFFSET_KIND), intent(in)          :: iodof(:)     ! global degrees of freedom for io decomposition
+    type (io_desc_t), intent(inout)     :: iodesc
+    integer :: piotype
+    integer(PIO_OFFSET_KIND), intent(in) :: start(:), count(:)
+
+    if(any(iodof/=compdof)) then
+       call piodie( __PIO_FILE__,__LINE__, &
+            'Not sure what to do here')
+    else
+       call PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc,PIO_REARR_SUBSET, start,count)
+    endif
+
+  end subroutine initdecomp_1dof_nf_i8
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! Implements the degrees of freedom decomposition for
+  !! PIO_initdecomp(). This provides the ability to describe a
+  !! computational decomposition in PIO using degrees of freedom
+  !! method. This is a decomposition that can not be easily described
+  !! using a start and count method.
+  !!
+  !! Optional parameters for this subroutine allows for the
+  !! specififcation of io decomposition using iostart and iocount
+  !! arrays. If iostart and iocount arrays are not specified by the
+  !! user, and rearrangement is turned on then PIO will calculate an
+  !! suitable IO decomposition.
+  !!
+  !! @note This subroutine was previously called \em
+  !! initdecomp_1dof_nf_box.
+  !!
+  !! @param iosystem @copydoc iosystem_desc_t
+  !! @param basepiotype @copydoc use_PIO_kinds
+  !! @param dims An array of the global length of each dimesion of the
+  !! variable(s)
+  !! @param compdof Mapping of the storage order for the computational
+  !! decomposition to its memory order
+  !! @param iodesc @copydoc iodesc_generate
+  !! @param rearr rearranger
+  !! @param iostart The start index for the block-cyclic io
+  !! decomposition
+  !! @param iocount The count for the block-cyclic io decomposition
+  !! @author Jim Edwards
+  !<
+  subroutine PIO_initdecomp_dof_i4(iosystem, basepiotype, dims, compdof, iodesc, rearr, iostart, iocount)
+    type (iosystem_desc_t), intent(inout) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)          :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer, optional, target :: rearr
+    integer (PIO_OFFSET_KIND), optional :: iostart(:), iocount(:)
+    type (io_desc_t), intent(inout)     :: iodesc
+    integer(PIO_OFFSET_KIND), pointer :: internal_compdof(:)
+    integer(i4), intent(in)           :: dims(:)
+
+    allocate(internal_compdof(size(compdof)))
+    internal_compdof = int(compdof,PIO_OFFSET_KIND)
+
+    if(present(iostart) .and. present(iocount) ) then
+       call pio_initdecomp_dof_i8(iosystem, basepiotype, dims, internal_compdof, iodesc, &
+            PIO_REARR_SUBSET, iostart, iocount)
+    else
+       call pio_initdecomp_dof_i8(iosystem, basepiotype, dims, internal_compdof, iodesc, rearr)
+    endif
+    deallocate(internal_compdof)
+
+  end subroutine PIO_initdecomp_dof_i4
+
+  subroutine PIO_initdecomp_internal(iosystem,basepiotype,dims,maplen, compdof, iodesc, rearr, iostart, iocount)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer, intent(in) :: maplen
+    integer (PIO_OFFSET_KIND), intent(in) :: compdof(maplen)   ! global degrees of freedom for computational decomposition
+    integer, optional, target :: rearr
+    integer (PIO_OFFSET_KIND), optional :: iostart(:), iocount(:)
+    type (io_desc_t), intent(inout)     :: iodesc
+
+    integer(c_int) :: ndims
+    integer(c_int), dimension(:), allocatable, target :: cdims
+    integer(PIO_OFFSET_KIND), dimension(:), allocatable, target :: cstart, ccount
+
+    type(C_PTR) :: crearr
+    interface
+       integer(C_INT) function PIOc_InitDecomp(iosysid,basetype,ndims,dims, &
+            maplen, compmap, ioidp, rearr, iostart, iocount)  &
+            bind(C,name="PIOc_InitDecomp")
+         use iso_c_binding
+         integer(C_INT), value :: iosysid
+         integer(C_INT), value :: basetype
+         integer(C_INT), value :: ndims
+         integer(C_INT) :: dims(*)
+         integer(C_INT), value :: maplen
+         integer(C_SIZE_T) :: compmap(*)
+         integer(C_INT) :: ioidp
+         type(C_PTR), value :: rearr
+         type(C_PTR), value :: iostart
+         type(C_PTR), value :: iocount
+       end function PIOc_InitDecomp
+    end interface
+    integer :: ierr,i
+
+    ndims = size(dims)
+    allocate(cdims(ndims))
+    do i=1,ndims
+       cdims(i) = dims(ndims-i+1)
+    end do
+
+    if(present(rearr)) then
+       crearr = C_LOC(rearr)
+    else
+       crearr = C_NULL_PTR
+    endif
+
+    if(present(iostart) .and. present(iocount)) then
+       allocate(cstart(ndims), ccount(ndims))
+       do i=1,ndims
+          cstart(i) = iostart(ndims-i+1)-1
+          ccount(i) = iocount(ndims-i+1)
+       end do
+
+       ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, &
+            maplen, compdof, iodesc%ioid, crearr, C_LOC(cstart), C_LOC(ccount))
+       deallocate(cstart, ccount)
+    else
+       ierr = PIOc_InitDecomp(iosystem%iosysid, basepiotype, ndims, cdims, &
+            maplen, compdof, iodesc%ioid, crearr, C_NULL_PTR, C_NULL_PTR)
+    end if
+
+    deallocate(cdims)
+
+  end subroutine PIO_initdecomp_internal
+
+  !>
+  !! @public
+  !! @ingroup PIO_initdecomp
+  !! I8 version of PIO_initdecomp_dof_i4.
+  !! @author Jim Edwards
+  subroutine PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc, rearr, iostart, iocount)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(in)           :: basepiotype
+    integer(i4), intent(in)           :: dims(:)
+    integer (PIO_OFFSET_KIND), intent(in) :: compdof(:)   ! global degrees of freedom for computational decomposition
+    integer, optional, target :: rearr
+    integer (PIO_OFFSET_KIND), optional :: iostart(:), iocount(:)
+    type (io_desc_t), intent(inout)     :: iodesc
+    integer :: maplen
+
+#ifdef TIMING
+    call t_startf("PIO:initdecomp_dof")
+#endif
+
+    maplen = size(compdof)
+
+    call PIO_initdecomp_internal(iosystem, basepiotype, dims, maplen, compdof, iodesc, rearr, iostart,iocount)
+
+#ifdef TIMING
+    call t_stopf("PIO:initdecomp_dof")
+#endif
+
+  end subroutine PIO_initdecomp_dof_i8
+
+  !>
+  !! @public
+  !! @ingroup PIO_init
+  !! Initialize the pio subsystem. This is a collective call. Input
+  !! parameters are read on comp_rank=0 values on other tasks are
+  !! ignored. This variation of PIO_init locates the IO tasks on a
+  !! subset of the compute tasks.
+  !!
+  !! @param comp_rank mpi rank of each participating task,
+  !! @param comp_comm the mpi communicator which defines the
+  !! collective.
+  !! @param num_iotasks the number of iotasks to define.
+  !! @param num_aggregator the mpi aggregator count
+  !! @param stride the stride in the mpi rank between io tasks.
+  !! @param rearr @copydoc PIO_rearr_method
+  !! @param iosystem a derived type which can be used in subsequent
+  !! pio operations (defined in PIO_types).
+  !! @param base @em optional argument can be used to offset the first
+  !! io task - default base is task 1.
+  !! @param rearr_opts the rearranger options.
+  !! @author Jim Edwards
+  !<
+  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
+    use pio_types, only : pio_internal_error, pio_rearr_opt_t
+    use iso_c_binding
+
+    integer(i4), intent(in) :: comp_rank
+    integer(i4), intent(in) :: comp_comm
+    integer(i4), intent(in) :: num_iotasks
+    integer(i4), intent(in) :: num_aggregator
+    integer(i4), intent(in) :: stride
+    integer(i4), intent(in) :: rearr
+    type (iosystem_desc_t), intent(out)  :: iosystem  ! io descriptor to initalize
+    integer(i4), intent(in),optional :: base
+    type (pio_rearr_opt_t), intent(in), optional :: rearr_opts
+
+    integer :: lbase
+    integer :: ierr
+    interface
+       integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks, stride,base,rearr,rearr_opts,iosysidp) &
+            bind(C,name="PIOc_Init_Intracomm_from_F90")
+         use iso_c_binding
+         use pio_types
+         integer(C_INT), value :: f90_comp_comm
+         integer(C_INT), value :: num_iotasks
+         integer(C_INT), value :: stride
+         integer(C_INT), value :: base
+         integer(C_INT), value :: rearr
+         type(pio_rearr_opt_t) :: rearr_opts
+         integer(C_INT) :: iosysidp
+       end function PIOc_Init_Intracomm_from_F90
+    end interface
+
+#ifdef TIMING
+    call t_startf("PIO:init")
+#endif
+    lbase=0
+    if(present(base)) lbase=base
+    ierr = PIOc_Init_Intracomm_from_F90(comp_comm,num_iotasks,stride,lbase,rearr,rearr_opts,iosystem%iosysid)
+
+    call CheckMPIReturn("Bad Initialization in PIO_Init_Intracomm:  ", ierr,__FILE__,__LINE__)
+#ifdef TIMING
+    call t_stopf("PIO:init")
+#endif
+  end subroutine init_intracom
+
+  !>
+  !! @public
+  !! @ingroup PIO_init
+  !! Initialize the pio subsystem. This is a collective call. Input
+  !! parameters are read on comp_rank=0 values on other tasks are
+  !! ignored. This variation of PIO_init sets up a distinct set of
+  !! tasks to handle IO, these tasks do not return from this
+  !! call. Instead they go to an internal loop and wait to receive
+  !! further instructions from the computational tasks.
+  !!
+  !! @param component_count The number of computational components to
+  !! associate with this IO component.
+  !! @param peer_comm The communicator from which all other
+  !! communicator arguments are derived.
+  !! @param comp_comms The computational communicator for each of the
+  !! computational components.
+  !! @param io_comm The io communicator.
+  !! @param iosystem a derived type which can be used in subsequent
+  !! pio operations (defined in PIO_types).
+  !! @author Jim Edwards
+  !<
+  subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosystem)
+    use pio_types, only : pio_internal_error, pio_rearr_box
+    integer, intent(in) :: component_count
+    integer, intent(in) :: peer_comm
+    integer, intent(in) :: comp_comms(component_count)   !  The compute communicator
+    integer, intent(in) :: io_comm     !  The io communicator
+
+    type (iosystem_desc_t), intent(out)  :: iosystem(component_count)  ! io descriptor to initalize
+#ifdef DOTHIS
+    integer :: ierr
+    logical :: is_inter
+    logical, parameter :: check=.true.
+
+    integer :: i, j, iam, io_leader, comp_leader
+    integer(i4), pointer :: iotmp(:)
+    character(len=5) :: cb_nodes
+    integer :: itmp
+
+#ifdef TIMING
+    call t_startf("PIO:init")
+#endif
+#if defined(NO_MPI2) || defined(_MPISERIAL)
+    call piodie( __PIO_FILE__,__LINE__, &
+         'The PIO async interface requires an MPI2 complient MPI library')
+#else
+    do i=1,component_count
+       iosystem(i)%error_handling = PIO_internal_error
+       iosystem(i)%comp_comm = comp_comms(i)
+       iosystem(i)%io_comm = io_comm
+       iosystem(i)%info = mpi_info_null
+       iosystem(i)%comp_rank= -1
+       iosystem(i)%io_rank  = -1
+       iosystem(i)%async_interface = .true.
+       iosystem(i)%comproot = MPI_PROC_NULL
+       iosystem(i)%ioroot = MPI_PROC_NULL
+       iosystem(i)%compmaster= MPI_PROC_NULL
+       iosystem(i)%iomaster = MPI_PROC_NULL
+       iosystem(i)%numOST = PIO_num_OST
+
+
+       if(io_comm/=MPI_COMM_NULL) then
+          ! Find the rank of the io leader in peer_comm
+          call mpi_comm_rank(io_comm,iosystem(i)%io_rank, ierr)
+          if(iosystem(i)%io_rank==0) then
+             call mpi_comm_rank(peer_comm, iam, ierr)
+          else
+             iam = -1
+          end if
+          call mpi_allreduce(iam, io_leader, 1, mpi_integer, MPI_MAX, peer_comm, ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+          ! Find the rank of the comp leader in peer_comm
+          iam = -1
+          call mpi_allreduce(iam, comp_leader, 1, mpi_integer, MPI_MAX, peer_comm, ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+          ! create the intercomm
+          call mpi_intercomm_create(io_comm, 0, peer_comm, comp_leader, i, iosystem(i)%intercomm, ierr)
+          ! create the union_comm
+          call mpi_intercomm_merge(iosystem(i)%intercomm, .true., iosystem(i)%union_comm, ierr)
+       else
+          ! Find the rank of the io leader in peer_comm
+          iam = -1
+          call mpi_allreduce(iam, io_leader, 1, mpi_integer, MPI_MAX, peer_comm, ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+
+          ! Find the rank of the comp leader in peer_comm
+          iosystem(i)%comp_rank = -1
+          if(comp_comms(i)/=MPI_COMM_NULL) then
+             call mpi_comm_rank(comp_comms(i),iosystem(i)%comp_rank, ierr)
+             if(iosystem(i)%comp_rank==0) then
+                call mpi_comm_rank(peer_comm, iam, ierr)
+             else
+                iam=-1
+             end if
+          end if
+          call mpi_allreduce(iam, comp_leader, 1, mpi_integer, MPI_MAX, peer_comm, ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+
+          ! create the intercomm
+          call mpi_intercomm_create(comp_comms(i), 0, peer_comm, io_leader, i, iosystem(i)%intercomm, ierr)
+          ! create the union comm
+          call mpi_intercomm_merge(iosystem(i)%intercomm, .false., iosystem(i)%union_comm, ierr)
+       end if
+       if(Debugasync) print *,__PIO_FILE__,__LINE__,i, iosystem(i)%intercomm, iosystem(i)%union_comm
+
+       if(iosystem(i)%union_comm /= MPI_COMM_NULL) then
+          call mpi_comm_rank(iosystem(i)%union_comm, iosystem(i)%union_rank, ierr)
+          if(check) call checkmpireturn('init: after call to comm_rank: ',ierr)
+          call mpi_comm_size(iosystem(i)%union_comm, iosystem(i)%num_tasks, ierr)
+          if(check) call checkmpireturn('init: after call to comm_size: ',ierr)
+
+
+          if(io_comm /= MPI_COMM_NULL) then
+             call mpi_comm_size(io_comm, iosystem(i)%num_iotasks, ierr)
+             if(check) call checkmpireturn('init: after call to comm_size: ',ierr)
+
+             if(iosystem(i)%io_rank==0) then
+                iosystem(i)%iomaster = MPI_ROOT
+                iosystem(i)%ioroot = iosystem(i)%union_rank
+             end if
+             iosystem(i)%ioproc = .true.
+             iosystem(i)%compmaster = 0
+
+             call pio_msg_handler_init(io_comm, iosystem(i)%io_rank)
+          end if
+
+
+          if(comp_comms(i) /= MPI_COMM_NULL) then
+             call mpi_comm_size(comp_comms(i), iosystem(i)%num_comptasks, ierr)
+             if(check) call checkmpireturn('init: after call to comm_size: ',ierr)
+
+             iosystem(i)%iomaster = 0
+             iosystem(i)%ioproc = .false.
+             if(iosystem(i)%comp_rank==0) then
+                iosystem(i)%compmaster = MPI_ROOT
+                iosystem(i)%comproot = iosystem(i)%union_rank
+             end if
+
+          end if
+
+          iosystem(i)%userearranger = .true.
+          iosystem(i)%rearr = PIO_rearr_box
+
+          if(Debugasync) print *,__PIO_FILE__,__LINE__
+
+          call MPI_allreduce(iosystem(i)%comproot, j, 1, MPI_INTEGER, MPI_MAX,iosystem(i)%union_comm,ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+
+          iosystem%comproot=j
+          call MPI_allreduce(iosystem(i)%ioroot, j, 1, MPI_INTEGER, MPI_MAX,iosystem(i)%union_comm,ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+
+          iosystem%ioroot=j
+
+          if(Debugasync) print *,__PIO_FILE__,__LINE__, i, iosystem(i)%comproot, iosystem(i)%ioroot
+
+          if(io_comm/=MPI_COMM_NULL) then
+             call mpi_bcast(iosystem(i)%num_comptasks, 1, mpi_integer, iosystem(i)%compmaster,iosystem(i)%intercomm, ierr)
+
+             call mpi_bcast(iosystem(i)%num_iotasks, 1, mpi_integer, iosystem(i)%iomaster, iosystem(i)%intercomm, ierr)
+
+             call alloc_check(iotmp,iosystem(i)%num_iotasks,'init:iotmp')
+             iotmp(:) = 0
+             iotmp( iosystem(i)%io_rank+1)=iosystem(i)%union_rank
+
+          end if
+          if(comp_comms(i)/=MPI_COMM_NULL) then
+             call mpi_bcast(iosystem(i)%num_comptasks, 1, mpi_integer, iosystem(i)%compmaster, iosystem(i)%intercomm, ierr)
+
+             call mpi_bcast(iosystem(i)%num_iotasks, 1, mpi_integer, iosystem(i)%iomaster, iosystem(i)%intercomm, ierr)
+
+             call alloc_check(iotmp,iosystem(i)%num_iotasks,'init:iotmp')
+             iotmp(:)=0
+
+          end if
+
+          iosystem(i)%my_comm = iosystem(i)%intercomm
+
+          call alloc_check(iosystem(i)%ioranks, iosystem(i)%num_iotasks,'init:n_ioranks')
+          if(Debugasync) print *,__PIO_FILE__,__LINE__,iotmp
+          call MPI_allreduce(iotmp,iosystem(i)%ioranks,iosystem(i)%num_iotasks,MPI_INTEGER,MPI_MAX,iosystem(i)%union_comm,ierr)
+          call CheckMPIReturn('Call to MPI_ALLREDUCE()',ierr,__FILE__,__LINE__)
+
+          if(Debugasync) print *,__PIO_FILE__,__LINE__,iosystem(i)%ioranks
+          call dealloc_check(iotmp)
+
+          !---------------------------------
+          ! initialize the rearranger system
+          !---------------------------------
+          if (iosystem(i)%userearranger) then
+             call rearrange_init(iosystem(i))
+          endif
+       end if
+
+#if defined(USEMPIIO) || defined(_PNETCDF) || defined(_NETCDF4)
+       call mpi_info_create(iosystem(i)%info,ierr)
+       ! turn on mpi-io aggregation
+       !DBG    print *,'PIO_init: before call to setnumagg'
+       !       itmp = num_aggregator
+       !       call mpi_bcast(itmp, 1, mpi_integer, 0, iosystem%union_comm, ierr)
+       !       if(itmp .gt. 0) then
+       !          write(cb_nodes,('(i5)')) itmp
+       !#ifdef BGQ
+       !          call PIO_set_hint(iosystem(i),"bgl_nodes_pset",trim(adjustl(cb_nodes)))
+       !#else
+       !          call PIO_set_hint(iosystem(i),"cb_nodes",trim(adjustl(cb_nodes)))
+       !#endif
+       !       endif
+
+#ifdef PIO_GPFS_HINTS
+       call PIO_set_hint(iosystem(i),"ibm_largeblock_io","true")
+#endif
+#ifdef PIO_LUSTRE_HINTS
+       call PIO_set_hint(iosystem(i), 'romio_ds_read','disable')
+       call PIO_set_hint(iosystem(i),'romio_ds_write','disable')
+#endif
+#endif
+    end do
+
+    if(DebugAsync) print*,__PIO_FILE__,__LINE__, iosystem(1)%ioranks
+
+
+    iosystem%num_aiotasks = iosystem%num_iotasks
+    iosystem%numost = PIO_NUM_OST
+
+    ! This routine does not return
+    if(io_comm /= MPI_COMM_NULL) call pio_msg_handler(component_count,iosystem)
+
+    if(DebugAsync) print*,__PIO_FILE__,__LINE__, iosystem(1)%ioranks
+#ifdef TIMING
+    call t_stopf("PIO:init")
+#endif
+#endif
+#endif
+  end subroutine init_intercom
+
+  !>
+  !! @public
+  !! @ingroup PIO_set_hint
+  !! Set file system hints using mpi_info_set. This is a collective
+  !! call.
+  !!
+  !! @param iosystem @copydoc io_desc_t
+  !! @param hint the string name of the hint to define
+  !! @param hintval the string value to set the hint to
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  subroutine PIO_set_hint(iosystem, hint, hintval)
+    type (iosystem_desc_t), intent(inout)  :: iosystem  ! io descriptor to initalize
+    character(len=*), intent(in) :: hint, hintval
+    integer :: ierr
+
+    interface
+       integer(C_INT) function PIOc_set_hint(iosysid, key, val) &
+            bind(C,name="PIOc_set_hint")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+         character(C_CHAR), intent(in) :: key
+         character(C_CHAR), intent(in) :: val
+       end function PIOc_set_hint
+    end interface
+
+    ierr = PIOc_set_hint(iosystem%iosysid, hint, hintval)
+
+  end subroutine PIO_set_hint
+
+  !>
+  !! @public
+  !! @ingroup PIO_finalize
+  !! Finalizes an IO System. This is a collective call.
+  !!
+  !! @param iosystem @copydoc io_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
+  subroutine finalize(iosystem,ierr)
+    type (iosystem_desc_t), intent(inout) :: iosystem
+    integer(i4), intent(out) :: ierr
+    interface
+       integer(C_INT) function PIOc_finalize(iosysid) &
+            bind(C,name="PIOc_finalize")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+       end function PIOc_finalize
+    end interface
+    if(iosystem%iosysid /= -1) then
+       ierr = PIOc_finalize(iosystem%iosysid)
+    endif
+  end subroutine finalize
+
+  !>
+  !! @public
+  !! @ingroup PIO_getnumiotasks
+  !! Return the number of IO-tasks that PIO is using.
+  !!
+  !! @param iosystem a defined pio system descriptor, see PIO_types
+  !! @param numiotasks the number of IO-tasks
+  !! @author Jim Edwards
+  !<
+  subroutine getnumiotasks(iosystem,numiotasks)
+    type (iosystem_desc_t), intent(in) :: iosystem
+    integer(i4), intent(out) :: numiotasks
+    integer :: ierr
+    interface
+       integer(C_INT) function PIOc_get_numiotasks(iosysid,numiotasks) &
+            bind(C,name="PIOc_get_numiotasks")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+         integer(C_INT), intent(out) :: numiotasks
+       end function PIOc_get_numiotasks
+    end interface
+    ierr = PIOc_get_numiotasks(iosystem%iosysid, numiotasks)
+
+  end subroutine getnumiotasks
+
+  !> Is an iotype available?
+  logical function pio_iotype_available( iotype) result(available)
+    integer, intent(in) :: iotype
+    interface
+       integer(C_INT) function PIOc_iotype_available(iotype) &
+            bind(C,name="PIOc_iotype_available")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iotype
+       end function PIOc_iotype_available
+    end interface
+    available= (PIOc_iotype_available(iotype) == 1)
+
+  end function pio_iotype_available
+
+  !>
+  !! @public
+  !! @ingroup PIO_createfile
+  !! Create a NetCDF file using PIO. Input parameters are read on
+  !! comp task 0 and ignored elsewhere.
+  !!
+  !! @param iosystem A defined PIO system descriptor created by a
+  !! call to @ref PIO_init (see PIO_init)
+  !! @param file The returned file descriptor
+  !! @param iotype @copydoc PIO_iotype
+  !! @param fname The name of the file to open
+  !! @param amode_in The NetCDF creation mode flag - NC_NOWRITE for
+  !! read-only access or NC_WRITE for read-write access.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
+  integer function createfile(iosystem, file,iotype, fname, amode_in) result(ierr)
+    type (iosystem_desc_t), intent(inout), target :: iosystem
+    type (file_desc_t), intent(out) :: file
+    integer, intent(in) :: iotype
+    character(len=*), intent(in)  :: fname
+    integer, optional, intent(in) :: amode_in
+    integer :: mode
+    interface
+       integer(C_INT) function PIOc_createfile(iosysid, fh, iotype, fname,mode) &
+            bind(C,NAME='PIOc_createfile')
+         use iso_c_binding
+         implicit none
+         integer(c_int), value :: iosysid
+         integer(c_int) :: fh
+         integer(c_int) :: iotype
+         character(kind=c_char) :: fname(*)
+         integer(c_int), value :: mode
+       end function PIOc_createfile
+    end interface
+    character, allocatable :: cfname(:)
+    integer :: i, nl
+#ifdef TIMING
+    call t_startf("PIO:createfile")
+#endif
+    mode = 0
+    if(present(amode_in)) mode = amode_in
+    nl = len_trim(fname)
+    allocate(cfname(nl+1))
+    do i=1,nl
+       cfname(i) = fname(i:i)
+    enddo
+    cfname(nl+1)=C_NULL_CHAR
+    ierr = PIOc_createfile(iosystem%iosysid, file%fh, iotype, cfname, mode)
+    deallocate(cfname)
+    file%iosystem => iosystem
+#ifdef TIMING
+    call t_stopf("PIO:createfile")
+#endif
+  end function createfile
+
+  !>
+  !! @public
+  !! @ingroup PIO_openfile
+  !! Open an existing file using PIO. Input parameters are read on
+  !! comp task 0 and ignored elsewhere.
+  !!
+  !! @param iosystem a defined PIO system descriptor created by a call
+  !! to @ref PIO_init (see PIO_int)
+  !! @param file the returned file descriptor
+  !! @param iotype @copybrief PIO_iotype
+  !! @param fname the name of the file to open
+  !! @param mode PIO_nowrite or PIO_write.
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
+  integer function PIO_openfile(iosystem, file, iotype, fname,mode) result(ierr)
+
+    !    use ifcore, only: tracebackqq
+    type (iosystem_desc_t), intent(inout), target :: iosystem
+    type (file_desc_t), intent(out) :: file
+    integer, intent(in) :: iotype
+    character(len=*), intent(in)  :: fname
+    integer, optional, intent(in) :: mode
+    integer :: iorank
+    interface
+       integer(C_INT) function PIOc_openfile(iosysid, fh, iotype, fname,mode) &
+            bind(C,NAME='PIOc_openfile')
+         use iso_c_binding
+         implicit none
+         integer(c_int), value :: iosysid
+         integer(c_int) :: fh
+         integer(c_int) :: iotype
+         character(kind=c_char) :: fname(*)
+         integer(c_int), value :: mode
+       end function PIOc_openfile
+    end interface
+    integer :: imode=0, i, nl
+    character, allocatable :: cfname(:)
+#ifdef TIMING
+    call t_startf("PIO:openfile")
+#endif
+    if(present(mode)) imode = mode
+    nl = len_trim(fname)
+    allocate(cfname(nl+1))
+    do i=1,nl
+       cfname(i) = fname(i:i)
+    enddo
+    cfname(nl+1)=C_NULL_CHAR
+    ierr = PIOc_openfile( iosystem%iosysid, file%fh, iotype, cfname, imode)
+    deallocate(cfname)
+    file%iosystem => iosystem
+
+#ifdef TIMING
+    call t_stopf("PIO:openfile")
+#endif
+  end function PIO_openfile
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_syncfile
+  !! Synchronizing a file, forcing all writes to complete before the
+  !! subroutine returns.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
+  subroutine syncfile(file)
+    implicit none
+    type (file_desc_t), target :: file
+    integer :: ierr
+    interface
+       integer(C_INT) function PIOc_sync(ncid) &
+            bind(C,name="PIOc_sync")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: ncid
+       end function PIOc_sync
+    end interface
+
+    ierr = PIOc_sync(file%fh)
+
+  end subroutine syncfile
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_freedecomp
+  !! @brief free all allocated storage associated with this decomposition
+  !! @details
+  !! @param ios :  a defined pio system descriptor created by call to @ref PIO_init (see PIO_types)
+  !! @param iodesc @copydoc io_desc_t
+  !! @author Jim Edwards
+  !<
+  subroutine freedecomp_ios(ios,iodesc)
+    implicit none
+    type (iosystem_desc_t) :: ios
+    type (io_desc_t) :: iodesc
+    integer :: ierr
+    interface
+       integer(C_INT) function PIOc_freedecomp(iosysid, ioid) &
+            bind(C,name="PIOc_freedecomp")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid, ioid
+       end function PIOc_freedecomp
+    end interface
+
+    ierr = PIOc_freedecomp(ios%iosysid, iodesc%ioid)
+
+  end subroutine freedecomp_ios
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_freedecomp
+  !! Free all allocated storage associated with this decomposition.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @param iodesc : @copydoc io_desc_t
+  !! @retval ierr @copydoc error_return
+  !! @author Jim Edwards
+  !<
+  subroutine freedecomp_file(file,iodesc)
+    implicit none
+    type (file_desc_t) :: file
+    type (io_desc_t) :: iodesc
+
+    call syncfile(file)
+
+    call freedecomp_ios(file%iosystem, iodesc)
+
+  end subroutine freedecomp_file
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_closefile
+  !! Close a disk file.
+  !!
+  !! @param file @copydoc file_desc_t
+  !! @author Jim Edwards
+  !<
+  subroutine closefile(file)
+    type(file_desc_t) :: file
+    integer :: ierr
+    interface
+       integer(c_int) function PIOc_closefile(ncid) &
+            bind(C,name="PIOc_closefile")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+       end function PIOc_closefile
+    end interface
+#ifdef TIMING
+    call t_startf("PIO:closefile")
+#endif
+    ierr = PIOc_closefile(file%fh)
+    nullify(file%iosystem)
+#ifdef TIMING
+    call t_stopf("PIO:closefile")
+#endif
+
+  end subroutine closefile
+
+
+  !>
+  !! @public
+  !! @ingroup PIO_deletefile
+  !! Delete a file.
+  !!
+  !! @param ios a pio system handle
+  !! @param fname a filename
+  !! @author Jim Edwards
+  !<
+  subroutine pio_deletefile(ios, fname)
+    type(iosystem_desc_t) :: ios
+    character(len=*) :: fname
+    integer :: ierr
+    interface
+       integer(c_int) function PIOc_deletefile(iosid, fname) &
+            bind(C,name="PIOc_deletefile")
+         use iso_c_binding
+         integer(C_INT), value :: iosid
+         character(kind=c_char) :: fname
+       end function PIOc_deletefile
+    end interface
+
+    ierr = PIOc_deletefile(ios%iosysid, trim(fname)//C_NULL_CHAR)
+
+  end subroutine pio_deletefile
+
+  !>
+  !! @public
+  !! @ingroup PIO_set_rearr_opts
+  !! Set the rerranger options.
+  !!
+  !! @param ios handle to pio iosystem
+  !! @param comm_type @copydoc PIO_rearr_comm_t
+  !! @param fcd : @copydoc PIO_rearr_comm_dir
+  !! @param enable_hs_c2i Enable handshake (compute procs to io procs)
+  !! @param enable_isend_c2i Enable isends (compute procs to io procs)
+  !! @param max_pend_req_c2i Maximum pending requests (compute procs
+  !! to io procs)
+  !! @param enable_hs_i2c Enable handshake (io procs to compute procs)
+  !! @param enable_isend_i2c Enable isends (io procs to compute procs)
+  !! @param max_pend_req_i2c Maximum pending requests (io procs to
+  !! compute procs)
+  !! @copydoc PIO_rearr_comm_fc_options
+  !! @author Jim Edwards
+  !<
+  function pio_set_rearr_opts(ios, comm_type, fcd,&
+       enable_hs_c2i, enable_isend_c2i,&
+       max_pend_req_c2i,&
+       enable_hs_i2c, enable_isend_i2c,&
+       max_pend_req_i2c) result(ierr)
+
+    type(iosystem_desc_t), intent(inout) :: ios
+    integer, intent(in) :: comm_type, fcd
+    logical, intent(in) :: enable_hs_c2i, enable_hs_i2c
+    logical, intent(in) :: enable_isend_c2i, enable_isend_i2c
+    integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c
+    integer :: ierr
+    interface
+       integer(c_int) function PIOc_set_rearr_opts(iosysid, comm_type, fcd,&
+            enable_hs_c2i, enable_isend_c2i,&
+            max_pend_req_c2i,&
+            enable_hs_i2c, enable_isend_i2c,&
+            max_pend_req_i2c)&
+            bind(C,name="PIOc_set_rearr_opts")
+         use iso_c_binding
+         integer(C_INT), intent(in), value :: iosysid
+         integer(C_INT), intent(in), value :: comm_type
+         integer(C_INT), intent(in), value :: fcd
+         logical(C_BOOL), intent(in), value :: enable_hs_c2i
+         logical(C_BOOL), intent(in), value :: enable_isend_c2i
+         integer(C_INT), intent(in), value :: max_pend_req_c2i
+         logical(C_BOOL), intent(in), value :: enable_hs_i2c
+         logical(C_BOOL), intent(in), value :: enable_isend_i2c
+         integer(C_INT), intent(in), value :: max_pend_req_i2c
+       end function PIOc_set_rearr_opts
+    end interface
+
+    ierr = PIOc_set_rearr_opts(ios%iosysid, comm_type, fcd,&
+         logical(enable_hs_c2i, kind=c_bool),&
+         logical(enable_isend_c2i, kind=c_bool),&
+         max_pend_req_c2i,&
+         logical(enable_hs_i2c, kind=c_bool),&
+         logical(enable_isend_i2c, kind=c_bool),&
+         max_pend_req_i2c)
+
+  end function pio_set_rearr_opts
+
+end module piolib_mod
diff --git a/tests/Makefile.am b/tests/Makefile.am
new file mode 100644
index 00000000000..a643bbcd976
--- /dev/null
+++ b/tests/Makefile.am
@@ -0,0 +1,18 @@
+# This file is part of PIO. It generates the Makefiles for the tests
+# directory.
+
+# Ed Hartnett
+
+# Does the user want to build fortran? If so, there are two additional
+# test directories.
+if BUILD_FORTRAN
+UNIT = unit
+GENERAL = general
+if USE_GPTL
+PERFORMANCE = performance
+endif
+endif
+
+SUBDIRS = cunit ${UNIT} ${GENERAL} ${PERFORMANCE}
+
+EXTRA_DIST = CMakeLists.txt