diff --git a/COPYRIGHT b/COPYRIGHT
new file mode 100644
index 00000000000..8652d01dda1
--- /dev/null
+++ b/COPYRIGHT
@@ -0,0 +1,16 @@
+/******************************************************************************
+ *
+ *
+ *
+ * Copyright (C) 2009-2019
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation under the terms of the GNU General Public License is hereby
+ * granted. No representations are made about the suitability of this software
+ * for any purpose. It is provided "as is" without express or implied warranty.
+ * See the GNU General Public License for more details.
+ *
+ * Documents produced by Doxygen are derivative works derived from the
+ * input used in their production; they are not affected by this license.
+ *
+ */
\ No newline at end of file
diff --git a/Makefile.am b/Makefile.am
index 5fac0f9fcd0..d844a527697 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,4 +12,4 @@ endif
SUBDIRS = src tests examples ${DOC} scripts
-EXTRA_DIST = CMakeLists.txt set_flags.am
+EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT
diff --git a/configure.ac b/configure.ac
index 16205f37532..74afa75012b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -253,6 +253,7 @@ AC_OUTPUT(Makefile
tests/performance/Makefile
doc/Makefile
doc/source/Makefile
+ doc/images/Makefile
examples/Makefile
examples/c/Makefile
scripts/Makefile)
diff --git a/doc/Makefile.am b/doc/Makefile.am
index c5c2523479a..83ea41382d9 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -10,7 +10,9 @@ all:
check: all
-SUBDIRS = source
+# Include these subdirs to include the documention files in the
+# distribution.
+SUBDIRS = source images
CLEANFILES = *.log
diff --git a/doc/images/Makefile.am b/doc/images/Makefile.am
new file mode 100644
index 00000000000..b4d8ec5e447
--- /dev/null
+++ b/doc/images/Makefile.am
@@ -0,0 +1,8 @@
+# This is part of PIO. It creates the doc/images Makefile.
+
+# Ed Hartnett 5/25/19
+
+# These are the images used in the documentation.
+EXTRA_DIST = block-cyclic.png block-cyclic-rearr.png dof.png \
+dof-rearr.png PIO_Intercomm1.png PIO_Intracomm1.png \
+PIO_Library_Architecture1.jpg PIO_Decomposition.png
diff --git a/doc/images/PIO_Decomposition.png b/doc/images/PIO_Decomposition.png
new file mode 100644
index 00000000000..cfa0de11e56
Binary files /dev/null and b/doc/images/PIO_Decomposition.png differ
diff --git a/doc/images/PIO_Intercomm1.png b/doc/images/PIO_Intercomm1.png
new file mode 100644
index 00000000000..accd27400a4
Binary files /dev/null and b/doc/images/PIO_Intercomm1.png differ
diff --git a/doc/images/PIO_Intracomm1.png b/doc/images/PIO_Intracomm1.png
new file mode 100644
index 00000000000..127b2ffe8f2
Binary files /dev/null and b/doc/images/PIO_Intracomm1.png differ
diff --git a/doc/images/PIO_Library_Architecture1.jpg b/doc/images/PIO_Library_Architecture1.jpg
new file mode 100644
index 00000000000..d8058dfb184
Binary files /dev/null and b/doc/images/PIO_Library_Architecture1.jpg differ
diff --git a/doc/source/Decomp.txt b/doc/source/Decomp.txt
index 9ccce62a58f..f47c8bf18e2 100644
--- a/doc/source/Decomp.txt
+++ b/doc/source/Decomp.txt
@@ -1,48 +1,42 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page decomp Describing decompositions
-
-
-One of the biggest challenges to working with PIO is setting up
-the call to \ref PIO_initdecomp. The user must properly describe
-how the data within each MPI tasks memory should be placed or retrieved from
-disk. PIO provides two methods to rearrange data from compute tasks to IO tasks.
-The first method, called box rearrangement is the only one provided in PIO1.
-The second called subset rearrangement is introduced in PIO2.
-
-\section BOXREARR Box rearrangement
+/** @page decomp Describing decompositions
-In this method data is rearranged from compute to IO tasks such that
-the arrangement of data on the IO tasks optimizes the call from the IO
-tasks to the underlying (NetCDF) IO library. In this case each
-compute task will transfer data to one or more IO tasks.
+One of the biggest challenges to working with PIO is setting up the
+decomposition of the data (Fortran users see @ref PIO_initdecomp, C
+users @ref PIO_initdecomp_c). The user must properly describe how the
+data within each MPI tasks memory should be placed or retrieved from
+disk.
+
+@section The Compmap
+
+When initializing a new decomposition, each task calling
+PIOc_init_decomp() or PIO_initdecomp().
+@image html PIO_Decomposition.png
+@section Rearrangers
+PIO provides two methods to rearrange data from compute tasks to
+IO tasks.
+@subsection BOXREARR Box rearrangement
-\section SUBSETREARR Subset rearrangement
+In this method data is rearranged from compute to IO tasks such that
+the arrangement of data on the IO tasks optimizes the call from the IO
+tasks to the underlying (NetCDF) IO library. In this case each compute
+task will transfer data to one or more IO tasks.
+
+@subsection SUBSETREARR Subset rearrangement
In this method each IO task is associated with a unique subset of
compute tasks so that each compute task will transfer data to one and
-only one IO task. Since this technique does not guarantee that data
-on the IO node represents a contiguous block of data on the file it
-may require multiple calls to the underlying (NetCDF) IO library.
+only one IO task. Since this technique does not guarantee that data on
+the IO node represents a contiguous block of data on the file it may
+require multiple calls to the underlying (NetCDF) IO library.
+
+As an example suppose we have a global two dimensional grid of size
+4x5 decomposed over 5 tasks. We represent the two dimensional grid in
+terms of offset from the initial element ie
-As an example suppose we have a global two dimensional grid of size 4x5 decomposed over 5 tasks. We represent the two dimensional grid in terms of offset from the initial element ie
0 1 2 3
4 5 6 7
@@ -50,7 +44,9 @@ As an example suppose we have a global two dimensional grid of size 4x5 decompos
12 13 14 15
16 17 18 19
+
Now suppose this data is distributed over the compute tasks as follows:
+
0: { 0 4 8 12 }
1: { 16 1 5 9 }
@@ -60,11 +56,14 @@ Now suppose this data is distributed over the compute tasks as follows:
If we have 2 io tasks the Box rearranger would give:
+
0: { 0 1 2 3 4 5 6 7 8 9 }
1: { 10 11 12 13 14 15 16 17 18 19 }
+
While the subset rearranger would give:
+
0: { 0 1 4 5 8 9 12 16 }
1: { 2 3 6 7 10 11 13 14 15 17 18 19 }
@@ -72,17 +71,16 @@ While the subset rearranger would give:
Note that while the box rearranger gives a data layout which is well
balanced and well suited for the underlying io library, it had to
-communicate with every compute task to do so. On the other hand the
-subset rearranger communicated with only a portion of the compute tasks
-but requires more work on the part of the underlying io library to complete
-the operation.
-
-Also note if every task is an IO task then the box rearranger will need
-to do an alltoall communication, while the subset rearranger does none.
-In fact using the subset rearranger with every compute task an IO task
-provides a measure of what you might expect the performance of the underlying
-IO library to be if it were used without PIO.
-
+communicate with every compute task to do so. On the other hand the
+subset rearranger communicated with only a portion of the compute
+tasks but requires more work on the part of the underlying io library
+to complete the operation.
+
+Also note if every task is an IO task then the box rearranger will
+need to do an alltoall communication, while the subset rearranger does
+none. In fact using the subset rearranger with every compute task an
+IO task provides a measure of what you might expect the performance of
+the underlying IO library to be if it were used without PIO.
*/
diff --git a/doc/source/Error.txt b/doc/source/Error.txt
index 72c0da23e20..41371d8455b 100644
--- a/doc/source/Error.txt
+++ b/doc/source/Error.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*!
+/*!
\page error Error Handling
By default, PIO handles errors internally by printing a string
diff --git a/doc/source/Examples.txt b/doc/source/Examples.txt
index e18d2926c7c..932d2789578 100644
--- a/doc/source/Examples.txt
+++ b/doc/source/Examples.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page examp Examples
+/*! \page examp Examples
## Examples Included with PIO Distribution
diff --git a/doc/source/GettingStarted.txt b/doc/source/GettingStarted.txt
index aee7661482b..af73190bbe7 100644
--- a/doc/source/GettingStarted.txt
+++ b/doc/source/GettingStarted.txt
@@ -1,21 +1,3 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */
-
-
\tableofcontents
/*! \page intro Introduction
diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt
index 05aa5d1dd61..c49a419197e 100644
--- a/doc/source/Installing.txt
+++ b/doc/source/Installing.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2013
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page install Installing PIO
+/*! \page install Installing PIO
The PIO code is currently stored on github at . For questions about downloading or developing this code, consult the [CIME Git Wiki](https://github.com/CESM-Development/cime/wiki/CIME-Git-Workflow) or email .
diff --git a/doc/source/contributing_code.txt b/doc/source/contributing_code.txt
index faca49aab08..776af900c13 100644
--- a/doc/source/contributing_code.txt
+++ b/doc/source/contributing_code.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2016
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page contributing_code Guide for Contributors
+/*! \page contributing_code Guide for Contributors
# Introduction #
diff --git a/doc/source/faq.txt b/doc/source/faq.txt
index dab793d8667..34f9c3db88c 100644
--- a/doc/source/faq.txt
+++ b/doc/source/faq.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page faq Frequently Asked Questions
+/*! \page faq Frequently Asked Questions
Here is a list of frequently asked questions and their answers.
diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt
index a63e287704d..db8dcfd4537 100644
--- a/doc/source/mach_walkthrough.txt
+++ b/doc/source/mach_walkthrough.txt
@@ -1,19 +1,4 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2013
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */ /*! \page mach_walkthrough Install Walk-through
+/*! \page mach_walkthrough Install Walk-through
This document provides specific instructions for installing PIO using a variety of compilers on a few commonly used super computers. Click on the link below to go directly to the machine of interest.
diff --git a/doc/source/testpio_example.txt b/doc/source/testpio_example.txt
index 07fb3b04b47..c5d4ef7e5ca 100644
--- a/doc/source/testpio_example.txt
+++ b/doc/source/testpio_example.txt
@@ -1,19 +1,3 @@
-/******************************************************************************
- *
- *
- *
- * Copyright (C) 2009
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
- * for any purpose. It is provided "as is" without express or implied warranty.
- * See the GNU General Public License for more details.
- *
- * Documents produced by Doxygen are derivative works derived from the
- * input used in their production; they are not affected by this license.
- *
- */
/*! \page testpio_example testpio: a regression and benchmarking code
The testpio directory, included with the release package, tests both the accuracy