Updated documentation for IOFlexIO.

The documentation.tex file now also describes the IEEEIO file layout
used by IOFlexIO and how to import external data into Cactus.
The CreateIOFlexIOdatafile.c example serves as a template C source
to build a converter program for IEEEIO files which can be fed back
into Cactus.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOFlexIO/trunk@179 ebee0441-1374-4afa-a3b5-247f3ba15b9a

2 files changed, 289 insertions, 53 deletions

diff --git a/doc/CreateIOFlexIOdatafile.c b/doc/CreateIOFlexIOdatafile.c
new file mode 100644
index 0000000..e7ae1e2
--- /dev/null
+++ b/doc/CreateIOFlexIOdatafile.c
@@ -0,0 +1,165 @@
+ /*@@
+   @file      CreateIOFlexIOdatafile.c
+   @date      Mon 12 Mar 2001
+   @author    Thomas Radke
+   @desc
+              Example program to create an unchunked IOFlexIO datafile
+              with a single dataset which can be read as input data
+              into Cactus.
+   @enddesc
+   @version   $Id$
+ @@*/
+
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/* FlexIO includes */
+#include "IOProtos.h"
+#include "IEEEIO.h"
+
+
+/* the name of our sample data file */
+#define DATAFILENAME  "x_3d.ieee"
+
+/* the number of dimensions of our sample data array and its size */
+#define  NDIM         3
+#define  NSIZE        20
+
+/* the type of Cactus variable for the sample dataset (a grid function)
+   This definition was taken from Cactus header files - for ease of use
+   we didn't include these here. */
+#define CCTK_GF                  2
+
+/* a simple macro to do a FlexIO call with return code checking
+   in case of an error it will issue an error message and exit */
+#define CHECK_ERROR(flexio_call)                                                \
+          do                                                                  \
+          {                                                                   \
+            int _error_code = flexio_call;                                    \
+                                                                              \
+                                                                              \
+            if (_error_code < 0)                                              \
+            {                                                                 \
+              fprintf (stderr, "ERROR: line %d: FlexIO call '%s' returned "   \
+                               "error code %d\n",                             \
+                                __LINE__, #flexio_call, _error_code);         \
+              return (-1);                                                    \
+            }                                                                 \
+          } while (0)
+
+
+ /*@@
+   @routine    main
+   @date       Mon 12 Mar 2001
+   @author     Thomas Radke
+   @desc
+               Main routine creating a sample FlexIO datafile
+   @enddesc
+
+   @returntype int
+   @returndesc
+               0 for success, negative return values indicate an error
+   @endreturndesc
+@@*/
+int main (void)
+{
+  void *data;
+  int i, elements, dims[NDIM];
+  Int timelevel, ntimelevels, grouptype;
+  Int nprocs, ioproc_every, unchunked;
+  Int dimsI[NDIM];
+  char *varname, *groupname;
+  IOFile datafile;
+
+
+  /* create a datafile */
+  datafile = IEEEopen (DATAFILENAME, "w");
+  if (! IOisValid (datafile))
+  {
+    fprintf (stderr, "Could not create output file '%s'\n", DATAFILENAME);
+    return (-1);
+  }
+
+  /* set the dimensions of our sample data array
+     count the number of elements */
+  elements = 1;
+  for (i = 0; i < NDIM; i++)
+  {
+    dims[i] = NSIZE;
+    dimsI[i] = NSIZE;
+    elements *= NSIZE;
+  }
+  /* allocate the data array
+     we are lazy here and only initialize it to zero */
+  data = calloc (elements, sizeof (double));
+
+
+  /**************************************************************************/
+  /* write your data as a dataset into the file                             */
+  /**************************************************************************/
+
+  /* write the data */
+  CHECK_ERROR (IOwrite (datafile, FLOAT64, NDIM, dims, data)); 
+
+  /**************************************************************************/
+  /* add the necessary attributes describing the dataset                    */
+  /* as a Cactus grid variable                                              */
+  /**************************************************************************/
+
+  /* the variable's name (as specified in the interface.ccl file) */
+  varname = "grid::x";
+  CHECK_ERROR (IOwriteAttribute (datafile, "name", CHAR,
+                                 strlen (varname) + 1, varname));
+
+  /* the variable's group name (as specified in the interface.ccl file) */
+  groupname = "grid::coordinates";
+  CHECK_ERROR (IOwriteAttribute (datafile, "groupname", CHAR,
+                                 strlen (groupname) + 1, groupname));
+
+  /* the variable's group type (as specified in the interface.ccl file) */
+  grouptype = CCTK_GF;
+  CHECK_ERROR (IOwriteAttribute (datafile, "grouptype", INT32, 1, &grouptype));
+
+  /* the number of timelevels of the variable (as specified in the
+     interface.ccl file) */
+  ntimelevels = 1;
+  CHECK_ERROR (IOwriteAttribute (datafile, "ntimelevels", INT32, 1,
+                                 &ntimelevels));
+
+  /* the timelevel of the variable */
+  timelevel = 1;
+  CHECK_ERROR (IOwriteAttribute (datafile, "timelevel", INT32, 1, &timelevel));
+
+  /* the dimensions of the variable (must correspond with the grid size
+     in your parameter file) */
+  CHECK_ERROR (IOwriteAttribute (datafile, "global_size", INT32, NDIM, dimsI));
+
+
+  /**************************************************************************/
+  /* add the attributes describing how the data in this file was written    */
+  /* these attributes must be attached to the first dataset in the file     */
+  /**************************************************************************/
+
+  /* we are writing unchunked data */
+  unchunked = 1;
+  CHECK_ERROR (IOwriteAttribute (datafile, "unchunked", INT32, 1, &unchunked));
+
+  /* the number of processors isn't really needed here
+     (only for chunked data) */
+  nprocs = 1;
+  CHECK_ERROR (IOwriteAttribute (datafile, "GH$nprocs", INT32, 1, &nprocs));
+
+  /* the number of I/O processors isn't really needed here
+     (only for chunked data) */
+  ioproc_every = 1;
+  CHECK_ERROR (IOwriteAttribute (datafile, "GH$ioproc_every", INT32, 1,
+                                 &ioproc_every));
+
+
+  /* close the file and free allocated resources */
+  CHECK_ERROR (IOclose (datafile));
+  free (data);
+
+  return (0);
+}
diff --git a/doc/documentation.tex b/doc/documentation.tex
index b639c9b..f85cdc6 100644
--- a/doc/documentation.tex
+++ b/doc/documentation.tex
@@ -7,70 +7,141 @@
 \date{1998-1999}
 \maketitle
 
-\abstract{Thorn IOFlexIO provides IO methods for 2D and 3D output of variables
-using John Shalf's FlexIO library ({\tt http://bach.ncsa.uiuc.edu/FlexIO/}).\\
-It also implements checkpointing/recovery functionality.}
+\abstract{Thorn IOFlexIO provides I/O methods to output variables in IEEEIO
+file format. It also implements checkpointing/recovery functionality.}
 %
 \section{Purpose}
 %
-Thorn IOFlexIO registers 2 IO methods with the IO interface in the flesh:
-%
-\begin{itemize}
-  \item {\tt IOFlexIO\_2D} creates two-dimensional output of grid functions
-    extracting the xy, xz, yz planes by slicing through the edge (in the octant
-    case) or center (in all origin centered cases).
-  \item {\tt IOFlexIO\_3D} creates three-dimensional output of grid functions
-    as well as plain output of scalar variables.
-\end{itemize}
-%
-Data for both methods is written in IEEEIO binary file format
-({\tt http://bach.ncsa.uiuc.edu/IEEEIO/}) and goes into files named
-{\tt "<varname>\_2D.ieee"} and {\tt "<varname>\_3D.ieee"} respectively.\\
-Further processing of these data can be done with visualization tools like
-Amira, AVS, and IDL.\\
-%
-\newline
-%
-You obtain output by an IO method by either
+Thorn IOFlexIO uses John Shalf's FlexIO library (see {\tt
+http://bach.ncsa.uiuc.edu/FlexIO/} for details) to output any type of grid
+variables (grid scalars, grid functions, and arrays of arbitrary dimension)
+in the IEEEIO file format.\\
+
+The thorn registers two I/O methods with the flesh's I/O interface at startup:
 %
 \begin{itemize}
-  \item setting the appropriate IO parameters
-  \item calling one the routines of the IO function interface provided by the flesh
+  \item method {\tt IOFlexIO\_3D} outputs all types of grid variables with
+    arbitrary dimensions
+  \item method {\tt IOFlexIO\_2D} outputs two-dimensional slices (xy-, xz-,
+    and yz-slice) of three-dimensional grid functions and arrays
 \end{itemize}
-%
-%
-IOFlexIO also provides checkpointing/recovery functionality by registering
-%
+
+Data is written into files named {\tt "<varname>.ieee"} (for method {\tt IOFlexIO\_3D}) and {\tt "<varname>\_2d\_<plane>.ieee"} (for method {\tt IOFlexIO\_2D}).
+Such datafiles can be used for further postprocessing (eg. visualization)
+or fed back into Cactus via the filereader capabilities of thorn IOUtil.\\[3ex]
+
+
+{\bf Parallel File I/O}\\
+
+According to the ouptput mode parameter settings ({\tt IO::out3D\_mode,
+IO::out3D\_unchunked, IO::out3D\_procs}) of thorn IOUtil, thorn IOFlexIO
+will output distributed data either
 \begin{itemize}
-  \item a checkpoint method with the Runtime Function Repository in the flesh
-     if checkpointing was requested for this thorn.
-     This method is then scheduled at {\em CCTK\_CPINITIAL} and/or
-     {\em CCTK\_CHECKPOINT} (to checkpoint initial and/or evolution data).\\
-     Controled via checkpoint parameters, it decides when to save the current
-     state of simulation by dumping the contents of all Cactus variables and
-     parameters into a checkpoint file which is in IEEEIO file format.\\
-     To write this file the thorn's IO method {\tt IOFlexIO\_3D} is used.
-  \item a recovery method with the generic recovery function interface of thorn IOUtil.\\
-     At recovery time this method is called by IOUtil's generic recovery
-     routine. It gets passed a filename which is tested to identify a checkpoint
-     file in IEEEIO format. If successful the method will then restore the
-     contents of all Cactus variables and parameters from the given checkpoint
-     file.
+  \item in serial into a single unchunked file
+\begin{verbatim}
+  IO::out3D_mode      = "onefile"
+  IO::out3D_unchunked = "yes"
+\end{verbatim}
+  \item in parallel, that is, into separate files containing chunks of the
+        individual processors' patches of the distributed array
+\begin{verbatim}
+  IO::out3D_mode      = "proc | np"
+\end{verbatim}
 \end{itemize}
-%
-For a description of IO and checkpoint/recovery parameters and the generic
-recovery function interface please see also the documentation of thorn IOUtil.\\
-For a description of IO function interface to invoke IO methods by application
-thorns please see the flesh documentation.
-%
-%
+The default is to output data in parallel, in order to get maximum I/O
+performance. If needed, you can recombine the resulting chunked datafiles
+into a single unchunked file using the recombiner utility program provided
+in {\tt IOFlexIO/src/util/}.\\
+
+To build the recombiner just do a
+
+\begin{verbatim}
+  make <configuration>-utils
+\end{verbatim}
+
+in the Cactus toplevel directory. The recombiner executable {\tt
+ieee\_recombiner} will be placed in the {\tt exe/<configuration>/}
+subdirectory.\\[3ex]
+
+
+{\bf Checkpointing \& Recovery}\\
+
+Thorn IOFlexIO can also be used for creating IEEEIO checkpoint files and
+recovering from such files later on.\\
+
+Checkpoint routines are scheduled at several timebins so that you can save
+the current state of your simulation atfer the initial data phase,
+during evolution, or at termination.
+A recovery routine is registered with thorn IOUtil in order to restart
+a new simulation from a given IOFlexIO checkpoint.
+The very same recovery mechanism is used to implement a filereader
+functionality to feed back data into Cactus.\\
+
+Checkpointing and recovery are controlled by corresponding checkpoint/recovery
+parameters of thorn IOUtil (for a description of these parameters please refer
+to this thorn's documentation).
+
+
 \section{Comments}
 
-Since IOFlexIO uses parameters and the recovery function interface from IOUtil
-it also needs this I/O skeleton thorn compiled into Cactus and activated.\\
-%
+{\bf Importing external data into Cactus with IOFlexIO}\\
+
+In order to import external data into Cactus (eg. to initialize some variable)
+you first need to convert this data into an IEEEIO datafile which then can be
+processed by the registered recovery routine of thorn IOFlexIO.\\
+
+The following description explains the IEEEIO file layout of an unchunked
+datafile which thorn IOFlexIO expects in order to restore Cactus variables
+from it properly. There is also a well-documented example C program provided
+({\tt IOFlexIO/doc/CreateIOFlexIOdatafile.c}) which illustrates how to create
+a datafile with IEEEIO file layout. This working example can be used as a
+template for building your own data converter program.\\
+
+\begin{enumerate}
+  \item Actual data is stored as multidimensional datasets in an IEEEIO file.
+
+  \item The type of your data as well as its dimensions are already
+        inherited by a dataset itself as metainformation. But this is not
+        enough for IOFlexIO to savely match it against a specific Cactus
+        variable.
+        For that reason, the variable's name, its groupname, its grouptype, the
+        timelevel to restore, and the
+        total number of timelevels must be attached to every dataset
+        as attribute information.
+
+  \item Finally, the recovery routine needs to know how the datafile to
+        recover from was created:
+        \begin{itemize}
+          \item Does the file contain chunked or unchunked data ?
+          \item How many processors were used to produce the data ?
+          \item How many I/O processors were used to write the data ?
+        \end{itemize}
+        Such information is attached as attributes to the very first dataset
+        in the file. Since we assume unchunked data here
+        the processor information isn't relevant -- unchunked data can
+        be fed back into a Cactus simulation running on an arbitrary
+        number of processors.
+\end{enumerate}
+
+The example C program goes through all of these steps and creates a datafile
+{\tt x\_3d.ieee} in IEEEIO file layout which contains a single dataset named
+{\tt "grid::x"}, with groupname {\tt "grid::coordinates"}, grouptype {\tt
+CCTK\_GF} (thus identifying the variable as a grid function), the timelevel
+to restore set to 0, and the total number of timelevels set to 1.\\
+The global attributes are set to
+{\tt "unchunked" $=$ "yes", nprocs $=$ 1,} and {\tt ioproc\_every $=$ 1}.\\
+
+Once you've built and ran the program you can easily verify if it worked
+properly with
+\begin{verbatim}
+  ioinfo -showattrdata x_3d.ieee
+\end{verbatim}
+which lists all objects in the datafile along with their values.
+Since the single dataset in it only contains zeros
+it would probably not make much sense to feed this datafile into Cactus for
+initializing your x coordinate grid function :-)
 %
-% Automatically created from the ccl files 
+% Automatically created from the ccl files
 % Do not worry for now.
 \include{interface}
 \include{param}