% *======================================================================*
%  Cactus Thorn template for ThornGuide documentation
%  Author: Ian Kelley
%  Date: Sun Jun 02, 2002
%
%  Thorn documentation in the latex file doc/documentation.tex 
%  will be included in ThornGuides built with the Cactus make system.
%  The scripts employed by the make system automatically include 
%  pages about variables, parameters and scheduling parsed from the 
%  relevent thorn CCL files.
%  
%  This template contains guidelines which help to assure that your     
%  documentation will be correctly added to ThornGuides. More 
%  information is available in the Cactus UsersGuide.
%                                                    
%  Guidelines:
%   - Do not change anything before the line
%       % START CACTUS THORNGUIDE",
%     except for filling in the title, author, date etc. fields.
%        - Each of these fields should only be on ONE line.
%        - Author names should be sparated with a \\ or a comma
%   - You can define your own macros are OK, but they must appear after
%     the START CACTUS THORNGUIDE line, and do not redefine standard 
%     latex commands.
%   - To avoid name clashes with other thorns, 'labels', 'citations', 
%     'references', and 'image' names should conform to the following 
%     convention:          
%       ARRANGEMENT_THORN_LABEL
%     For example, an image wave.eps in the arrangement CactusWave and 
%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
%   - Graphics should only be included using the graphix package. 
%     More specifically, with the "includegraphics" command. Do
%     not specify any graphic file extensions in your .tex file. This 
%     will allow us (later) to create a PDF version of the ThornGuide
%     via pdflatex. |
%   - References should be included with the latex "bibitem" command. 
%   - use \begin{abstract}...\end{abstract} instead of \abstract{...}
%   - For the benefit of our Perl scripts, and for future extensions, 
%     please use simple latex.     
%
% *======================================================================* 
% 
% Example of including a graphic image:
%    \begin{figure}[ht]
% 	\begin{center}
%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
% 	\end{center}
% 	\caption{Illustration of this and that}
% 	\label{MyArrangement_MyThorn_MyLabel}
%    \end{figure}
%
% Example of using a label:
%   \label{MyArrangement_MyThorn_MyLabel}
%
% Example of a citation:
%    \cite{MyArrangement_MyThorn_Author99}
%
% Example of including a reference
%   \bibitem{MyArrangement_MyThorn_Author99}
%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999), 
%   1--16. {\tt http://www.nowhere.com/}}
%
% *======================================================================* 

\documentclass{article}

% Use the Cactus ThornGuide style file
% (Automatically used from Cactus distribution, if you have a 
%  thorn without the Cactus Flesh download this from the Cactus
%  homepage at www.cactuscode.org)
\usepackage{../../../../doc/ThornGuide/cactus}

\begin{document}

% The author of the documentation
\author{Erik Schnetter $<$schnetter@uni-tuebingen.de$>$\\
        Christian D. Ott $<$cott@aei.mpg.de$>$\\
        Thomas Radke $<$tradke@aei.mpg.de$>$}

% The title of the document (not necessarily the name of the Thorn)
\title{CarpetIOHDF5}

% the date your document was last changed, if your document is in CVS, 
% please use:
\date{1 December 2004}

\maketitle

% Do not delete next line
% START CACTUS THORNGUIDE


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{abstract}
{\bf CarpetIOHDF5} provides HDF5-based output to the {\em Carpet} mesh
refinement driver in {\em Cactus}.
This document explains {\bf CarpetIOHDF5}'s usage and contains a specification
of the HDF5 file format that was adapted from John Shalf's FlexIO library.
\end{abstract}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Having encountered various problems with the Carpet I/O thorn
{\bf CarpetIOFlexIO} and the underlying FlexIO library,
Erik Schnetter decided to write this thorn {\bf CarpetIOHDF5} which bypasses
any intermediate binary I/O layer and outputs in HDF5 file format directly.

{\bf CarpetIOHDF5} provides output for the {\em Carpet} Mesh Refinement driver
within the Cactus Code. Christian D. Ott added  a file reader (analogous to
Erik Schnetter's implementation present in {\bf CarpetIOFlexIO}) 
as well as checkpoint/recovery functionality to {\bf CarpetIOHDF5}.
Thomas Radke has taken over maintainence of this I/O thorn and is continuously
working on fixing known bugs and improving the code functionality and
efficiency.

Right now, {\bf CarpetIOHDF5} uses serial I/O -- all data are copied to/from
processor 0 for any file I/O operations.

This document aims at giving the user a first handle on how to use
{\bf CarpetIOHDF5}. It also documents the HDF5 file layout used.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Using This Thorn}

\subsection{Obtaining This Thorn}

You can get a checkout from the stable version of Carpet in CVS via

\begin{verbatim}
  cvs -d :pserver:cvs\_anon@cvs.carpetcode.org:/home/cvs/carpet \
      checkout Carpet/CarpetIOHDF5
\end{verbatim}


\subsection{Basic Usage}

First, you have to activate the thorn in your Cactus parameter file:

\begin{verbatim}
  ActiveThorns = "CarpetIOHDF5"
\end{verbatim}

\subsubsection{CarpetIOHDF5 Output Parameters}

\begin{itemize}
  \item {\tt IOHDF5::out\_vars = "$<$variable list$>$"}\\
        list of full names of Cactus grid variables to output;
        Each variable name can have an option string attached in which you
        can specify a different output frequency for that individual variable
        (e.g. {\tt IOHDF5::out\_vars = "wavetoy::phi\{ out\_every = 4 \}"})
  \item {\tt IOHDF5::out\_criterion = "$<$keyword choice$>$"}\\
        criterion to select output intervals (overwrites {\tt IO::out\_criterion})
  \item {\tt IOHDF5::out\_every = $<$integer$>$}\\
        output every {\tt integer} iterations (overwrites {\tt IO::out\_every})
  \item {\tt IOHDF5::out\_dt = $<$number$>$}\\
        output in intervals of that much coordinate time (overwrites {\tt IO::out\_dt})
  \item {\tt IOHDF5::out\_dir = "$<$out\_dir$>$"}\\
        the output directory for HDF5 files (overwrites {\tt IO::out\_dir})
  \item {\tt IO::out\_single\_precision = "yes/no"}\\
        output double-precision data in single precision
\end{itemize}

\subsubsection{Input Parameters}

There are two ways to use the input capabilities:

\begin{enumerate}
  \item For evolutions using ADMBase, one may use the thorn IDFileADM and the following parameter settings:
    \begin{itemize}
      \item {\tt ADMBase::initial\_data  = "read from file"}
      \item {\tt IO::filereader\_ID\_files = "space separated list of files containing the ADM variables"}
      \item {\tt IO::filereader\_ID\_vars = "space separated list of variables to be read in"}
    \end{itemize}
  \item For evolutions not using ADMBase one may try to read in data by setting
    \begin{itemize}
      \item {\tt IOHDF5::in\_dir = "directory from where to read data"}
      \item {\tt IOHDF5::in\_vars = "space separated list of variables to be read in"}
    \end{itemize}
\end{enumerate}


\subsubsection{Checkpointing}

{\bf CarpetIOHDF5} uses the Cactus checkpoint/recovery infrastructure provided
by {\bf CactusBase/IOUtil}.

\begin{itemize}
  \item {\tt IOHDF5::checkpoint = "yes/no"}\\
        Enables/disables checkpointing
  \item {\tt IO::checkpoint\_every = n}\\
        Checkpoint every {\tt n} iterations
  \item {\tt IO::checkpoint\_ID = "yes/no"}\\
        Enables/disables checkpointing after initial data
  \item {\tt IO::checkpoint\_dir = "your preferred checkpoint directory"} 
  \item {\tt IO::checkpoint\_keep = n}\\
        Keep {\tt n} checkpoint files around
\end{itemize}


\subsubsection{Recovery}

{\bf CarpetIOHDF5} uses the Cactus checkpoint/recovery infrastructure provided
by {\bf CactusBase/IOUtil}.
Currently all the checkpoint information is copied onto processor 0 and
written into a single file whose name is invented by {\bf IOUtil}.

In principle, {\bf CarpetIOHDF5} is able to restart on any number of CPUs
from a checkpoint file of a run using any (other or same) number of CPUs.

\begin{itemize}
  \item {\tt IO::recover = "auto"}\\
        Recover from the most recent Checkpoint file. This bombs,
    if no checkpoint file is found.
  \item {\tt IO::recover = "autoprobe"}\\
        Recover from the most recent Checkpoint file. This continues
    without recovering if no checkpoint file is found.
  \item {\tt IO::recover\_dir = "directory containing the checkpoint file"} 
  \item {\tt IO::recover = "manual"}\\
        Recover from a file specified by {\tt iohdf5::recover\_file}. This
     bombs if the file is not found.
  \item {\tt IO::recover\_file = "file you want to recover from"}\\
        Only needs to be set if {\tt IO::recover = "manual"}.
\end{itemize}


\section{CarpetIOHDF5's HDF5 file layout}

The HDF5 file layout of {\bf CarpetIOHDF5} is quite simple.
There are no groups besides the standard HDF5 root data object group:

Each dataset is named according to this template:

\begin{verbatim}
  <group::varname> it=<cctk_iteration> tl=<timelevel> [ml=<mglevel>] [m=<map>]
  [rl=<reflevel>] [c=<component>]}
\end{verbatim}

where optional parts only contribute to the name if they vary (if there is
more than one multigrid level, map, refinement level, component respectively).

Each HDF5 dataset has the following attributes associated with it:

\begin{itemize}
  \item {\tt level} : Carpet::reflevel
  \item {\tt origin} : 1-D array of length vdim. \\
        origin[d] = CCTK\_ORIGIN\_SPACE(d) + cctk\_lbnd[d] * delta[d]
  \item {\tt delta} : 1-D array of length vdim. \\
        delta[d] = CCTK\_DELTA\_SPACE(d)
  \item {\tt time} : cctk\_time
  \item {\tt timestep} : cctk\_iteration
  \item {\tt iorigin} : 1-D array of length vdim. \\ iorigin[d] = (Carpet::ext.lower() / Carpet::ext.stride())[d]
  \item {\tt name} : CCTK\_FullName(variable index)
  \item {\tt cctk\_bbox} : 1-D array of length 2*Carpet::dim. cctk\_box
  \item {\tt cctk\_nghostzones} : 1-D array of length Carpet::dim. cctk\_nghostzones
  \item {\tt carpet\_mglevel} : Carpet::mglevel
  \item {\tt carpet\_reflevel} : Carpet::reflevel
\end{itemize}


\subsection{Attributes needed by the file reader}

The number of attributes needed by the CarpetIOHDF5 file reader is much smaller then the total
number of attributes attached to each dataset:

\begin{itemize}
  \item {\tt name}
  \item {\tt level}
  \item {\tt iorigin}
\end{itemize}


%\subsection{Interaction With Other Thorns}
%
%\subsection{Support and Feedback}
%
%\section{History}
%
%\subsection{Thorn Source Code}
%
%\subsection{Thorn Documentation}
%
%\subsection{Acknowledgements}
%
%
%\begin{thebibliography}{9}
%
%\end{thebibliography}
%
% Do not delete next line
% END CACTUS THORNGUIDE

\end{document}