Switched IOFlexIO to use the new Hyperslabbing API for outputting arbitrary

hyperslabs.
You must update thorn CactusBase/IOUtil now.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOFlexIO/trunk@237 ebee0441-1374-4afa-a3b5-247f3ba15b9a

7 files changed, 675 insertions, 1001 deletions

diff --git a/src/DumpGH.c b/src/DumpGH.c
index e411532..beab46f 100644
--- a/src/DumpGH.c
+++ b/src/DumpGH.c
@@ -2,12 +2,12 @@
    @file      DumpGH.c
    @date      Wed Jun 10 14:13:35 1998
    @author    Paul Walker
-   @desc 
+   @desc
               Checkpoint routines scheduled at CCTK_CPINITIAL, CCTK_CHECKPOINT,
               and CCTK_TERMINATE.
               They check the IO checkpointing parameters and - if it's time
               to do so - call the routine which finally creates a checkpoint.
-   @enddesc 
+   @enddesc
    @version   $Id$
  @@*/
 
@@ -15,16 +15,13 @@
 #include "cctk.h"
 #include "cctk_Version.h"
 #include "cctk_Parameters.h"
+#include "Hyperslab.h"
 #include "CactusBase/IOUtil/src/ioGH.h"
 #include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
 #include "ioFlexGH.h"
 
-#include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
 
 /* the rcs ID and its dummy funtion to use it */
 static const char *rcsid = "$Id$";
@@ -34,9 +31,9 @@ CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_DumpGH_c)
 /********************************************************************
  ********************    External Routines   ************************
  ********************************************************************/
-void IOFlexIO_InitialDataCheckpoint (cGH *GH);
-void IOFlexIO_EvolutionCheckpoint (cGH *GH);
-void IOFlexIO_TerminationCheckpoint (cGH *GH);
+void IOFlexIO_InitialDataCheckpoint (const cGH *GH);
+void IOFlexIO_EvolutionCheckpoint (const cGH *GH);
+void IOFlexIO_TerminationCheckpoint (const cGH *GH);
 
 
 /********************************************************************
@@ -47,20 +44,20 @@ static int Checkpoint (const cGH *GH, int called_from);
    @routine    IOFlexIO_InitialDataCheckpoint
    @date       Fri Aug 21 14:46:28 1998
    @author     Gerd Lanfermann
-   @desc 
+   @desc
                This routine is registered at CCTK_CPINITIAL.
                It checks if initial data should be checkpointed.
-   @enddesc 
+   @enddesc
 
    @calls      Checkpoint
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
-   @vtype      cGH *
+   @vdesc      pointer to CCTK grid hierarchy
+   @vtype      const cGH *
    @vio        in
    @endvar
 @@*/
-void IOFlexIO_InitialDataCheckpoint (cGH *GH)
+void IOFlexIO_InitialDataCheckpoint (const cGH *GH)
 {
   DECLARE_CCTK_PARAMETERS
 
@@ -76,20 +73,20 @@ void IOFlexIO_InitialDataCheckpoint (cGH *GH)
    @routine    IOFlexIO_EvolutionCheckpoint
    @date       Fri Aug 21 14:38:25 1998
    @author     Gabrielle Allen
-   @desc 
+   @desc
                This routine is registered at CCTK_CHECKPOINT.
                It periodically checks if it's time to checkpoint evolution data.
-   @enddesc 
+   @enddesc
 
    @calls      Checkpoint
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
-   @vtype      cGH *
+   @vdesc      pointer to CCTK grid hierarchy
+   @vtype      const cGH *
    @vio        in
    @endvar
 @@*/
-void IOFlexIO_EvolutionCheckpoint (cGH *GH)
+void IOFlexIO_EvolutionCheckpoint (const cGH *GH)
 {
   DECLARE_CCTK_PARAMETERS
 
@@ -119,20 +116,20 @@ void IOFlexIO_EvolutionCheckpoint (cGH *GH)
    @routine    IOFlexIO_TerminationCheckpoint
    @date       Fri Aug 21 14:40:21 1998
    @author     Gabrielle Allen
-   @desc 
+   @desc
                This routine is registered at CCTK_TERMINATE.
                It checks if the last iteration should be checkpointed.
-   @enddesc 
+   @enddesc
 
    @calls      Checkpoint
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
-   @vtype      cGH *
+   @vdesc      pointer to CCTK grid hierarchy
+   @vtype      const cGH *
    @vio        in
    @endvar
 @@*/
-void IOFlexIO_TerminationCheckpoint (cGH *GH)
+void IOFlexIO_TerminationCheckpoint (const cGH *GH)
 {
   DECLARE_CCTK_PARAMETERS
 
@@ -155,7 +152,7 @@ void IOFlexIO_TerminationCheckpoint (cGH *GH)
    @calls      IOUtil_GetAllParameters
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
+   @vdesc      pointer to CCTK grid hierarchy
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -197,7 +194,7 @@ void IOFlexIOi_DumpParameters (const cGH *GH, int all, IOFile file)
    @enddesc
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
+   @vdesc      pointer to CCTK grid hierarchy
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -246,11 +243,11 @@ void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
    @routine    Checkpoint
    @date       Fri Aug 21 15:13:13 1998
    @author     Paul Walker
-   @desc 
+   @desc
                The heart of checkpointing.
                Called by the different wrappers, this routine creates
                a new checkpoint file and then dumps away all grid variables.
-   @enddesc 
+   @enddesc
 
    @calls      IOUtil_PrepareFilename
                IOUtil_PrintTimings
@@ -258,8 +255,8 @@ void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
                IOFlexIOi_DumpGHExtensions
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
-   @vtype      cGH *
+   @vdesc      pointer to CCTK grid hierarchy
+   @vtype      const cGH *
    @vio        in
    @endvar
    @var        called_from
@@ -269,10 +266,10 @@ void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
    @vio        in
    @endvar
 
-   @history 
+   @history
    @hdate      Oct 4 1998
    @hauthor    Gabrielle Allen
-   @hdesc      Removed code which checked and reset unchunked, assume that this 
+   @hdesc      Removed code which checked and reset unchunked, assume that this
                is done when the variable is first set.
    @hdate      Nov 4 1998
    @hauthor    Gabrielle Allen
@@ -281,7 +278,7 @@ void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
    @hauthor    Thomas Radke
    @hdesc      Removed forced sync before checkpointing (just do a normal sync)
                Introduced a ring buffer for keeping last <checkpoint_keep> files
-   @endhistory 
+   @endhistory
 
    @returntype int
    @returndesc
@@ -292,38 +289,23 @@ void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
 @@*/
 static int Checkpoint (const cGH *GH, int called_from)
 {
-  char filename [1024], tmpfilename [1024];
+  char filename[1024], tmpfilename[1024];
   IOFile file;
-  int myproc, first_vindex, vindex, gindex, maxdim, retval;
-  int timelevel, last_timelevel;
+  int myproc, first_vindex, gindex, retval;
   cGroup gdata;
-  int *old_downsample;
-  int old_out_single;
-  ioGH *ioUtilGH;
+  ioRequest *request;
   flexioGH *myGH;
+  const ioGH *ioUtilGH;
+  char *fullname;
   static char **cp_filenames = NULL;
   static int cp_fileindex = 0;
-  /*** FIXME: can CCTK_SyncGroup() have a 'const cGH *' parameter ?? ***/
-  union
-  {
-    const cGH *const_ptr;
-    cGH *non_const_ptr;
-  } GH_fake_const;
-  const char *timer_descriptions [3] = {"Time to dump parameters: ",
-                                        "Time to dump datasets:   ",
-                                        "Total time to checkpoint:"};
+  const char *timer_descriptions[3] = {"Time to dump parameters: ",
+                                       "Time to dump datasets:   ",
+                                       "Total time to checkpoint:"};
   DECLARE_CCTK_PARAMETERS
 
 
   retval = 0;
-  GH_fake_const.const_ptr = GH;
-
-  /* get the handles for IOUtil and IOFlexIO extensions */
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
-
-  myproc = CCTK_MyProc (GH);
-  maxdim = CCTK_MaxDim ();
 
   /* allocate the ring buffer for checkpoint filenames */
   if (! cp_filenames)
@@ -331,33 +313,24 @@ static int Checkpoint (const cGH *GH, int called_from)
     cp_filenames = (char **) calloc (checkpoint_keep, sizeof (char *));
   }
 
-  /* disable downsampling after saving original downsampling params */
-  old_downsample = (int *) malloc (maxdim * sizeof (int));
-  for (vindex = 0; vindex < maxdim; vindex++)
-  {
-    old_downsample [vindex] = ioUtilGH->downsample [vindex];
-    ioUtilGH->downsample [vindex] = 1;
-  }
-
-  /* disable output in single precision */
-  old_out_single = ioUtilGH->out_single;
-  ioUtilGH->out_single = 0;
-
   /* start the total timer */
+  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
   if (myGH->print_timing_info)
   {
     CCTK_TimerStartI (myGH->timers[2]);
   }
 
   /* get the base filename ... */
+  myproc = CCTK_MyProc (GH);
+  ioUtilGH = (const ioGH *) CCTK_GHExtension (GH, "IO");
   IOUtil_PrepareFilename (GH, NULL, filename, called_from,
                           myproc / ioUtilGH->ioproc_every, ioUtilGH->unchunked);
 
   /* ... and append the extension */
   sprintf (tmpfilename,  "%s.tmp.ieee", filename);
-  sprintf (filename, "%s.ieee",     filename);
+  sprintf (filename, "%s.ieee", filename);
 
-  /* Now open the file */
+  /* now open the file */
   if (myproc == ioUtilGH->ioproc)
   {
     if (verbose)
@@ -405,6 +378,10 @@ static int Checkpoint (const cGH *GH, int called_from)
   }
 
   /* ... now the variables, sorted by groups */
+  if (verbose)
+  {
+    CCTK_INFO ("Dumping Grid Variables ...");
+  }
   for (gindex = CCTK_NumGroups () - 1; gindex >= 0; gindex--)
   {
     /* only dump groups which have storage assigned */
@@ -413,28 +390,46 @@ static int Checkpoint (const cGH *GH, int called_from)
       continue;
     }
 
-    /* sync the group */
-    CCTK_SyncGroupI (GH_fake_const.non_const_ptr, gindex);
-
     /* dump all timelevels except the oldest (for multi-level groups) */
     CCTK_GroupData (gindex, &gdata);
-    last_timelevel = gdata.numtimelevels;
-    if (last_timelevel > 1)
+    if (gdata.numtimelevels > 1)
     {
-      last_timelevel--;
+      gdata.numtimelevels--;
     }
+
     first_vindex = CCTK_FirstVarIndexI (gindex);
 
-    for (vindex = first_vindex; vindex < first_vindex + gdata.numvars; vindex++)
+    /* get the default I/O request for this group */
+    request = IOUtil_DefaultIORequest (GH, first_vindex);
+
+    /* disable checking for old data objects, disable datatype conversion
+       and downsampling */
+    request->check_exist = 0;
+    request->hdatatype = gdata.vartype;
+    for (request->hdim = 0; request->hdim < request->vdim; request->hdim++)
     {
-      if (verbose)
-      {
-        CCTK_VInfo (CCTK_THORNSTRING, "  %s", CCTK_VarName (vindex));
-      }
+      request->downsample[request->hdim] = 1;
+    }
 
-      for (timelevel = 0; timelevel < last_timelevel; timelevel++)
+    /* loop over all variables in this group */
+    for (request->vindex = first_vindex;
+         request->vindex < first_vindex + gdata.numvars;
+         request->vindex++)
+    {
+      /* loop over all timelevels of this variable */
+      for (request->timelevel = 0;
+           request->timelevel < gdata.numtimelevels;
+           request->timelevel++)
       {
-        retval += IOFlexIO_DumpVar (GH, vindex, timelevel, file);
+        if (verbose)
+        {
+          fullname = CCTK_FullName (request->vindex);
+          CCTK_VInfo (CCTK_THORNSTRING, "  %s (timelevel %d)",
+                      fullname, request->timelevel);
+          free (fullname);
+        }
+
+        retval += IOFlexIO_DumpVar (GH, request, file);
       }
     } /* end of loop over all variables */
   } /* end of loop over all groups */
@@ -447,7 +442,10 @@ static int Checkpoint (const cGH *GH, int called_from)
 
   /* close the temporary checkpoint file */
   FLEXIO_ERROR (IOclose (file));
-  
+
+  /* free I/O request */
+  IOUtil_FreeIORequest (&request);
+
   if (myproc == ioUtilGH->ioproc)
   {
     if (verbose)
@@ -471,24 +469,17 @@ static int Checkpoint (const cGH *GH, int called_from)
 
   /* delete the oldest dumpfile if checkpoint_keep_all isn't set
      and put the new filename into the ring buffer */
-  if (cp_filenames [cp_fileindex])
+  if (cp_filenames[cp_fileindex])
   {
     if (! checkpoint_keep_all)
     {
-      remove (cp_filenames [cp_fileindex]);
+      remove (cp_filenames[cp_fileindex]);
     }
-    free (cp_filenames [cp_fileindex]);
+    free (cp_filenames[cp_fileindex]);
   }
-  cp_filenames [cp_fileindex] = strdup (filename);
+  cp_filenames[cp_fileindex] = strdup (filename);
   cp_fileindex = (cp_fileindex + 1) % checkpoint_keep;
 
-  /* restore output precision flag */
-  ioUtilGH->out_single = old_out_single;
-
-  /* restore original downsampling params */
-  memcpy (ioUtilGH->downsample, old_downsample, maxdim * sizeof (int));
-  free (old_downsample);
-
   /* stop total checkpoint timer and print timing info */
   if (myGH->print_timing_info)
   {
diff --git a/src/DumpVar.c b/src/DumpVar.c
index 6932d00..a27f447 100644
--- a/src/DumpVar.c
+++ b/src/DumpVar.c
@@ -9,11 +9,11 @@
  @@*/
 
 
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cctk.h"
 #include "cctk_Parameters.h"
+#include "Hyperslab.h"
 #include "CactusPUGH/PUGH/src/include/pugh.h"
 #include "CactusBase/IOUtil/src/ioGH.h"
 #include "ioFlexGH.h"
@@ -29,41 +29,24 @@ CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_DumpVar_c)
  ********************************************************************/
 /* #define IOFLEXIO_DEBUG  1 */
 
-#define IOTAGBASE 20000 /* This may break on more than 2000 processors */
-
-
-/********************************************************************
- ********************    Internal Typedefs   ************************
- ********************************************************************/
-/* info structure describing how to dump a given CCTK variable type */
-typedef struct
-{
-  int flexio_type;              /* the FLEXIO type to use */
-  int element_size;             /* size of a single data element */
-#ifdef CCTK_MPI
-  MPI_Datatype mpi_type;        /* the data type for MPI communication */
-#endif
-} dumpInfo_t;
+/* tag base for MPI messages (this may break on more than 2000 processors) */
+#define MPITAGBASE 20000
 
 
 /********************************************************************
  ********************    Internal Routines   ************************
  ********************************************************************/
-static int DumpGS (const cGH *GH, int vindex, int timelevel, IOFile file,
-                   int flexio_type);
-static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
-                   dumpInfo_t *info);
-static int GetDumpData (const cGH *GH, int vindex, int timelevel, void **outme,
-                        int *free_outme, CCTK_INT4 *geom, int element_size);
-static void EachProcDump (const cGH *GH, int vindex, int timelevel, void *outme,
-                          CCTK_INT4 *geom, IOFile file, int flexio_type);
-static void AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
+static int WriteGS (const cGH *GH, const ioRequest *request, IOFile file);
+static int WriteGA (const cGH *GH, const ioRequest *request, IOFile file);
+static void EachProcDump (const cGH *GH, const ioRequest *request, void *hdata,
+                          IOFile file);
+static void AddChunkAttributes (const cGH *GH, const ioRequest *request,
                                 IOFile file);
-static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
-                                 CCTK_INT4 *gsz, IOFile file);
+static void AddCommonAttributes (const cGH *GH, const ioRequest *request,
+                                 IOFile file);
 #ifdef CCTK_MPI
-static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
-                      CCTK_INT4 *geom, int flexio_type);
+static void ProcDump (const cGH *GH, const ioRequest *request, void *hdata,
+                      IOFile file);
 #endif
 
 
@@ -78,18 +61,13 @@ static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
    @calls
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
+   @vdesc      pointer to CCTK grid hierarchy
    @vtype      const cGH *
    @vio        in
    @endvar
-   @var        vindex
-   @vdesc      global index of the variable to be dumped
-   @vtype      int
-   @vio        in
-   @endvar
-   @var        timelevel
-   @vdesc      the timelevel to store
-   @vtype      int
+   @var        request
+   @vdesc      reference to the I/O request description
+   @vtype      const ioRequest *
    @vio        in
    @endvar
    @var        file
@@ -100,71 +78,24 @@ static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
 
    @returntype int
    @returndesc
-               -1 if variable has unknown type
-               or return code of either @seeroutine DumpGS or @seeroutine DumpGA
+               return code of either @seeroutine WriteGS or @seeroutine WriteGA
+               or -1 if variable has unknown type
    @endreturndesc
 @@*/
-int IOFlexIO_DumpVar (const cGH *GH, int vindex, int timelevel, IOFile file)
+int IOFlexIO_DumpVar (const cGH *GH, const ioRequest *request, IOFile file)
 {
-  int vtype, gtype, retval;
-  pGH *pughGH;
-  ioGH *ioUtilGH;
-  dumpInfo_t info;
+  int gtype, retval;
 
 
-  /* get the handle for PUGH and IOUtil extensions */
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-  pughGH = (pGH *) CCTK_GHExtension (GH, "PUGH");
-
-  vtype = CCTK_VarTypeI (vindex);
-
-  /* downgrade the output precision if requested */
-  if (ioUtilGH->out_single)
-  {
-    /* Do we only want to downgrade generic CCTK types ? */
-#ifdef CCTK_REAL4
-    if (vtype == CCTK_VARIABLE_REAL)
-    {
-      vtype = CCTK_VARIABLE_REAL4;
-    }
-    else if (vtype == CCTK_VARIABLE_COMPLEX)
-    {
-      vtype = CCTK_VARIABLE_COMPLEX8;
-    }
-#endif
-#ifdef CCTK_INT4
-    if (vtype == CCTK_VARIABLE_INT)
-    {
-      vtype = CCTK_VARIABLE_INT4;
-    }
-#endif
-  }
-  info.element_size = CCTK_VarTypeSize (vtype);
-#ifdef CCTK_MPI
-  info.mpi_type = PUGH_MPIDataType (pughGH, vtype);
-#endif
-  info.flexio_type = IOFlexIO_DataType (vtype);
-
-  if (info.element_size <= 0 ||
-#ifdef CCTK_MPI
-      info.mpi_type == MPI_DATATYPE_NULL ||
-#endif
-      info.flexio_type < 0)
-  {
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Unsupported variable type %d", vtype);
-    return (-1);
-  }
-
-  /* now branch to the appropriate dump routine */
-  gtype = CCTK_GroupTypeFromVarI (vindex);
+  /* branch to the appropriate dump routine */
+  gtype = CCTK_GroupTypeFromVarI (request->vindex);
   if (gtype == CCTK_SCALAR)
   {
-    retval = DumpGS (GH, vindex, timelevel, file, info.flexio_type);
+    retval = WriteGS (GH, request, file);
   }
   else if (gtype == CCTK_ARRAY || gtype == CCTK_GF)
   {
-    retval = DumpGA (GH, vindex, timelevel, file, &info);
+    retval = WriteGA (GH, request, file);
   }
   else
   {
@@ -225,7 +156,7 @@ int IOFlexIO_DataType (int cctk_type)
     case CCTK_VARIABLE_REAL16: retval = FLEXIO_REAL16; break;
 #endif
 
-    default: CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+    default: CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                          "Unsupported CCTK variable datatype %d", cctk_type);
              retval = -1;
              break;
@@ -239,7 +170,7 @@ int IOFlexIO_DataType (int cctk_type)
  ********************    Internal Routines   ************************
  ********************************************************************/
 /*@@
-   @routine    DumpGS
+   @routine    WriteGS
    @date       16 Apr 1999
    @author     Thomas Radke
    @desc
@@ -253,14 +184,9 @@ int IOFlexIO_DataType (int cctk_type)
    @vtype      const cGH *
    @vio        in
    @endvar
-   @var        vindex
-   @vdesc      global index of the variable to be dumped
-   @vtype      int
-   @vio        in
-   @endvar
-   @var        timelevel
-   @vdesc      the timelevel to store
-   @vtype      int
+   @var        request
+   @vdesc      reference to the I/O request description
+   @vtype      const ioRequest *
    @vio        in
    @endvar
    @var        file
@@ -268,71 +194,62 @@ int IOFlexIO_DataType (int cctk_type)
    @vtype      IOFile
    @vio        in
    @endvar
-   @var        flexio_type
-   @vdesc      the IOFlexIO datatype to store
-   @vtype      int
-   @vio        in
-   @endvar
 
    @returntype int
    @returndesc
                0 for success, or -1 if file handle is invalid
    @endreturndesc
 @@*/
-static int DumpGS (const cGH *GH, int vindex, int timelevel, IOFile file,
-                   int flexio_type)
+static int WriteGS (const cGH *GH, const ioRequest *request, IOFile file)
 {
-  ioGH *ioUtilGH;
-  CCTK_INT gsz[] = {0, 0, 0};    /* not needed here ? */
-  int dim[] = {1};               /* size of scalar */
+  /* const */ int dim = 1;
+  const ioGH *ioUtilGH;
 
 
-  /* immediately return if file handle is invalid */
-  if (! IOisValid (file))
-  {
-    return (-1);
-  }
-
-  /* get the handle for IOUtil extensions */
+  /* only I/O processors write data */
   ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-
   if (CCTK_MyProc (GH) != ioUtilGH->ioproc)
   {
     return (0);
   }
 
+  /* check if file handle is invalid */
+  if (! IOisValid (file))
+  {
+    return (-1);
+  }
+
   /* first dump the data then add the attributes */
-  FLEXIO_ERROR (IOwrite (file, flexio_type, 1, dim,
-                         CCTK_VarDataPtrI (GH, timelevel, vindex)));
-  AddCommonAttributes (GH, vindex, timelevel, gsz, file);
+  FLEXIO_ERROR (IOwrite (file, IOFlexIO_DataType(request->hdatatype), dim, &dim,
+                         CCTK_VarDataPtrI (GH, request->timelevel,
+                                           request->vindex)));
+  AddCommonAttributes (GH, request, file);
 
   return (0);
 }
 
 
 /*@@
-   @routine    DumpGA
+   @routine    WriteGA
    @date       July 1998
    @author     Paul Walker
    @desc
                Dumps a grid array variable into a IEEEIO file.
    @enddesc
 
-   @calls      GetDumpData
+   @calls      Hyperslab_DefineLocalMappingByIndex
+               Hyperslab_FreeMapping
+               Hyperslab_Get
+               WriteData
 
    @var        GH
-   @vdesc      Pointer to CCTK grid hierarchy
+   @vdesc      pointer to CCTK grid hierarchy
    @vtype      const cGH *
    @vio        in
    @endvar
-   @var        vindex
-   @vdesc      global index of the variable to be dumped
-   @vtype      int
-   @vio        in
-   @endvar
-   @var        timelevel
-   @vdesc      the timelevel to store
-   @vtype      int
+   @var        request
+   @vdesc      reference to the I/O request description
+   @vtype      const ioRequest *
    @vio        in
    @endvar
    @var        file
@@ -340,157 +257,176 @@ static int DumpGS (const cGH *GH, int vindex, int timelevel, IOFile file,
    @vtype      IOFile
    @vio        in
    @endvar
-   @var        info
-   @vdesc      info structure describing
-                 - the IOFlexIO datatype to store
-                 - the size of an element of variable
-                 - the MPI datatype to communicate
-   @vtype      dumpInfo_t
-   @vio        in
-   @endvar
 
    @returntype int
    @returndesc
-               0 for success
-               or returncode of @seeroutine GetDumpData
+                0 for success, or<BR>
+               -1 if hyperslab mapping couldn't be defined, or<BR>
+               -2 if hyperslab couldn't be extracted
    @endreturndesc
 @@*/
-static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
-                   dumpInfo_t *info)
+static int WriteGA (const cGH *GH, const ioRequest *request, IOFile file)
 {
-  int myproc, nprocs, dim, free_outme, retval;
-  pGH *pughGH;
-  ioGH *ioUtilGH;
-  CCTK_INT4 *geom;
-  void *outme;
+  int i, myproc, mapping, hdatasize, retval;
+  void *hdata;
+  char *fullname;
+  const ioGH *ioUtilGH;
 #ifdef CCTK_MPI
-  int i, j, incoming, outgoing;
-  int *local_size;
-  MPI_Status ms;
+  int nprocs;
+  int *hsize_chunk;
   void *tmpd;
+  MPI_Comm comm;
+  MPI_Status ms;
+  MPI_Datatype mpitype;
 #endif
   DECLARE_CCTK_PARAMETERS
 
 
-  /* get the handles for PUGH and IO extensions */
-  pughGH = PUGH_pGH (GH);
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
+  /* define the hyperslab mapping */
+  mapping = Hyperslab_DefineLocalMappingByIndex (GH, request->vindex,
+                                                 request->hdim,
+                                                 request->direction,
+                                                 request->origin,
+                                                 request->extent,
+                                                 request->downsample,
+                                                 -1, NULL,
+                                                 request->hsize_chunk,
+                                                 request->hsize,
+                                                 request->hoffset);
+  if (mapping < 0)
+  {
+    fullname = CCTK_FullName (request->vindex);
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Failed to define hyperslab mapping for variable '%s'",
+                fullname);
+    free (fullname);
+    return (-1);
+  }
 
-  myproc = CCTK_MyProc (GH);
-  nprocs = CCTK_nProcs (GH);
+  /* calculate the size of the hyperslab */
+  request->hsize_chunk[request->hdim] = 1;
+  for (i = 0; i < request->hdim; i++)
+  {
+    request->hsize_chunk[request->hdim] *= request->hsize_chunk[i];
+  }
 
-  /* get the dimension of the variable */
-  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
+  /* get the hyperslab */
+  hdatasize = CCTK_VarTypeSize (request->hdatatype);
+  hdata = request->hsize_chunk[request->hdim] > 0 ?
+          malloc (request->hsize_chunk[request->hdim] * hdatasize) : NULL;
+  retval = Hyperslab_Get (GH, mapping, -1, request->vindex, request->timelevel,
+                          request->hdatatype, hdata);
 
-  /* allocate the geometry buffer */
-  geom = (CCTK_INT4 *) malloc (3*dim * sizeof (CCTK_INT4));
+  /* release the mapping structure */
+  Hyperslab_FreeMapping (mapping);
 
-  /* get the pointer to the data we want to output (includes downsampling) */
-  retval = GetDumpData (GH, vindex, timelevel, &outme, &free_outme, geom,
-                        info->element_size);
-  if (retval < 0)
+  if (retval)
   {
-    return (retval);
+    fullname = CCTK_FullName (request->vindex);
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Failed to extract hyperslab for variable '%s'", fullname);
+    free (fullname);
+    if (hdata)
+    {
+      free (hdata);
+    }
+    return (-2);
   }
 
+  /* dump data held on I/O processor */
+  myproc = CCTK_MyProc (GH);
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
   if (ioUtilGH->ioproc_every == 1)
   {
-    EachProcDump (GH, vindex, timelevel, outme, geom, file, info->flexio_type);
+    EachProcDump (GH, request, hdata, file);
   }
 #ifdef CCTK_MPI
-  else if (myproc != ioUtilGH->ioproc)
+  else
   {
-    /*
-     * Send the geometry and data from non-IO processors to the IO processors
-     */
-    outgoing = 1;
-    for (i = 0; i < dim; i++)
-    {
-      outgoing *= geom[dim + i];
-    }
+    nprocs = CCTK_nProcs (GH);
+    comm = PUGH_pGH (GH)->PUGH_COMM_WORLD;
+    mpitype = PUGH_MPIDataType (PUGH_pGH (GH), request->hdatatype);
 
-    /* send geometry */
-    CACTUS_MPI_ERROR (MPI_Send (geom, 3*dim, PUGH_MPI_INT4, ioUtilGH->ioproc,
-                                2*myproc+IOTAGBASE+1, pughGH->PUGH_COMM_WORLD));
-    /* send data */
-    if (outgoing > 0)
-    {
-      CACTUS_MPI_ERROR (MPI_Send (outme, outgoing, info->mpi_type,
-                                  ioUtilGH->ioproc, 2*myproc+IOTAGBASE,
-                                  pughGH->PUGH_COMM_WORLD));
-#ifdef IOFLEXIO_DEBUG
-      printf ("Processor %d sent %d data points\n", myproc, outgoing);
-#endif
-    }
-  }
-  else if (myproc == ioUtilGH->ioproc)
-  {
-    if (ioUtilGH->unchunked)
+    if (myproc == ioUtilGH->ioproc)
     {
-      local_size = (int *) malloc (dim * sizeof (int));
-
-      /* first calculate the local size and reserve the chunk */
-      if (ioUtilGH->ioproc_every >= nprocs)
+      if (ioUtilGH->unchunked)
       {
-        /* one output file ... the local chunk size is the global size */
-        for (i = 0; i < dim; i++)
+        /* need to copy CCTK_INT[] to int[] */
+        hsize_chunk = (int *) malloc (request->hdim * sizeof (int));
+
+        /* first calculate the local size and reserve the chunk */
+        for (i = 0; i < request->hdim; i++)
         {
-          local_size[i] = geom[2*dim + i];
+          hsize_chunk[i] = request->hsize_chunk[i];
         }
-      }
-
-      IOreserveChunk (file, info->flexio_type, dim, local_size);
 
-      free (local_size);
-    }
+        IOreserveChunk (file, IOFlexIO_DataType (request->hdatatype),
+                        request->hdim, hsize_chunk);
 
-    /* write the data first then the attributes */
-    ProcDump (file, GH, vindex, outme, geom, info->flexio_type);
-    /* delay adding attributes for unchunked files
-       until all chunks were written (otherwise attributes get lost !) */
-    if (! ioUtilGH->unchunked)
-    {
-      AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
-    }
-
-    /* dump data from all other processors */
-    for (i = myproc+1; i < myproc+ioUtilGH->ioproc_every && i < nprocs; i++)
-    {
-      /* receive geometry */
-      CACTUS_MPI_ERROR (MPI_Recv (geom, 3*dim, PUGH_MPI_INT4, i,2*i+IOTAGBASE+1,
-                                  pughGH->PUGH_COMM_WORLD, &ms));
-
-      incoming = 1;
-      for (j = 0; j < dim; j++)
-      {
-        incoming *= geom[dim + j];
+        free (hsize_chunk);
       }
 
-      /* receive data */
-      if (incoming > 0)
+      /* write the data first then the attributes */
+      ProcDump (GH, request, hdata, file);
+      /* delay adding attributes for unchunked files
+         until all chunks were written (otherwise attributes get lost !) */
+      if (! ioUtilGH->unchunked)
       {
-        tmpd = malloc (incoming * info->element_size);
-        CACTUS_MPI_ERROR (MPI_Recv (tmpd, incoming, info->mpi_type, i,
-                                    2*i+IOTAGBASE,pughGH->PUGH_COMM_WORLD,&ms));
+        AddCommonAttributes (GH, request, file);
       }
-      else
+
+      /* dump data from all other processors */
+      for (i = myproc+1; i < myproc+ioUtilGH->ioproc_every && i < nprocs; i++)
       {
+        /* receive geometry (this assumes the geometry arrays
+           to be contiguous starting at hoffset */
+        CACTUS_MPI_ERROR (MPI_Recv (request->hoffset, 3*request->hdim + 1,
+                                    PUGH_MPI_INT, i, 2*i + MPITAGBASE + 1,
+                                    comm, &ms));
+
+        /* receive data */
         tmpd = NULL;
-      }
+        if (request->hsize_chunk[request->hdim] > 0)
+        {
+          tmpd = malloc (request->hsize_chunk[request->hdim] * hdatasize);
+          CACTUS_MPI_ERROR (MPI_Recv (tmpd, request->hsize_chunk[request->hdim],
+                                      mpitype, i, 2*i + MPITAGBASE, comm, &ms));
+        }
+
+        /* write data */
+        ProcDump (GH, request, tmpd, file);
+
+        if (tmpd)
+        {
+          free (tmpd);
+        }
 
-      /* write data */
-      ProcDump (file, GH, vindex, tmpd, geom, info->flexio_type);
+      } /* end loop over processors */
 
-      if (tmpd)
+      /* now add the attributes for unchunked files */
+      if (ioUtilGH->unchunked)
       {
-        free (tmpd);
+        AddCommonAttributes (GH, request, file);
       }
-    } /* end loop over processors */
-
-    /* now add the attributes for unchunked files */
-    if (ioUtilGH->unchunked)
+    }
+    else
     {
-      AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
+      /* send geometry (this assumes the geometry arrays to be contiguous
+         starting at hoffset) */
+      CACTUS_MPI_ERROR (MPI_Send (request->hoffset, 3*request->hdim + 1,
+                                  PUGH_MPI_INT, ioUtilGH->ioproc,
+                                  2*myproc + MPITAGBASE + 1, comm));
+      /* send data */
+      if (request->hsize_chunk[request->hdim] > 0)
+      {
+        CACTUS_MPI_ERROR (MPI_Send (hdata, request->hsize_chunk[request->hdim],
+                                    mpitype, ioUtilGH->ioproc,
+                                    2*myproc + MPITAGBASE, comm));
+#ifdef IOFLEXIO_DEBUG
+        printf ("Processor %d sent %d data points\n",
+                myproc, request->hsize_chunk[request->hdim]);
+#endif
+      }
     }
   }
 
@@ -498,13 +434,12 @@ static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
   CCTK_Barrier (GH);
 #endif /* MPI */
 
-  if (free_outme)
+  /* free allocated resources */
+  if (hdata)
   {
-    free (outme);
+    free (hdata);
   }
 
-  free (geom);
-
   return (retval);
 }
 
@@ -539,42 +474,38 @@ static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
                chunked/unchunked output files
    @endhistory
 @@*/
-static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
-                                 CCTK_INT4 *gsz, IOFile file)
+static void AddCommonAttributes (const cGH *GH, const ioRequest *request,
+                                 IOFile file)
 {
-  int i, vdim;
+  int i;
   CCTK_REAL *dtmp;
-  CCTK_INT4 itmp;
+  CCTK_INT4 *itmp;
   char *name;
   ioGH *ioUtilGH;
   char coord_system_name[20];
   DECLARE_CCTK_PARAMETERS
 
 
-  /* get the handle for IO extensions */
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-
-  /* get the dimension of the variable */
-  vdim = CCTK_GroupDimFromVarI (vindex);
+  itmp = (CCTK_INT4 *) malloc (request->hdim * sizeof (CCTK_INT4));
 
-  name = CCTK_FullName (vindex);
+  name = CCTK_FullName (request->vindex);
   FLEXIO_ERROR (IOwriteAttribute (file, "name", FLEXIO_CHAR,
                                   strlen (name) + 1, name));
   free (name);
 
-  name = CCTK_GroupNameFromVarI (vindex);
+  name = CCTK_GroupNameFromVarI (request->vindex);
   FLEXIO_ERROR (IOwriteAttribute (file, "groupname", FLEXIO_CHAR,
                                   strlen (name) + 1, name));
   free (name);
 
-  itmp = CCTK_GroupTypeFromVarI (vindex);
-  FLEXIO_ERROR (IOwriteAttribute (file, "grouptype", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = CCTK_GroupTypeFromVarI (request->vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "grouptype", FLEXIO_INT4, 1, itmp));
 
-  itmp = CCTK_NumTimeLevelsFromVarI (vindex);
-  FLEXIO_ERROR (IOwriteAttribute (file, "ntimelevels", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = CCTK_NumTimeLevelsFromVarI (request->vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "ntimelevels", FLEXIO_INT4, 1, itmp));
 
-  itmp = timelevel;
-  FLEXIO_ERROR (IOwriteAttribute (file, "timelevel", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = request->timelevel;
+  FLEXIO_ERROR (IOwriteAttribute (file, "timelevel", FLEXIO_INT4, 1, itmp));
 
   FLEXIO_ERROR (IOwriteAttribute (file, "time", FLEXIO_REAL, 1,&GH->cctk_time));
 
@@ -584,49 +515,60 @@ static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
   /* FIXME: This is hardcoded for cartesian coordinate systems.
             A better solution would be to be able to query the coordinate
             system which is associated with the variable. */
-  sprintf (coord_system_name, "cart%dd", vdim);
-  if (CCTK_GroupTypeFromVarI (vindex) == CCTK_GF &&
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
+  sprintf (coord_system_name, "cart%dd", request->hdim);
+  if (CCTK_GroupTypeFromVarI (request->vindex) == CCTK_GF &&
       CCTK_CoordSystemHandle (coord_system_name) >= 0)
   {
-    dtmp = (CCTK_REAL *) malloc (3 * vdim * sizeof (CCTK_REAL));
-    for (i = 0; i < vdim; i++)
+    dtmp = (CCTK_REAL *) malloc (3 * request->hdim * sizeof (CCTK_REAL));
+    for (i = 0; i < request->hdim; i++)
     {
-      CCTK_CoordRange (GH, &dtmp[i], &dtmp[i + vdim], i + 1,
+      CCTK_CoordRange (GH, &dtmp[i], &dtmp[i + request->hdim], i + 1,
                        NULL, coord_system_name);
-      dtmp[i + 2*vdim] = GH->cctk_delta_space[i] * ioUtilGH->downsample[i];
+      dtmp[i + 2*request->hdim] = GH->cctk_delta_space[i] *
+                                  ioUtilGH->downsample[i];
     }
 
-    FLEXIO_ERROR (IOwriteAttribute (file, "origin", FLEXIO_REAL, vdim, dtmp));
-    FLEXIO_ERROR (IOwriteAttribute (file, "min_ext", FLEXIO_REAL, vdim, dtmp));
-    FLEXIO_ERROR (IOwriteAttribute (file, "max_ext", FLEXIO_REAL, vdim,
-                                    dtmp + vdim));
-    FLEXIO_ERROR (IOwriteAttribute (file, "delta", FLEXIO_REAL, vdim,
-                                    dtmp + 2*vdim));
+    FLEXIO_ERROR (IOwriteAttribute (file, "origin", FLEXIO_REAL, request->hdim,
+                                    dtmp));
+    FLEXIO_ERROR (IOwriteAttribute (file, "min_ext", FLEXIO_REAL, request->hdim,
+                                    dtmp));
+    FLEXIO_ERROR (IOwriteAttribute (file, "max_ext", FLEXIO_REAL, request->hdim,
+                                    dtmp + request->hdim));
+    FLEXIO_ERROR (IOwriteAttribute (file, "delta", FLEXIO_REAL, request->hdim,
+                                    dtmp + 2*request->hdim));
 
     if (ioUtilGH->downsample[0] > 1 ||
         ioUtilGH->downsample[1] > 1 ||
         ioUtilGH->downsample[2] > 1)
     {
       FLEXIO_ERROR (IOwriteAttribute (file, "evolution_delta", FLEXIO_REAL,
-                                      vdim, GH->cctk_delta_space));
+                                      request->hdim, GH->cctk_delta_space));
     }
 
     free (dtmp);
   }
 
-  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4, vdim, gsz));
+  for (i = 0; i < request->hdim; i++)
+  {
+    itmp[i] = request->hsize[i];
+  }
+  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4,
+                                  request->hdim, itmp));
+
+  itmp[0] = CCTK_nProcs (GH);
+  FLEXIO_ERROR (IOwriteAttribute (file, "nprocs", FLEXIO_INT4, 1, itmp));
 
-  itmp = CCTK_nProcs (GH);
-  FLEXIO_ERROR (IOwriteAttribute (file, "nprocs", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = ioUtilGH->ioproc_every;
+  FLEXIO_ERROR (IOwriteAttribute (file, "ioproc_every", FLEXIO_INT4, 1, itmp));
 
-  itmp = ioUtilGH->ioproc_every;
-  FLEXIO_ERROR (IOwriteAttribute (file, "ioproc_every", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = ioUtilGH->unchunked;
+  FLEXIO_ERROR (IOwriteAttribute (file, "unchunked", FLEXIO_INT4, 1, itmp));
 
-  itmp = ioUtilGH->unchunked;
-  FLEXIO_ERROR (IOwriteAttribute (file, "unchunked", FLEXIO_INT4, 1, &itmp));
+  itmp[0] = GH->cctk_iteration;
+  FLEXIO_ERROR (IOwriteAttribute (file, "iteration", FLEXIO_INT4, 1, itmp));
 
-  itmp = GH->cctk_iteration;
-  FLEXIO_ERROR (IOwriteAttribute (file, "iteration", FLEXIO_INT4, 1, &itmp));
+  free (itmp);
 }
 
 
@@ -648,12 +590,12 @@ static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
   @hdesc       Abstracted WRITE_ATTRIBUTE to merge in Szu-Wen's Panda calls.
   @endhistory
 @@*/
-static void AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
+static void AddChunkAttributes (const cGH *GH, const ioRequest *request,
                                 IOFile file)
 {
-  int vdim;
-  char *name;
-  CCTK_INT4 itmp;
+  int i;
+  char *fullname;
+  CCTK_INT4 *itmp;
 
 
   /* there is nothing to do for a serial run */
@@ -662,293 +604,30 @@ static void AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
     return;
   }
 
-  /* get the dimension of the variable */
-  vdim = CCTK_GroupDimFromVarI (vindex);
-
-  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_origin", FLEXIO_INT4, vdim,
-                                  &geom[0]));
-  FLEXIO_ERROR (IOwriteAttribute (file, "subchunk_lb", FLEXIO_INT4, vdim,
-                                  &geom[0]));
-  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4, vdim,
-                                  &geom[2*vdim]));
-
-  itmp = GH->cctk_iteration;
-  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_dataset", FLEXIO_INT4, 1,&itmp));
-
-  name = CCTK_FullName (vindex);
-  FLEXIO_ERROR (IOwriteAttribute (file, "name", FLEXIO_CHAR,
-                                  strlen (name)+1, name));
-  free (name);
-}
-
-
-/*@@
-  @routine GetDumpData
-  @author  Paul Walker
-  @date    Feb 1997
-  @desc
-  Bounds and data to be output, takes into account downsampling
-  @enddesc
-  @history
-  @hauthor Gabrielle Allen @hdate Oct 5 1998
-  @hdesc Made into subroutine
-  @endhistory
-  @var     GH
-  @vdesc   Identifies grid hierachy
-  @vtype   const cGH *
-  @vio     in
-  @vcomment
-  @endvar
-  @var     vindex
-  @vdesc   index of the variable to dump
-  @vtype   int
-  @vio     in
-  @vcomment
-  @endvar
-  @var     timelevel
-  @vdesc   timelevel of the variable to dump
-  @vtype   int
-  @vio     in
-  @vcomment
-  @endvar
-  @var     outme
-  @vdesc   data segment to output
-  @vtype   void **
-  @vio     out
-  @vcomment This is just GF->data if there is no downsampling
-  @endvar
-  @var     free_outme
-  @vdesc   Specifies whether or not to free the storage associated with outme
-  @vtype   int *
-  @vio     out
-  @vcomment 1: free storage; 0: don't free storage
-  @endvar
-  @var     geom
-  @vdesc   bounds, local size and global shape of the output
-  @vtype   CCTK_INT4[3*dim]
-  @vio     out
-  @vcomment geom[0*dim..1*dim-1] lower bounds and geom[3],geom[4],geom[5]
-            geom[1*dim..2*dim-1] local size of data to send
-            geom[2*dim..3*dim-1] global shape
-  @endvar
-  @var     element_size
-  @vdesc   the size of an element of variable
-  @vtype   int
-  @vio     in
-  @endvar
-@@*/
-static int GetDumpData (const cGH *GH, int vindex, int timelevel, void **outme,
-                        int *free_outme, CCTK_INT4 *geom, int element_size)
-{
-  DECLARE_CCTK_PARAMETERS
-  int i, myproc, do_downsample, dim;
-  ioGH *ioUtilGH;
-  pGExtras *extras;
-  CCTK_REAL4 *single_ptr;
-  CCTK_REAL *real_ptr;
-  CCTK_BYTE *char_ptr;
-  CCTK_INT *int_ptr;
-  void *data = CCTK_VarDataPtrI (GH, timelevel, vindex);
-
-
-  /* to make the compiler happy */
-  single_ptr = NULL;
-  real_ptr = NULL;
-  char_ptr = NULL;
-  int_ptr = NULL;
-
-  myproc = CCTK_MyProc (GH);
+  itmp = (CCTK_INT4 *) malloc (2 * request->hdim * sizeof (CCTK_INT4));
 
-  /* get GH extensions for IO */
-  ioUtilGH = (ioGH *) GH->extensions[CCTK_GHExtensionHandle ("IO")];
-
-  /* get the pGExtras pointer as a shortcut */
-  extras = ((pGA ***)PUGH_pGH (GH)->variables)[vindex][timelevel]->extras;
-
-  /* get the dimension of the variable */
-  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
-
-  do_downsample = 0;
-  for (i = 0; i < dim; i++)
-    do_downsample |= ioUtilGH->downsample[i] > 1;
-
-  /* All the downsampling code is hard-coded for 3D data.
-     For other-dimensional variables we print a warning
-     and return the non-downsampled data. */
-  if (do_downsample && dim != 3)
+  /* copy from CCTK_INT[] to CCTK_INT4[] */
+  for (i = 0; i < request->hdim; i++)
   {
-    char *fullname = CCTK_FullName (vindex);
-
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "No downsampling for %dD variable '%s' ! Downsampling is only "
-                "supported for 3D variables.", dim, fullname);
-    free (fullname);
-    do_downsample = 0;
+    itmp[0*request->hdim + i] = request->hoffset[i];
+    itmp[1*request->hdim + i] = request->hsize[i];
   }
 
-  /* Simple case if no downsampling */
-  if (! do_downsample)
-  {
-
-    if (ioUtilGH->out_single)
-    {
-      single_ptr = (CCTK_REAL4 *) malloc (extras->npoints*sizeof (CCTK_REAL4));
-
-      for (i = 0; i < extras->npoints; i++)
-      {
-        single_ptr[i] = (CCTK_REAL4) ((CCTK_REAL *) data)[i];
-      }
-
-      *outme = single_ptr;
-      *free_outme = 1;
-    }
-    else
-    {
-      *outme = data;
-      *free_outme = 0;
-    }
-
-    for (i = 0; i < dim; i++)
-    {
-      geom[i + 0*dim] = extras->lb[myproc][i];;     /* the bounds */
-      geom[i + 1*dim] = extras->lnsize[i];          /* the sizes */
-      geom[i + 2*dim] = extras->nsize[i];           /* the global space */
-    }
-
-  } else {
-
-    /* NOTE: the following downsampling code is hard-coded for 3D data */
-
-    int start[3], end[3];
-    int j, k, l;
+  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_origin", FLEXIO_INT4,
+                                  request->hdim, &itmp[0]));
+  FLEXIO_ERROR (IOwriteAttribute (file, "subchunk_lb", FLEXIO_INT4,
+                                  request->hdim, &itmp[0]));
+  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4,
+                                  request->hdim, &itmp[request->hdim]));
 
-    /* Downsampling code ... */
-    for (i = 0; i < 3; i++) {
-      geom[i + 6] = extras->nsize[i] / ioUtilGH->downsample[i];
-      if (extras->nsize[i] % ioUtilGH->downsample[i])
-        geom[i + 6]++;
-    }
-
-    if (verbose)
-      CCTK_VInfo (CCTK_THORNSTRING, "Downsampled sizes (%d, %d, %d) -> "
-                  "(%d, %d, %d)",
-                  extras->nsize[0], extras->nsize[1], extras->nsize[2],
-                  (int) geom[6], (int) geom[7], (int) geom[8]);
-
-    /* Now figure out the local downsampling */
-    /* The local starts are the lb modded into the downsample */
-    for (i = 0; i < 3; i++) {
-      geom[i]   = extras->lb[myproc][i] / ioUtilGH->downsample[i];
-      start[i] = geom[i] * ioUtilGH->downsample[i];
-      if (start[i] <
-          extras->lb[myproc][i] + extras->ownership[PUGH_NO_STAGGER][0][i]) {
-        start[i] += ioUtilGH->downsample[i];
-        geom[i] ++;
-      }
-      end[i]   = ((extras->lb[myproc][i] +
-                    extras->ownership[PUGH_NO_STAGGER][1][i] - 1) /
-                   ioUtilGH->downsample[i]) * ioUtilGH->downsample[i];
-      geom[i+3] = (end[i] - start[i]) / ioUtilGH->downsample[i] + 1;
-    }
-
-    if (verbose) {
-      CCTK_VInfo (CCTK_THORNSTRING, "Downsample ranges (%d, %d, %d) -> "
-                  "(%d, %d, %d)", start[0], start[1], start[2],
-                  end[0], end[1], end[2]);
-      CCTK_VInfo (CCTK_THORNSTRING, "Local size/bound  (%d, %d, %d)  "
-                  "(%d, %d, %d)",
-                  (int) geom[3], (int) geom[4], (int) geom[5],
-                  (int) geom[0], (int) geom[1], (int) geom[2]);
-    }
-
-    /* compute local ranges */
-    for (i = 0; i < 3; i++) {
-      start[i] -= extras->lb[myproc][i];
-      end[i] -= extras->lb[myproc][i];
-    }
-
-    *outme = malloc (geom[3] * geom[4] * geom[5] * element_size);
-    *free_outme = 1;
-
-    /* I hate it to repeat the loops for each case label
-       but that way produces much more efficient code */
-    l = 0;
-    switch (CCTK_VarTypeI (vindex)) {
-      case CCTK_VARIABLE_CHAR:
-        char_ptr = (CCTK_BYTE *) *outme;
-        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
-          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
-            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
-              char_ptr[l++] = ((CCTK_BYTE *) data)[DATINDEX (extras, i, j, k)];
-        break;
-
-      case CCTK_VARIABLE_INT:
-        int_ptr = (CCTK_INT *) *outme;
-        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
-          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
-            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
-              int_ptr[l++] = ((CCTK_INT *) data)[DATINDEX (extras, i, j, k)];
-        break;
-
-      case CCTK_VARIABLE_REAL:
-        if (ioUtilGH->out_single)
-          single_ptr = (CCTK_REAL4 *) *outme;
-        else
-          real_ptr = (CCTK_REAL *) *outme;
-        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
-          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
-            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
-              if (ioUtilGH->out_single)
-                single_ptr[l++] = (CCTK_REAL4)
-                      (((CCTK_REAL *) data)[DATINDEX (extras, i, j, k)]);
-              else
-                real_ptr[l++] =
-                      ((CCTK_REAL *) data)[DATINDEX (extras, i, j, k)];
-        break;
-
-#if 0
-/* FIXME: Don't know how to support COMPLEX type too !! */
-      case CCTK_VARIABLE_COMPLEX:
-        if (ioUtilGH->out_single)
-          single_ptr = (CCTK_REAL4 *) *outme;
-        else
-          cmplx_ptr = (CCTK_COMPLEX *) *outme;
-        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
-          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
-            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
-              if (ioUtilGH->out_single) {
-                single_ptr[l++] = (CCTK_REAL4)
-                       (((CCTK_REAL *) data)[0 + DATINDEX (extras, i, j, k)]);
-                single_ptr[l++] = (CCTK_REAL4)
-                       (((CCTK_REAL *) data)[1 + DATINDEX (extras, i, j, k)]);
-              } else
-                cmplx_ptr[l++] =
-                       ((CCTK_COMPLEX *) data)[DATINDEX (extras, i, j, k)];
-        break;
-#endif
-
-      default:
-        CCTK_WARN (1, "Unsupported variable type in GetDumpData");
-        return (-1);
-    }
-  }
-
-#ifdef IOFLEXIO_DEBUG
-  printf ("Lower bound: %d", (int) geom[0]);
-  for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom[i]);
-  printf ("\n");
-  printf ("Chunk size : %d", (int) geom[1*dim + 0]);
-  for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom[1*dim + i]);
-  printf ("\n");
-  printf ("Global size: %d", (int) geom[2*dim + 0]);
-  for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom[2*dim + i]);
-  printf ("\n");
-#endif
+  itmp[0] = GH->cctk_iteration;
+  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_dataset", FLEXIO_INT4, 1, itmp));
 
-  return (0);
+  fullname = CCTK_FullName (request->vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "name", FLEXIO_CHAR,
+                                  strlen (fullname)+1, fullname));
+  free (fullname);
+  free (itmp);
 }
 
 
@@ -965,31 +644,28 @@ static int GetDumpData (const cGH *GH, int vindex, int timelevel, void **outme,
   @hdesc       Made into subroutine
   @endhistory
 @@*/
-static void EachProcDump (const cGH *GH, int vindex, int timelevel, void *outme,
-                          CCTK_INT4 *geom, IOFile file, int flexio_type)
+static void EachProcDump (const cGH *GH, const ioRequest *request, void *hdata,
+                          IOFile file)
 {
-  int i, dim, *chunk_dims;
+  int i, *chunk_dims;
 
 
-  /* get the dimension of the variable */
-  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
-
-  chunk_dims = (int *) malloc (dim * sizeof (int));
-
-  /* set up the local shape */
-  for (i = 0; i < dim; i++)
+  /* copy from CCTK_INT[] to int[] */
+  chunk_dims = (int *) malloc (request->hdim * sizeof (int));
+  for (i = 0; i < request->hdim; i++)
   {
-    chunk_dims[i] = geom[dim + i];
+    chunk_dims[i] = request->hsize_chunk[i];
   }
 
   /* dump the data */
-  FLEXIO_ERROR (IOwrite (file, flexio_type, dim, chunk_dims, outme));
+  FLEXIO_ERROR (IOwrite (file, IOFlexIO_DataType (request->hdatatype),
+                         request->hdim, chunk_dims, hdata));
 
   /* add attributes for global space */
-  AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
+  AddCommonAttributes (GH, request, file);
 
   /* add chunk attributes */
-  AddChunkAttributes (GH, vindex, geom, file);
+  AddChunkAttributes (GH, request, file);
 
   free (chunk_dims);
 }
@@ -1009,38 +685,36 @@ static void EachProcDump (const cGH *GH, int vindex, int timelevel, void *outme,
   @endhistory
 @@*/
 #ifdef CCTK_MPI
-static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
-                      CCTK_INT4 *geom, int flexio_type)
+static void ProcDump (const cGH *GH, const ioRequest *request, void *hdata,
+                      IOFile file)
 {
   ioGH *ioUtilGH;
-  int i, dim, *chunk_dims, *chunk_origin;
-
+  int i, *chunk_dims, *chunk_origin;
 
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-
-  /* get the dimension of the variable */
-  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
 
-  chunk_origin = (int *) malloc (2*dim * sizeof (int));
-  chunk_dims = chunk_origin + dim;
-  for (i = 0; i < dim; i++)
+  /* copy from CCTK_INT[] to int[] */
+  chunk_origin = (int *) malloc (2*request->hdim * sizeof (int));
+  chunk_dims = chunk_origin + request->hdim;
+  for (i = 0; i < request->hdim; i++)
   {
-    chunk_origin[i] = geom[i];
-    chunk_dims[i] = geom[dim + i];
+    chunk_origin[i] = request->hoffset[i];
+    chunk_dims[i] = request->hsize_chunk[request->hdim + i];
   }
 
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
   if (ioUtilGH->unchunked)
   {
     /* unchunked data */
-    FLEXIO_ERROR (IOwriteChunk (file, chunk_dims, chunk_origin, outme));
+    FLEXIO_ERROR (IOwriteChunk (file, chunk_dims, chunk_origin, hdata));
   }
   else
   {
-    /* uhunked data */
-    FLEXIO_ERROR (IOwrite (file, flexio_type, dim, chunk_dims, outme));
+    /* chunked data */
+    FLEXIO_ERROR (IOwrite (file, IOFlexIO_DataType (request->hdatatype),
+                           request->hdim, chunk_dims, hdata));
 
     /* write chunk attributes */
-    AddChunkAttributes (GH, vindex, geom, file);
+    AddChunkAttributes (GH, request, file);
   }
 
   free (chunk_origin);
diff --git a/src/Output.c b/src/Output.c
index 1b67d68..776a473 100644
--- a/src/Output.c
+++ b/src/Output.c
@@ -24,9 +24,7 @@ CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_Output_c)
 /********************************************************************
  ********************    Internal Routines   ************************
  ********************************************************************/
-static int CheckOutputVar (int vindex);
-static void CheckSteerableParameters (flexioGH *myGH);
-static void SetOutputFlag (int vindex, const char *optstring, void *arg);
+static void CheckSteerableParameters (const cGH *GH);
 
 
 /*@@
@@ -37,11 +35,12 @@ static void SetOutputFlag (int vindex, const char *optstring, void *arg);
                Loops over all variables and outputs them if necessary
    @enddesc
 
-   @calls      IOFlexIO_TimeFor
+   @calls      CheckSteerableParameters
+               IOFlexIO_TimeFor
                IOFlexIO_OutputVarAs
 
    @var        GH
-   @vdesc      Pointer to CCTK GH
+   @vdesc      pointer to CCTK GH
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -55,17 +54,15 @@ static void SetOutputFlag (int vindex, const char *optstring, void *arg);
 int IOFlexIO_OutputGH (const cGH *GH)
 {
   int vindex, retval;
-  flexioGH *myGH;
+  const flexioGH *myGH;
   char *fullname;
   const char *name;
   DECLARE_CCTK_PARAMETERS
 
 
-  /* get the GH extension for IOFlexIO */
-  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
-
-  CheckSteerableParameters (myGH);
+  CheckSteerableParameters (GH);
 
+  myGH = (const flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
   if (myGH->out_every <= 0)
   {
     return (0);
@@ -87,7 +84,7 @@ int IOFlexIO_OutputGH (const cGH *GH)
 
       if (IOFlexIO_OutputVarAs (GH, fullname, name) == 0)
       {
-        /* Register variables as having output this iteration */
+        /* register variables as having output this iteration */
         myGH->out_last[vindex] = GH->cctk_iteration;
         retval++;
       }
@@ -108,10 +105,12 @@ int IOFlexIO_OutputGH (const cGH *GH)
                Unconditional output of a variable using the IOFlexIO I/O method.
    @enddesc
 
-   @calls      IOFlexIO_Write
+   @calls      IOUtil_ParseVarsForOutput
+               IOFlexIO_Write
+               IOUtil_FreeIORequest
 
    @var        GH
-   @vdesc      Pointer to CCTK GH
+   @vdesc      pointer to CCTK GH
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -134,7 +133,8 @@ int IOFlexIO_OutputGH (const cGH *GH)
 @@*/
 int IOFlexIO_OutputVarAs (const cGH *GH, const char *fullname, const char*alias)
 {
-  int vindex, retval;
+  int vindex, oneshot, retval;
+  const flexioGH *myGH;
   DECLARE_CCTK_PARAMETERS
 
 
@@ -146,9 +146,22 @@ int IOFlexIO_OutputVarAs (const cGH *GH, const char *fullname, const char*alias)
                 "index) = (%s, %s, %d)", fullname, alias, vindex);
   }
 
+  /* check whether the variable already has an I/O request entry */
+  myGH = (const flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
+  oneshot = myGH->requests[vindex] == NULL;
+  if (oneshot)
+  {
+    IOUtil_ParseVarsForOutput (GH, fullname, myGH->requests);
+  }
+
   /* do the output */
   retval = IOFlexIO_Write (GH, vindex, alias);
 
+  if (oneshot && myGH->requests[vindex])
+  {
+    IOUtil_FreeIORequest (&myGH->requests[vindex]);
+  }
+
   return (retval);
 }
 
@@ -165,7 +178,7 @@ int IOFlexIO_OutputVarAs (const cGH *GH, const char *fullname, const char*alias)
    @calls      CheckSteerableParameters
 
    @var        GH
-   @vdesc      Pointer to CCTK GH
+   @vdesc      pointer to CCTK GH
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -184,17 +197,15 @@ int IOFlexIO_OutputVarAs (const cGH *GH, const char *fullname, const char*alias)
 int IOFlexIO_TimeFor (const cGH *GH, int vindex)
 {
   int retval;
-  flexioGH *myGH;
   char *fullname;
+  const flexioGH *myGH;
 
 
-  /* get the GH extension for IOFlexIO */
-  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
-
-  CheckSteerableParameters (myGH);
+  CheckSteerableParameters (GH);
 
   /* check if this variable should be output */
-  retval = myGH->out_every > 0 && myGH->do_out[vindex] &&
+  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
+  retval = myGH->out_every > 0 && myGH->requests[vindex] &&
            GH->cctk_iteration % myGH->out_every == 0;
   if (retval)
   {
@@ -225,7 +236,7 @@ int IOFlexIO_TimeFor (const cGH *GH, int vindex)
    @calls      IOFlexIO_Write
 
    @var        GH
-   @vdesc      Pointer to CCTK GH
+   @vdesc      pointer to CCTK GH
    @vtype      const cGH *
    @vio        in
    @endvar
@@ -250,12 +261,11 @@ int IOFlexIO_TriggerOutput (const cGH *GH, int vindex)
 
   varname = CCTK_VarName (vindex);
 
-  /* get the GH extension for IOFlexIO */
-  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
-
   if (verbose)
+  {
     CCTK_VInfo (CCTK_THORNSTRING, "IOFlexIO_TriggerOutput: "
                 "name, index = %s, %d", varname, vindex);
+  }
 
   /* do the output */
   retval = IOFlexIO_Write (GH, vindex, varname);
@@ -263,6 +273,7 @@ int IOFlexIO_TriggerOutput (const cGH *GH, int vindex)
   if (retval == 0)
   {
     /* register variable as having output this iteration */
+    myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
     myGH->out_last[vindex] = GH->cctk_iteration;
   }
 
@@ -282,81 +293,53 @@ int IOFlexIO_TriggerOutput (const cGH *GH, int vindex)
                and does appropriate re-evaluation.
    @enddesc
 
-   @calls      CCTK_TraverseString
+   @calls      IOUtil_ParseVarsForOutput
 
-   @var        myGH
-   @vdesc      Pointer to IOFlexIO GH
-   @vtype      flexioGH *
+   @var        GH
+   @vdesc      pointer to CCTK GH
+   @vtype      const cGH *
    @vio        in
    @endvar
 @@*/
-static void CheckSteerableParameters (flexioGH *myGH)
+static void CheckSteerableParameters (const cGH *GH)
 {
-  int times_set;
+  int vindex, times_set;
+  flexioGH *myGH;
+  char *fullname;
   static int out_vars_lastset = -1;
   DECLARE_CCTK_PARAMETERS
 
 
   /* how often to output */
+  myGH = (flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
   myGH->out_every = out3D_every >= 0 ? out3D_every : out_every;
 
   /* re-parse the 'out3D_vars' parameter if it was changed */
   times_set = CCTK_ParameterQueryTimesSet ("out3D_vars", CCTK_THORNSTRING);
   if (times_set != out_vars_lastset)
   {
-    memset (myGH->do_out, 0, CCTK_NumVars ());
-    CCTK_TraverseString (out3D_vars, SetOutputFlag, myGH->do_out,
-                         CCTK_GROUP_OR_VAR);
+    IOUtil_ParseVarsForOutput (GH, out3D_vars, myGH->requests);
+
+    /*** FIXME: IEEEIO doesn't provide a COMPLEX datatype
+                so we should map CCTK_COMPLEX to two REALs here.
+                We have to check for this already here because if an IEEEIO file
+                is created and nothing is written to it, it will crash
+                at re-opening. ***/
+    for (vindex = CCTK_NumVars () - 1; vindex >= 0; vindex--)
+    {
+      if (myGH->requests[vindex] &&
+          strncmp (CCTK_VarTypeName (vindex), "CCTK_VARIABLE_COMPLEX", 21))
+      {
+        fullname = CCTK_FullName (vindex);
+        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "IOFlexIO output for complex variable '%s' not yet "
+                    "supported", fullname);
+        free (fullname);
+        IOUtil_FreeIORequest (&myGH->requests[vindex]);
+      }
+    }
 
-    /* Save the last setting of 'out3D_vars' parameter */
+    /* save the last setting of 'out3D_vars' parameter */
     out_vars_lastset = times_set;
   }
 }
-
-
-/* check if this variable can be output (static conditions) */
-static int CheckOutputVar (int vindex)
-{
-  int retval;
-  char *fullname;
-
-
-/*** FIXME: IEEEIO doesn't provide a COMPLEX datatype
-            so we should map CCTK_COMPLEX to two REALs here.
-            We have to check for this already here because if an IEEEIO file
-            is created and nothing is written to it, it will crash
-            at re-opening. ***/
-  retval = strncmp (CCTK_VarTypeName (CCTK_VarTypeI (vindex)),
-                    "CCTK_VARIABLE_COMPLEX", 21);
-  if (! retval)
-  {
-    fullname = CCTK_FullName (vindex);
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "IOFlexIO output for complex variable '%s' not yet "
-                "supported", fullname);
-    free (fullname);
-  }
-
-  return (! retval);
-}
-
-
-/* callback for CCTK_TraverseString() to set the output flag
-   for the given variable */
-static void SetOutputFlag (int vindex, const char *optstring, void *arg)
-{
-  char *flags = (char *) arg;
-
-
-  /* Check the variable type */
-  if (CheckOutputVar (vindex) == 0)
-  {
-    flags[vindex] = 1;
-  }
-
-  if (optstring)
-  {
-    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Optional string '%s' in variable name ignored", optstring);
-  }
-}
diff --git a/src/RestoreFile.c b/src/RestoreFile.c
index 995e0a8..6dbfc02 100644
--- a/src/RestoreFile.c
+++ b/src/RestoreFile.c
@@ -2,17 +2,12 @@
    @file      RestoreFile.c
    @date      Thu Jun 18 16:34:59 1998
    @author    Tom Goodale
-   @desc 
-   Routines to restore GFs from a given file
-   @enddesc 
-   @history
-   @hauthor Gabrielle Allen @hdate 19 Oct 1998      
-   @hdesc Changed names ready for thorn_IO
-   @endhistory
-   @version $Id$
+   @desc
+              Routines to restore grid variables from a given file
+   @enddesc
+   @version   $Id$
  @@*/
 
-
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -22,22 +17,25 @@
 #include "CactusBase/IOUtil/src/ioGH.h"
 #include "ioFlexGH.h"
 
+/* the rcs ID and its dummy function to use it */
+static const char *rcsid = "$Id$";
+CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_RestoreFile_c)
+
 
-/* MPI tag base */
-#define STARTUPBASE 1001        
+/********************************************************************
+ ********************    Macro Definitions   ************************
+ ********************************************************************/
+/* tag base for MPI messages */
+#define STARTUPBASE 1001
 /* The maximum number of dimensions we can deal with.
-   There's no routine provided by the IEEEIO lib to query the number of 
+   There's no routine provided by the IEEEIO lib to query the number of
    dims from a dataset. */
 #define	MAXDIM      10
 
 
-/* the rcs ID and its dummy funtion to use it */
-static const char *rcsid = "$Id$";
-CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_RestoreFile_c)
-
-
-/* prototypes of routines defined in this source file */
-int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file);
+/********************************************************************
+ ********************    Internal Routines   ************************
+ ********************************************************************/
 static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
                                 int *grouptype, int *timelevel,int has_version);
 #ifdef CCTK_MPI
@@ -45,29 +43,37 @@ static int GetChunkAttributes (IOFile fid, int vindex);
 #endif
 
 
+/********************************************************************
+ ********************    External Routines   ************************
+ ********************************************************************/
+int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file);
+
+
  /*@@
    @routine    IOFlexIO_RecoverVariables
    @date       Fri Jun 19 09:19:48 1998
    @author     Tom Goodale
-   @desc 
+   @desc
                Reads in data from an IEEEIO file.
                The file has to be opened already, and the file layout
                determined (ioproc, ioproc_every, unchunked).
                This information is used to read the file and distribute
-               the data among all processors. 
-   @enddesc 
-   @calledby   IOFlexIO_RecoverGH
-   @history 
-   @hauthor Gabrielle Allen @hdate Oct 17 1998
-   @hdesc Changed logic so that cactus stops if any of the dimensions of the 
-          input file and the current cactus run differ.
-   @hauthor Thomas Radke @hdate May 05 1999
-   @hdesc Added parameter unchunked
-   @endhistory 
+               the data among all processors.
+   @enddesc
+
+   @history
+   @hauthor    Gabrielle Allen
+   @hdate      Oct 17 1998
+   @hdesc      Changed logic so that cactus stops if any of the dimensions of
+               the input file and the current cactus run differ.
+   @hauthor    Thomas Radke
+   @hdate      May 05 1999
+   @hdesc      Added parameter unchunked
+   @endhistory
 
    @var        GH
    @vdesc      Pointer to CCTK grid hierarchy
-   @vtype      cGH
+   @vtype      cGH *
    @vio        in
    @endvar
    @var        file
@@ -78,38 +84,43 @@ static int GetChunkAttributes (IOFile fid, int vindex);
 @@*/
 int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
 {
-  DECLARE_CCTK_PARAMETERS
-  int vindex, gtype;
-  int myproc, nprocs;
+  int vindex, gtype, timelevel, myproc, nprocs;
   int nDatasets, currentDataset;
-  int timelevel;             /* current timelevel to be restored */
-  pGH *pughGH;               /* PUGH extension handle */
-  ioGH *ioUtilGH;            /* IOUtil extension handle */
+  const pGH *pughGH;
+  const ioGH *ioUtilGH;
+  char *fullname;
 #ifdef CCTK_MPI
-  pGExtras *extras;
   int i, proc;
-  CCTK_INT info [3];         /* communication buffer for MPI */
+  pGExtras *extras;
+  CCTK_INT info[3];
+  int dim, npoints, vtype;
+  void *buffer;
+  int *chunkdims, *chunkorigin;
+  int element_size, flexio_type;
+  MPI_Datatype mpi_type;
+  MPI_Status ms;
 #endif
+  DECLARE_CCTK_PARAMETERS
 
 
-  /* to make the compiler happy */
+  /* suppress compiler warnings about potentially uninitialized variables */
   nDatasets = 0;
 
-  /* Get the handles for PUGH IOUtil extensions */
-  pughGH = PUGH_pGH (GH);
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  pughGH = (const pGH *) PUGH_pGH (GH);
+  ioUtilGH = (const ioGH *) CCTK_GHExtension (GH, "IO");
 
   myproc = CCTK_MyProc (GH);
   nprocs = CCTK_nProcs (GH);
 
-  /* all IO procs determine the number of datasets in their checkpoint files */
-  if (myproc == file->ioproc) {
-
-    /* Get the number of sets */
-    nDatasets = IOnDatasets (file->fid); 
+  /* all I/O procs determine the number of datasets in their checkpoint files */
+  if (myproc == file->ioproc)
+  {
+    /* get the number of sets */
+    nDatasets = IOnDatasets (file->fid);
     if (verbose)
+    {
       CCTK_VInfo (CCTK_THORNSTRING, "  Input file has %d datasets", nDatasets);
-
+    }
   }
 
   /* In Cactus 3.x we had only datasets containing grid function data.
@@ -127,24 +138,25 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
     CCTK_WARN (0, "Number of datasets isn't a multiple of nioprocs");
   */
 
-  /* Now process the datasets.
-     All IO processors read the datasets from their checkpoint file
-     verify their contents and communicate them to the non-IO processors. */
-
-  /* At first the code for the IO processors ... */
-  if (myproc == file->ioproc) {
+  /* now process the datasets.
+     All I/O processors read the datasets from their checkpoint file
+     verify their contents and communicate them to the non-I/O processors. */
 
-    /* Seek here once to the beginning of the file, the file pointer
+  /* at first the code for the I/O processors ... */
+  if (myproc == file->ioproc)
+  {
+    /* seek here once to the beginning of the file, the file pointer
        is advanced then implicitely by subsequent calls to IOreadInfo() */
     FLEXIO_ERROR (IOseek (file->fid, 0));
 
-    /* Each IO processor loops over all available datasets, checks their
-       consistency and broadcasts them to the non-IO processors. */
-    for (currentDataset = 0; currentDataset < nDatasets; currentDataset++) {
-
+    /* each IO processor loops over all available datasets, checks their
+       consistency and broadcasts them to the non-IO processors */
+    for (currentDataset = 0; currentDataset < nDatasets; currentDataset++)
+    {
       /* read in the next dataset's attributes and verify them */
       if (GetCommonAttributes (GH, file->fid, file->unchunked, &vindex,
-                               &gtype, &timelevel, file->has_version) < 0) {
+                               &gtype, &timelevel, file->has_version) < 0)
+      {
         CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Ignoring dataset %d", currentDataset);
         continue;
@@ -152,52 +164,49 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
 
       /* if we read in initial data via the file reader interface
          check whether the user wants to have this variable restored */
-      if (ioUtilGH->do_inVars && ! ioUtilGH->do_inVars [vindex]) {
-        if (verbose) {
-          char *varname = CCTK_FullName (vindex);
-
+      if (ioUtilGH->do_inVars && ! ioUtilGH->do_inVars[vindex])
+      {
+        if (verbose)
+        {
+          fullname = CCTK_FullName (vindex);
           CCTK_VInfo (CCTK_THORNSTRING, "Ignoring variable '%s' for file "
-                      "reader recovery", varname);
-          free (varname);
+                      "reader recovery", fullname);
+          free (fullname);
         }
         continue;
       }
 
-      /* Read in the data */
-      if (verbose) {
-        char *fullname = CCTK_FullName (vindex);
-
+      /* read in the data */
+      if (verbose)
+      {
+        fullname = CCTK_FullName (vindex);
         CCTK_VInfo (CCTK_THORNSTRING, "    dataset %d: %s (timelevel %d)",
                     currentDataset, fullname, timelevel);
         free (fullname);
       }
 
-      if (file->ioproc_every == 1) 
-        FLEXIO_ERROR (
-                   IOread (file->fid, CCTK_VarDataPtrI (GH, timelevel, vindex)));
+      if (file->ioproc_every == 1)
+      {
+        FLEXIO_ERROR (IOread (file->fid,
+                              CCTK_VarDataPtrI (GH, timelevel, vindex)));
+      }
 #ifdef CCTK_MPI
-      else {
-        int dim, npoints, vtype;
-        void *buffer;
-        int *chunkdims, *chunkorigin;
-        int element_size, flexio_type;
-        MPI_Datatype mpi_type;
-
-
+      else
+      {
         vtype = CCTK_VarTypeI (vindex);
         element_size = CCTK_VarTypeSize (vtype);
         mpi_type = PUGH_MPIDataType (pughGH, vtype);
         flexio_type = IOFlexIO_DataType (vtype);
-  
+
         if (element_size <= 0 || mpi_type == MPI_DATATYPE_NULL || flexio_type<0)
-        { 
+        {
           CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                       "Unsupported variable type %d", vtype);
           continue;
         }
 
-        /* Get the pGExtras pointer as a shortcut */
-        extras = ((pGA ***) pughGH->variables) [vindex][timelevel]->extras;
+        /* get the pGExtras pointer as a shortcut */
+        extras = ((pGA ***) pughGH->variables)[vindex][timelevel]->extras;
 
         /* get the dimension of the variable */
         dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
@@ -208,12 +217,16 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
         /* read my own data directly into data,
            read others data into buffer and communicate it */
         if (! file->unchunked || gtype == CCTK_SCALAR)
+        {
           FLEXIO_ERROR (IOread (file->fid,
-                               CCTK_VarDataPtrI (GH, timelevel, vindex)));
-        else {
-          for (i = 0; i < dim; i++) {
-            chunkdims [i] = extras->rnsize [file->ioproc][i];
-            chunkorigin [i] = extras->lb [file->ioproc][i];
+                                CCTK_VarDataPtrI (GH, timelevel, vindex)));
+        }
+        else
+        {
+          for (i = 0; i < dim; i++)
+          {
+            chunkdims[i] = extras->rnsize[file->ioproc][i];
+            chunkorigin[i] = extras->lb[file->ioproc][i];
           }
 
           FLEXIO_ERROR (IOreadChunk (file->fid, chunkdims, chunkorigin,
@@ -221,60 +234,73 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
         }
 
         /* read data for non-IO processors */
-        if (gtype == CCTK_SCALAR) {
+        if (gtype == CCTK_SCALAR)
+        {
           npoints = 1;
           buffer = CCTK_VarDataPtrI (GH, timelevel, vindex);
-        } else {
+        }
+        else
+        {
           /* allocate memory for the biggest chunk */
-          npoints = extras->rnpoints [file->ioproc + 1];
+          npoints = extras->rnpoints[file->ioproc + 1];
           for (proc = 2; proc < file->ioproc_every; proc++)
-            if (npoints < extras->rnpoints [file->ioproc + proc])
-              npoints = extras->rnpoints [file->ioproc + proc];
+          {
+            if (npoints < extras->rnpoints[file->ioproc + proc])
+            {
+              npoints = extras->rnpoints[file->ioproc + proc];
+            }
+          }
           buffer = malloc (npoints * element_size);
         }
 
         for (proc = file->ioproc + 1;
              proc < file->ioproc + file->ioproc_every && proc < nprocs;
-             proc++) {
-
-          if (gtype != CCTK_SCALAR) {
-
-            if (! file->unchunked) {
+             proc++)
+        {
+          if (gtype != CCTK_SCALAR)
+          {
+            if (! file->unchunked)
+            {
               /* Also increment dataset counter here !!! */
               currentDataset++;
 
-              if (GetChunkAttributes (file->fid, vindex) < 0) { 
+              if (GetChunkAttributes (file->fid, vindex) < 0)
+              {
                 CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                             "Ignoring chunk in dataset %d", currentDataset+1);
                 continue;
               }
 
               FLEXIO_ERROR (IOread (file->fid, buffer));
-            } else {
-              for (i = 0; i < dim; i++) {
-                chunkdims [i] = extras->rnsize [proc][i];
-                chunkorigin [i] = extras->lb [proc][i];
+            }
+            else
+            {
+              for (i = 0; i < dim; i++)
+              {
+                chunkdims[i] = extras->rnsize[proc][i];
+                chunkorigin[i] = extras->lb[proc][i];
               }
 
-              FLEXIO_ERROR (IOreadChunk (file->fid, chunkdims,
-                                   chunkorigin, buffer));
+              FLEXIO_ERROR (IOreadChunk (file->fid, chunkdims, chunkorigin,
+                                         buffer));
             }
 
-            npoints = extras->rnpoints [proc];
+            npoints = extras->rnpoints[proc];
           }
 
           /* and finally send the index and the data */
-          info [0] = vindex; info [1] = timelevel; info [2] = npoints;
+          info[0] = vindex; info[1] = timelevel; info[2] = npoints;
           CACTUS_MPI_ERROR (MPI_Send (info, 3, PUGH_MPI_INT, proc,
-                            STARTUPBASE, pughGH->PUGH_COMM_WORLD));
+                                      STARTUPBASE, pughGH->PUGH_COMM_WORLD));
           CACTUS_MPI_ERROR (MPI_Send (buffer, npoints, mpi_type, proc,
-                            STARTUPBASE, pughGH->PUGH_COMM_WORLD));
-
+                                      STARTUPBASE, pughGH->PUGH_COMM_WORLD));
         }
 
         /* free allocated chunk */
         if (gtype != CCTK_SCALAR)
+        {
           free (buffer);
+        }
 
         free (chunkorigin);
 
@@ -284,57 +310,47 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
     } /* end of loop over all datasets */
 
 #ifdef CCTK_MPI
-    /* Finally an invalid variable index is communicated
-       to indicate completion to the non-IO processors. */
-    info [0] = -1;
+    /* finally an invalid variable index is communicated
+       to indicate completion to the non-I/O processors */
+    info[0] = -1;
     for (proc = file->ioproc + 1;
          proc < file->ioproc + file->ioproc_every && proc < nprocs;
          proc++)
+    {
       CACTUS_MPI_ERROR (MPI_Send (info, 3, PUGH_MPI_INT, proc,
-                        STARTUPBASE, pughGH->PUGH_COMM_WORLD));
+                                  STARTUPBASE, pughGH->PUGH_COMM_WORLD));
+    }
 #endif
 
-  } else {
-
-    /* And here the code for non-IO processors: */
+  }
+  else
+  {
+    /* and here the code for non-IO processors: */
 #ifdef CCTK_MPI
-    int npoints;
-    MPI_Datatype mpi_type;
-    MPI_Status ms;
-
     /* They don't know how many datasets there are, because the IO processors
        could skip some on the fly during their consistency checks.
        The IO Processor sends the index of the variable to be processed next.
        So, all non-IO processors execute a loop where the termination condition
        is when an invalid index was received.
     */
-    while (1) {
+    while (1)
+    {
       /* receive the next variable index from the IO processor */
       CACTUS_MPI_ERROR (MPI_Recv (info, 3, PUGH_MPI_INT, file->ioproc,
-                        STARTUPBASE, pughGH->PUGH_COMM_WORLD, &ms));
-      vindex = info [0]; timelevel = info [1]; npoints = info [2];
+                                  STARTUPBASE, pughGH->PUGH_COMM_WORLD, &ms));
+      vindex = info[0]; timelevel = info[1]; npoints = info[2];
 
       /* check for termination condition */
       if (vindex < 0)
+      {
         break;
-
-      switch (CCTK_VarTypeI (vindex)) {
-        case CCTK_VARIABLE_CHAR:    mpi_type = PUGH_MPI_CHAR; break;
-        case CCTK_VARIABLE_INT:     mpi_type = PUGH_MPI_INT;  break;
-        case CCTK_VARIABLE_REAL:    mpi_type = PUGH_MPI_REAL; break;
-#if 0
-/* FIXME: Don't know how to support COMPLEX types too !! */
-        case CCTK_VARIABLE_COMPLEX: mpi_type = pughGH->PUGH_mpi_complex; break;
-#endif
-        default:
-          CCTK_WARN (1, "Unsupported datatype");
-          continue;
       }
 
       /* receive following data from my IO processor */
+      mpi_type = PUGH_MPIDataType (pughGH, CCTK_VarTypeI (vindex));
       CACTUS_MPI_ERROR (MPI_Recv (CCTK_VarDataPtrI (GH, timelevel, vindex),
-                        npoints, mpi_type, file->ioproc,
-                        STARTUPBASE, pughGH->PUGH_COMM_WORLD, &ms));
+                                  npoints, mpi_type, file->ioproc,
+                                  STARTUPBASE, pughGH->PUGH_COMM_WORLD, &ms));
     }
 #endif
   }
@@ -342,8 +358,10 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
   return (0);
 }
 
-/************************* local routines ********************************/
 
+/********************************************************************
+ ********************    Internal Routines   ************************
+ ********************************************************************/
 /* local routine getDatasetAttributes() reads in the next dataset's attributes
    and verifies them:
 
@@ -364,30 +382,29 @@ int IOFlexIOi_RecoverVariables (cGH *GH, fileinfo_t *file)
 static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
                                 int *grouptype, int *timelevel, int has_version)
 {
-  int i, flag;
-  int atype;
+  int i, flag, atype, vartype_stored, rank_stored, result;
   Long asize;
   pGExtras *extras;
   cGroup groupdata;
-  int vartype_stored, rank_stored, dims_stored [MAXDIM];
   CCTK_INT4 grouptype_stored, numtimelevels_stored, timelevel_stored;
-  int result, *dims;
+  int *dims, dims_stored[MAXDIM];
   char *groupname;
-  char fullname [512], groupname_stored [512];
+  char fullname[512], groupname_stored[512];
 
 
   /* read the next dataset's info from the file */
   result = IOreadInfo (fid, &vartype_stored, &rank_stored, dims_stored, MAXDIM);
   FLEXIO_ERROR (result);
-  if (result == 0) {
+  if (result == 0)
+  {
     CCTK_WARN (1, "Can't read dataset info");
     return (-1);
   }
-      
+
   /* retrieve the name attribute */
   i = IOreadAttributeInfo (fid, "name", &atype, &asize);
-  if (i < 0 || atype != FLEXIO_CHAR ||
-      (unsigned int) asize >= sizeof (fullname)) {
+  if (i < 0 || atype != FLEXIO_CHAR || asize >= (int) sizeof (fullname))
+  {
     CCTK_WARN (2, "Can't read name attribute");
     return (-1);
   }
@@ -395,7 +412,8 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
 
   /* check if there is a matching variable */
   *vindex = CCTK_VarIndex (fullname);
-  if (*vindex < 0) {
+  if (*vindex < 0)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "No matching variable found for '%s'", fullname);
     return (-1);
@@ -404,14 +422,16 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
   /* read and verify the group name */
   i = IOreadAttributeInfo (fid, "groupname", &atype, &asize);
   if (i < 0 || atype != FLEXIO_CHAR ||
-      (unsigned int) asize >= sizeof (groupname_stored)) {
+      (unsigned int) asize >= sizeof (groupname_stored))
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Can't read groupname attribute of '%s'", fullname);
     return (-1);
   }
   FLEXIO_ERROR (IOreadAttribute (fid, i, groupname_stored));
   groupname = CCTK_GroupNameFromVarI (*vindex);
-  if (! CCTK_Equals (groupname_stored, groupname)) {
+  if (! CCTK_Equals (groupname_stored, groupname))
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Groupnames don't match for '%s'", fullname);
     return (-1);
@@ -420,7 +440,8 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
 
   /* read the group type */
   i = IOreadAttributeInfo (fid, "grouptype", &atype, &asize);
-  if (i < 0 || atype != FLEXIO_INT4 || asize != 1) {
+  if (i < 0 || atype != FLEXIO_INT4 || asize != 1)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Can't read grouptype attribute for '%s'", fullname);
     return (-1);
@@ -429,7 +450,8 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
 
   /* read the number of timelevels */
   i = IOreadAttributeInfo (fid, "ntimelevels", &atype, &asize);
-  if (i < 0 || atype != FLEXIO_INT4 || asize != 1) {
+  if (i < 0 || atype != FLEXIO_INT4 || asize != 1)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Can't read ntimelevels attribute for '%s'", fullname);
     return (-1);
@@ -438,7 +460,8 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
 
   /* read the timelevel to restore */
   i = IOreadAttributeInfo (fid, "timelevel", &atype, &asize);
-  if (i < 0 || atype != FLEXIO_INT4 || asize != 1) {
+  if (i < 0 || atype != FLEXIO_INT4 || asize != 1)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Can't read timelevel attribute for '%s'", fullname);
     return (-1);
@@ -447,17 +470,20 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
   *timelevel = (int) timelevel_stored;
 
   /* verify group type, variable type, dims, sizes and ntimelevels */
-  if (CCTK_GroupData (CCTK_GroupIndex (groupname_stored), &groupdata) != 0) {
+  if (CCTK_GroupData (CCTK_GroupIndex (groupname_stored), &groupdata) != 0)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Could not get group info for '%s'", fullname);
     return (-1);
   }
-  if (groupdata.grouptype != grouptype_stored) {
+  if (groupdata.grouptype != grouptype_stored)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Group types don't match for '%s'", fullname);
     return (-1);
   }
-  if (groupdata.numtimelevels != numtimelevels_stored) {
+  if (groupdata.numtimelevels != numtimelevels_stored)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Number of timelevels don't match for '%s'", fullname);
     return (-1);
@@ -478,42 +504,44 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
 #if 0
 /* FIXME: Don't know how to support COMPLEX type too !! */
       (vartype_stored == FLEXIO_REAL &&
-       groupdata.vartype == CCTK_VARIABLE_COMPLEX)) {
+       groupdata.vartype == CCTK_VARIABLE_COMPLEX))
 #else
-       0) {
+       0)
+  {
 #endif
     /* everything is okay */
-  } else {
+  }
+  else
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Variable types don't match for '%s'", fullname);
     return (-1);
   }
 
   /* verify the dims and sizes */
-  flag = 0;
-  if (groupdata.dim != rank_stored)
-    flag = 1;
-  switch (groupdata.grouptype) {
-    case CCTK_SCALAR:
-      if (dims_stored [0] != 1)
-        flag = 1;
-      break;
-    case CCTK_GF:
-    case CCTK_ARRAY:
-      extras = ((pGA ***) PUGH_pGH (GH)->variables)[*vindex][*timelevel]->extras;
-      dims = unchunked ? extras->nsize : extras->lnsize;
-      for (i = 0; i < groupdata.dim; i++)
-        if (dims [i] != dims_stored [i])
-          flag = 1;
-      break;
+  flag = groupdata.dim != rank_stored;
+  if (groupdata.grouptype == CCTK_SCALAR)
+  {
+    flag |= dims_stored[0] != 1;
+  }
+  else
+  {
+    extras = ((pGA ***) PUGH_pGH (GH)->variables)[*vindex][*timelevel]->extras;
+    dims = unchunked ? extras->nsize : extras->lnsize;
+    for (i = 0; i < groupdata.dim; i++)
+    {
+      flag |= dims[i] != dims_stored[i];
+    }
   }
-  if (flag) {
+  if (flag)
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Variable sizes don't match for '%s'", fullname);
     return (-1);
   }
 
-  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (*vindex))) {
+  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (*vindex)))
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Can't read into '%s': no storage assigned", fullname);
     return (-1);
@@ -529,37 +557,34 @@ static int GetCommonAttributes (cGH *GH, IOFile fid, int unchunked, int *vindex,
    that should be a chunk.
    It verifies via the name attribute that this chunk belongs to the
    current variable given by its index. */
-
 #ifdef CCTK_MPI
 static int GetChunkAttributes (IOFile fid, int vindex)
 {
-  int i;
-  int atype;
+  int i, atype, vtype_stored, rank_stored;
   Long asize;
-  int result;
-  int vtype_stored, rank_stored, dims_stored [MAXDIM];
-  char fullname [512];
+  int dims_stored[MAXDIM];
+  char fullname[512];
 
 
   /* read the next dataset's info from the file */
-  result = IOreadInfo (fid, &vtype_stored, &rank_stored, dims_stored, MAXDIM);
-  FLEXIO_ERROR (result);
-  if (result == 0) {
+  if (! IOreadInfo (fid, &vtype_stored, &rank_stored, dims_stored, MAXDIM))
+  {
     CCTK_WARN (1, "Can't read dataset info");
     return (-1);
   }
-      
+
   /* retrieve the name attribute */
   i = IOreadAttributeInfo (fid, "name", &atype, &asize);
-  if (i < 0 || atype != FLEXIO_CHAR ||
-      (unsigned int) asize >= sizeof (fullname)) {
+  if (i < 0 || atype != FLEXIO_CHAR || asize >= (int) sizeof (fullname))
+  {
     CCTK_WARN (2, "Can't read name attribute");
     return (-1);
   }
   FLEXIO_ERROR (IOreadAttribute (fid, i, fullname));
 
   /* check if there is a matching variable */
-  if (vindex != CCTK_VarIndex (fullname)) {
+  if (vindex != CCTK_VarIndex (fullname))
+  {
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "No matching variable found for '%s'", fullname);
     return (-1);
diff --git a/src/Startup.c b/src/Startup.c
index 29b27e6..42cafce 100644
--- a/src/Startup.c
+++ b/src/Startup.c
@@ -28,13 +28,13 @@ CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_Startup_c)
  ********************    External Routines   ************************
  ********************************************************************/
 void IOFlexIO_Startup (void);
-int IOFlexIO_TerminateGH (cGH *GH);
+int IOFlexIO_TerminateGH (const cGH *GH);
 
 
 /********************************************************************
  ********************    Internal Routines   ************************
  ********************************************************************/
-static void *SetupGH (tFleshConfig *config, int convergence_level, cGH *GH);
+static void *SetupGH (tFleshConfig *config, int conv_level, cGH *GH);
 
 
  /*@@
@@ -79,7 +79,7 @@ void IOFlexIO_Startup (void)
 
    @var        GH
    @vdesc      pointer to grid hierarchy
-   @vtype      cGH *
+   @vtype      const cGH *
    @vio        in
    @endvar
 
@@ -88,7 +88,7 @@ void IOFlexIO_Startup (void)
                0 for success
    @endreturndesc
 @@*/
-int IOFlexIO_TerminateGH (cGH *GH)
+int IOFlexIO_TerminateGH (const cGH *GH)
 {
   int i;
   flexioGH *myGH;
@@ -187,7 +187,7 @@ int IOFlexIO_TerminateGH (cGH *GH)
    @vtype      tFleshConfig *
    @vio        in
    @endvar
-   @var        convergence_level
+   @var        conv_level
    @vdesc      convergence level (unused)
    @vtype      int
    @vio        in
@@ -203,7 +203,7 @@ int IOFlexIO_TerminateGH (cGH *GH)
                pointer to the allocated GH extension structure
    @endreturndesc
 @@*/
-static void *SetupGH (tFleshConfig *config, int convergence_level, cGH *GH)
+static void *SetupGH (tFleshConfig *config, int conv_level, cGH *GH)
 {
   int i, numvars, maxdim;
   flexioGH *myGH;
@@ -217,7 +217,7 @@ static void *SetupGH (tFleshConfig *config, int convergence_level, cGH *GH)
 
   /* suppress compiler warnings about unused parameters */
   (void) (config + 0);
-  (void) (convergence_level + 0);
+  (void) (conv_level + 0);
 
   /* register the IOFlexIO routines as I/O methods  */
   i = CCTK_RegisterIOMethod ("IOFlexIO_2D");
@@ -241,7 +241,7 @@ static void *SetupGH (tFleshConfig *config, int convergence_level, cGH *GH)
   /* allocate a new GH extension structure */
   numvars = CCTK_NumVars ();
   myGH = (flexioGH *) malloc (sizeof (flexioGH));
-  myGH->do_out = (char *) malloc (numvars * sizeof (char));
+  myGH->requests  = (ioRequest **) calloc (numvars, sizeof (ioRequest *));
   myGH->do_out2D = (char *) malloc (numvars * sizeof (char));
   myGH->out_last = (int *) malloc (numvars * sizeof (int));
   myGH->out2D_last = (int *) malloc (numvars * sizeof (int));
diff --git a/src/Write.c b/src/Write.c
index 3eab8c3..44ace87 100644
--- a/src/Write.c
+++ b/src/Write.c
@@ -9,12 +9,10 @@
  @@*/
 
 
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cctk.h"
 #include "cctk_Parameters.h"
-#include "CactusPUGH/PUGH/src/include/pugh.h"
 #include "CactusBase/IOUtil/src/ioGH.h"
 #include "CactusBase/IOUtil/src/ioutil_Utils.h"
 #include "ioFlexGH.h"
@@ -67,7 +65,8 @@ static IEEEfile_t *OpenFile (const cGH *GH, const char *alias, int *is_new_file)
 @@*/
 int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias)
 {
-  ioGH *ioUtilGH;
+  const flexioGH *myGH;
+  const ioGH *ioUtilGH;
   int len, is_new_file, retval;
   char buffer[128];
   char *fullname;
@@ -85,9 +84,6 @@ int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias)
     return (-1);
   }
 
-  /* get the handle for IOUtil and IOFlexIO extensions */
-  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
-
   if (verbose)
   {
     CCTK_VInfo (CCTK_THORNSTRING, "IOFlexIO output of variable '%s', "
@@ -98,6 +94,7 @@ int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias)
   file = OpenFile (GH, alias, &is_new_file);
 
   /* nothing to do for an already opened file which is kept open all the time */
+  ioUtilGH = (const ioGH *) CCTK_GHExtension (GH, "IO");
   if (CCTK_MyProc (GH) == ioUtilGH->ioproc)
   {
     if (is_new_file || reuse_filehandles)
@@ -150,7 +147,7 @@ int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias)
         FLEXIO_ERROR (IOresume (file->iofile));
       }
 
-      /* Turn on buffer cache (with default size chosen by IEEEIO lib) */
+      /* turn on buffer cache (with default size chosen by IEEEIO lib) */
       if (file->iofile)
       {
         IEEEbufferOn (file->iofile, 0);
@@ -170,7 +167,8 @@ int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias)
   }
 
   /* output the data */
-  retval = IOFlexIO_DumpVar (GH, vindex, 0, file->iofile);
+  myGH = (const flexioGH *) CCTK_GHExtension (GH, "IOFlexIO");
+  retval = IOFlexIO_DumpVar (GH, myGH->requests[vindex], file->iofile);
 
   /* output the GH extensions and parameters */
   if (file->iofile)
diff --git a/src/ioFlexGH.h b/src/ioFlexGH.h
index 6691a6c..88d550d 100644
--- a/src/ioFlexGH.h
+++ b/src/ioFlexGH.h
@@ -12,7 +12,7 @@
 #define _IOFLEXIO_IOFLEXGH_H_ 1
 
 #include "StoreNamedData.h"
-
+#include "CactusBase/IOUtil/src/ioutil_Utils.h"
 #include "CactusExternal/FlexIO/src/IOProtos.h"
 #include "CactusExternal/FlexIO/src/IEEEIO.h"
 
@@ -63,19 +63,19 @@
    in an IEEEIO file. */
 #define GLOBAL_PARAMETERS  "global_parameters"
 
-/* Check error flags from IOFlexIO */
+/* check error flags from IOFlexIO */
 #define FLEXIO_ERROR(fn_call)                                                 \
-        do                                                                    \
         {                                                                     \
-          int error_code = fn_call;                                           \
+          int _retval = fn_call;                                              \
+                                                                              \
                                                                               \
-          if (error_code < 0)                                                 \
+          if (_retval < 0)                                                    \
           {                                                                   \
             CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,              \
                         "IEEEIO call '%s' returned error code %d\n",          \
-                        #fn_call, error_code);                                \
+                        #fn_call, _retval);                                   \
           }                                                                   \
-        } while (0)
+        }
 
 
 /* info structure describing IEEEIO output files
@@ -106,8 +106,11 @@ typedef struct
 /* IOFlexIO GH extension structure */
 typedef struct IOFlexIOGH
 {
-  /* flags indicating output for var [i] */
-  char *do_out2D, *do_out;
+  /* flags indicating 2D output for var [i] */
+  char *do_out2D;
+
+  /* I/O request descriptions for all CCTK variables */
+  ioRequest **requests;
 
   /* how often to output */
   int out2D_every, out_every;
@@ -148,7 +151,7 @@ int IOFlexIO_DataType (int cctk_type);
 int IOFlexIO_Write2D (const cGH *GH, int vindex, const char *alias);
 int IOFlexIO_Write (const cGH *GH, int vindex, const char *alias);
 void IOFlexIO_DumpGH (const cGH *GH, int called_from);
-int IOFlexIO_DumpVar (const cGH *GH, int vindex, int timelevel, IOFile iof);
+int IOFlexIO_DumpVar (const cGH *GH, const ioRequest *request, IOFile file);
 void IOFlexIOi_DumpParameters (const cGH *GH, int all, IOFile file);
 void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file);
 int IOFlexIO_Recover (cGH *GH, const char *basefilename, int called_from);