Tidying up the code to make it consistent with the structure in IOHDF5.

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOFlexIO/trunk@230 ebee0441-1374-4afa-a3b5-247f3ba15b9a

1 files changed, 547 insertions, 520 deletions

diff --git a/src/DumpVar.c b/src/DumpVar.c
index 5ca15ca..6932d00 100644
--- a/src/DumpVar.c
+++ b/src/DumpVar.c
@@ -1,35 +1,16 @@
 /*@@
    @file      DumpVar.c
-   @desc 
-   Do the actual writing of a 3D grid function, for output or for checkpointing
-   @enddesc 
-   @history
-   @hauthor Gabrielle Allen @hdate Oct 17 1998
-   @hdesc Split into subroutines and tidied.
-   @hauthor Gabrielle Allen @hdate 19 Oct 1998      
-   @hdesc Changed names ready for thorn_IO
-   @hauthor Thomas Radke @hdate 16 Mar 1999
-   @hdesc Converted to Cactus 4.0
-          introduced separate downsample values for each dimension
-   @hauthor Thomas Radke @hdate 17 Mar 1999
-   @hdesc Changed naming: IEEEIO -> IOFlexIO
-   @hauthor Thomas Radke @hdate 17 Mar 1999
-   @hdesc fixed errors for MPI
-   @hauthor Thomas Radke @hdate 12 Apr 1999
-   @hdesc Store full variable name in IOFlexIO_AddCommonAttributes()
-   @hauthor Thomas Radke @hdate 16 Apr 1999
-   @hdesc Added groupname, grouptype, ntimelevels, and current timelevel
-          as common attributes
-   @hendhistory
-   @version $Id$
+   @author    Paul Walker, Gabrielle Allen, Tom Goodale, Thomas Radke
+   @desc
+              Routines for writing variables into IOFlexIO data or
+              checkpoint files.
+   @enddesc
+   @version   $Id$
  @@*/
 
 
 #include <stdio.h>
 #include <stdlib.h>
-#ifdef SGI
-#include <time.h>
-#endif
 
 #include "cctk.h"
 #include "cctk_Parameters.h"
@@ -38,17 +19,25 @@
 #include "ioFlexGH.h"
 
 
-#define IOTAGBASE 20000 /* This may break on more than 2000 processors */
-
-/*#define IOFLEXIO_DEBUG  1 */
-
 /* the rcs ID and its dummy funtion to use it */
 static const char *rcsid = "$Id$";
 CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_DumpVar_c)
 
 
+/********************************************************************
+ ********************    Macro Definitions   ************************
+ ********************************************************************/
+/* #define IOFLEXIO_DEBUG  1 */
+
+#define IOTAGBASE 20000 /* This may break on more than 2000 processors */
+
+
+/********************************************************************
+ ********************    Internal Typedefs   ************************
+ ********************************************************************/
 /* info structure describing how to dump a given CCTK variable type */
-typedef struct {
+typedef struct
+{
   int flexio_type;              /* the FLEXIO type to use */
   int element_size;             /* size of a single data element */
 #ifdef CCTK_MPI
@@ -57,73 +46,78 @@ typedef struct {
 } dumpInfo_t;
 
 
-/* local function prototypes */
-static void IOFlexIO_DumpGS (const cGH *GH, int vindex, int timelevel, IOFile iof,
-               int flexio_type);
-static void IOFlexIO_DumpGA (const cGH *GH, int vindex, int timelevel, IOFile iof,
-               dumpInfo_t *info);
-static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
-             void **outme, int *free_outme, CCTK_INT4 *geom, int element_size);
-static void IOFlexIO_eachProcDump (const cGH *GH, int vindex, int timelevel,
-               void *outme, CCTK_INT4 *geom, IOFile iof, int flexio_type);
-static void IOFlexIO_AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
-               IOFile iof);
-static void IOFlexIO_AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
-               CCTK_INT4 *gsz, IOFile iof);
+/********************************************************************
+ ********************    Internal Routines   ************************
+ ********************************************************************/
+static int DumpGS (const cGH *GH, int vindex, int timelevel, IOFile file,
+                   int flexio_type);
+static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
+                   dumpInfo_t *info);
+static int GetDumpData (const cGH *GH, int vindex, int timelevel, void **outme,
+                        int *free_outme, CCTK_INT4 *geom, int element_size);
+static void EachProcDump (const cGH *GH, int vindex, int timelevel, void *outme,
+                          CCTK_INT4 *geom, IOFile file, int flexio_type);
+static void AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
+                                IOFile file);
+static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
+                                 CCTK_INT4 *gsz, IOFile file);
 #ifdef CCTK_MPI
-static void IOFlexIO_procDump (IOFile iof, const cGH *GH, int vindex, void *outme,
-               CCTK_INT4 *geom, int flexio_type);
+static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
+                      CCTK_INT4 *geom, int flexio_type);
 #endif
 
 
-
 /*@@
-  @routine    IOFlexIO_DumpVar
-  @date       16 Apr 1999
-  @author     Thomas Radke
-  @desc 
-              Generic dump routine, just calls the appropriate dump routine
-  @enddesc 
-  @calls
-  @calledby   IOFlexIO_DumpGH IOFlexIO_Write3D
-
-  @var        GH
-  @vdesc      Pointer to CCTK grid hierarchy
-  @vtype      const cGH *
-  @vio        in
-  @endvar
-  @var        vindex
-  @vdesc      global index of the variable to be dumped
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        timelevel
-  @vdesc      the timelevel to store
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        iof
-  @vdesc      the IEEEIO file to dump to
-  @vtype      IOFile
-  @vio        in
-  @endvar
-  @history 
-  @endhistory 
+   @routine    IOFlexIO_DumpVar
+   @date       16 Apr 1999
+   @author     Thomas Radke
+   @desc
+               Generic dump routine, just calls the appropriate dump routine
+   @enddesc
+
+   @calls
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      const cGH *
+   @vio        in
+   @endvar
+   @var        vindex
+   @vdesc      global index of the variable to be dumped
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        timelevel
+   @vdesc      the timelevel to store
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        file
+   @vdesc      the IEEEIO file to dump to
+   @vtype      IOFile
+   @vio        in
+   @endvar
 
+   @returntype int
+   @returndesc
+               -1 if variable has unknown type
+               or return code of either @seeroutine DumpGS or @seeroutine DumpGA
+   @endreturndesc
 @@*/
-void IOFlexIO_DumpVar (const cGH *GH, int vindex, int timelevel, IOFile iof)
+int IOFlexIO_DumpVar (const cGH *GH, int vindex, int timelevel, IOFile file)
 {
+  int vtype, gtype, retval;
   pGH *pughGH;
   ioGH *ioUtilGH;
-  int vtype;
   dumpInfo_t info;
 
 
-  /* Get the handle for PUGH and IOUtil extensions */
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
-  pughGH = (pGH *) GH->extensions [CCTK_GHExtensionHandle ("PUGH")];
+  /* get the handle for PUGH and IOUtil extensions */
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
+  pughGH = (pGH *) CCTK_GHExtension (GH, "PUGH");
 
   vtype = CCTK_VarTypeI (vindex);
+
   /* downgrade the output precision if requested */
   if (ioUtilGH->out_single)
   {
@@ -159,24 +153,27 @@ void IOFlexIO_DumpVar (const cGH *GH, int vindex, int timelevel, IOFile iof)
   {
     CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Unsupported variable type %d", vtype);
-    return;
+    return (-1);
   }
 
-  switch (CCTK_GroupTypeFromVarI (vindex)) {
-    case CCTK_SCALAR:
-      IOFlexIO_DumpGS (GH, vindex, timelevel, iof, info.flexio_type);
-      break;
-
-    case CCTK_GF:
-    case CCTK_ARRAY:
-      IOFlexIO_DumpGA (GH, vindex, timelevel, iof, &info);
-      break;
-
-    default:
-      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Invalid group type %d", CCTK_GroupTypeFromVarI (vindex));
-      return;
+  /* now branch to the appropriate dump routine */
+  gtype = CCTK_GroupTypeFromVarI (vindex);
+  if (gtype == CCTK_SCALAR)
+  {
+    retval = DumpGS (GH, vindex, timelevel, file, info.flexio_type);
+  }
+  else if (gtype == CCTK_ARRAY || gtype == CCTK_GF)
+  {
+    retval = DumpGA (GH, vindex, timelevel, file, &info);
+  }
+  else
+  {
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Invalid group type %d", gtype);
+    retval = -1;
   }
+
+  return (retval);
 }
 
 
@@ -206,162 +203,178 @@ int IOFlexIO_DataType (int cctk_type)
 
   switch (cctk_type)
   {
-    case CCTK_VARIABLE_CHAR:      retval = FLEXIO_CHAR; break;
-    case CCTK_VARIABLE_INT:       retval = FLEXIO_INT; break;
-    case CCTK_VARIABLE_REAL:      retval = FLEXIO_REAL; break;
+    case CCTK_VARIABLE_CHAR:   retval = FLEXIO_CHAR; break;
+    case CCTK_VARIABLE_INT:    retval = FLEXIO_INT; break;
+    case CCTK_VARIABLE_REAL:   retval = FLEXIO_REAL; break;
 #ifdef CCTK_INT2
-    case CCTK_VARIABLE_INT2:      retval = FLEXIO_INT2; break;
+    case CCTK_VARIABLE_INT2:   retval = FLEXIO_INT2; break;
 #endif
 #ifdef CCTK_INT4
-    case CCTK_VARIABLE_INT4:      retval = FLEXIO_INT4; break;
+    case CCTK_VARIABLE_INT4:   retval = FLEXIO_INT4; break;
 #endif
 #ifdef CCTK_INT8
-    case CCTK_VARIABLE_INT8:      retval = FLEXIO_INT8; break;
+    case CCTK_VARIABLE_INT8:   retval = FLEXIO_INT8; break;
 #endif
 #ifdef CCTK_REAL4
-    case CCTK_VARIABLE_REAL4:     retval = FLEXIO_REAL4; break;
+    case CCTK_VARIABLE_REAL4:  retval = FLEXIO_REAL4; break;
 #endif
 #ifdef CCTK_REAL8
-    case CCTK_VARIABLE_REAL8:     retval = FLEXIO_REAL8; break;
+    case CCTK_VARIABLE_REAL8:  retval = FLEXIO_REAL8; break;
 #endif
 #ifdef CCTK_REAL16
-    case CCTK_VARIABLE_REAL16:    retval = FLEXIO_REAL16; break;
+    case CCTK_VARIABLE_REAL16: retval = FLEXIO_REAL16; break;
 #endif
 
     default: CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                          "Unsupported CCTK variable datatype %d", cctk_type);
              retval = -1;
+             break;
   }
 
   return (retval);
 }
 
-/************************** local routines ******************************/
 
+/********************************************************************
+ ********************    Internal Routines   ************************
+ ********************************************************************/
 /*@@
-  @routine    IOFlexIO_DumpGS
-  @date       16 Apr 1999
-  @author     Thomas Radke
-  @desc 
-              Dumps a SCALAR type variable into a IEEEIO file
-  @enddesc 
-  @calls
-  @calledby   IOFlexIO_DumpVar
-
-  @var        GH
-  @vdesc      Pointer to CCTK grid hierarchy
-  @vtype      const cGH *
-  @vio        in
-  @endvar
-  @var        vindex
-  @vdesc      global index of the variable to be dumped
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        timelevel
-  @vdesc      the timelevel to store
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        iof
-  @vdesc      the IEEEIO file to dump to
-  @vtype      IOFile
-  @vio        in
-  @endvar
-  @var        flexio_type
-  @vdesc      the IOFlexIO datatype to store
-  @vtype      int
-  @vio        in
-  @endvar
-  @history 
-  @endhistory 
+   @routine    DumpGS
+   @date       16 Apr 1999
+   @author     Thomas Radke
+   @desc
+               Dumps a SCALAR type variable into a IEEEIO file
+   @enddesc
+
+   @calls      AddCommonAttributes
 
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      const cGH *
+   @vio        in
+   @endvar
+   @var        vindex
+   @vdesc      global index of the variable to be dumped
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        timelevel
+   @vdesc      the timelevel to store
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        file
+   @vdesc      the IEEEIO file to dump to
+   @vtype      IOFile
+   @vio        in
+   @endvar
+   @var        flexio_type
+   @vdesc      the IOFlexIO datatype to store
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for success, or -1 if file handle is invalid
+   @endreturndesc
 @@*/
-static void IOFlexIO_DumpGS (const cGH *GH, int vindex, int timelevel, IOFile iof,
-                             int flexio_type)
+static int DumpGS (const cGH *GH, int vindex, int timelevel, IOFile file,
+                   int flexio_type)
 {
   ioGH *ioUtilGH;
-  CCTK_INT gsz [] = {0, 0, 0};    /* not needed here ? */
-  int dim [] = {1};               /* size of scalar */
+  CCTK_INT gsz[] = {0, 0, 0};    /* not needed here ? */
+  int dim[] = {1};               /* size of scalar */
+
 
+  /* immediately return if file handle is invalid */
+  if (! IOisValid (file))
+  {
+    return (-1);
+  }
 
-  /* Get the handle for IOUtil extensions */
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  /* get the handle for IOUtil extensions */
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
 
   if (CCTK_MyProc (GH) != ioUtilGH->ioproc)
-    return;
+  {
+    return (0);
+  }
 
   /* first dump the data then add the attributes */
-  CACTUS_IEEEIO_ERROR (IOwrite (iof, flexio_type, 1, dim,
-                       CCTK_VarDataPtrI (GH, timelevel, vindex)));
-  IOFlexIO_AddCommonAttributes (GH, vindex, timelevel, gsz, iof);
+  FLEXIO_ERROR (IOwrite (file, flexio_type, 1, dim,
+                         CCTK_VarDataPtrI (GH, timelevel, vindex)));
+  AddCommonAttributes (GH, vindex, timelevel, gsz, file);
+
+  return (0);
 }
 
 
 /*@@
-  @routine    IOFlexIO_DumpGA
-  @date       July 1998
-  @author     Paul Walker
-  @desc 
-  
-  @enddesc 
-  @calls     
-  @calledby   IOFlexIO_DumpVar
-
-  @var        GH
-  @vdesc      Pointer to CCTK grid hierarchy
-  @vtype      const cGH *
-  @vio        in
-  @endvar
-  @var        vindex
-  @vdesc      global index of the variable to be dumped
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        timelevel
-  @vdesc      the timelevel to store
-  @vtype      int
-  @vio        in
-  @endvar
-  @var        iof
-  @vdesc      the IEEEIO file to dump to
-  @vtype      IOFile
-  @vio        in
-  @endvar
-  @var        info
-  @vdesc      info structure describing
-                - the IOFlexIO datatype to store
-                - the size of an element of variable
-                - the MPI datatype to communicate
-  @vtype      dumpInfo_t
-  @vio        in
-  @endvar
+   @routine    DumpGA
+   @date       July 1998
+   @author     Paul Walker
+   @desc
+               Dumps a grid array variable into a IEEEIO file.
+   @enddesc
+
+   @calls      GetDumpData
 
-  @history 
-  @endhistory 
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      const cGH *
+   @vio        in
+   @endvar
+   @var        vindex
+   @vdesc      global index of the variable to be dumped
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        timelevel
+   @vdesc      the timelevel to store
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        file
+   @vdesc      the IEEEIO file to dump to
+   @vtype      IOFile
+   @vio        in
+   @endvar
+   @var        info
+   @vdesc      info structure describing
+                 - the IOFlexIO datatype to store
+                 - the size of an element of variable
+                 - the MPI datatype to communicate
+   @vtype      dumpInfo_t
+   @vio        in
+   @endvar
 
+   @returntype int
+   @returndesc
+               0 for success
+               or returncode of @seeroutine GetDumpData
+   @endreturndesc
 @@*/
-static void IOFlexIO_DumpGA (const cGH *GH, int vindex, int timelevel, IOFile iof,
-                             dumpInfo_t *info)
+static int DumpGA (const cGH *GH, int vindex, int timelevel, IOFile file,
+                   dumpInfo_t *info)
 {
-  DECLARE_CCTK_PARAMETERS
-  ioGH *ioUtilGH;
+  int myproc, nprocs, dim, free_outme, retval;
   pGH *pughGH;
-  int myproc;
-  int nprocs;
-  int dim;
-  CCTK_INT4 *geom;      /* Lower bounds, size and global size of data we send */
-  void *outme;          /* pointer to the data to be dumped */
-  int free_outme;       /* indicates whether data buffer needs freeing */
+  ioGH *ioUtilGH;
+  CCTK_INT4 *geom;
+  void *outme;
 #ifdef CCTK_MPI
-  int i, j;
+  int i, j, incoming, outgoing;
+  int *local_size;
   MPI_Status ms;
+  void *tmpd;
 #endif
+  DECLARE_CCTK_PARAMETERS
 
 
-  /* Get the handles for PUGH and IO extensions */
+  /* get the handles for PUGH and IO extensions */
   pughGH = PUGH_pGH (GH);
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
 
   myproc = CCTK_MyProc (GH);
   nprocs = CCTK_nProcs (GH);
@@ -372,183 +385,198 @@ static void IOFlexIO_DumpGA (const cGH *GH, int vindex, int timelevel, IOFile io
   /* allocate the geometry buffer */
   geom = (CCTK_INT4 *) malloc (3*dim * sizeof (CCTK_INT4));
 
-  /* Get the pointer to the data we want to output (includes downsampling) */
-  IOFlexIO_getDumpData (GH, vindex, timelevel, &outme, &free_outme, geom,
+  /* get the pointer to the data we want to output (includes downsampling) */
+  retval = GetDumpData (GH, vindex, timelevel, &outme, &free_outme, geom,
                         info->element_size);
-
-  if (ioUtilGH->ioproc_every == 1) {
-  
-    /* 
-     * OUTPUT ON EVERY PROCESSOR
-     */
-
-    IOFlexIO_eachProcDump (GH, vindex, timelevel, outme, geom, iof, info->flexio_type);
-
+  if (retval < 0)
+  {
+    return (retval);
   }
 
+  if (ioUtilGH->ioproc_every == 1)
+  {
+    EachProcDump (GH, vindex, timelevel, outme, geom, file, info->flexio_type);
+  }
 #ifdef CCTK_MPI
-  else if (myproc != ioUtilGH->ioproc) {
-
-    int outgoing = geom [dim];
-
-    for (i = 1; i < dim; i++)
-      outgoing *= geom [dim + i];
-
-    /* 
-     * OUTPUT ON A SUBSET OF PROCESSORS
-     */
-
-    /* Send the geometry and data from the processors which aren't 
-     * outputing to the ones that are 
+  else if (myproc != ioUtilGH->ioproc)
+  {
+    /*
+     * Send the geometry and data from non-IO processors to the IO processors
      */
+    outgoing = 1;
+    for (i = 0; i < dim; i++)
+    {
+      outgoing *= geom[dim + i];
+    }
 
-    /* GEOMETRY SEND */
+    /* send geometry */
     CACTUS_MPI_ERROR (MPI_Send (geom, 3*dim, PUGH_MPI_INT4, ioUtilGH->ioproc,
-                      2*myproc+IOTAGBASE+1, pughGH->PUGH_COMM_WORLD));
-    /* DATA SEND */
-    CACTUS_MPI_ERROR (MPI_Send (outme, outgoing, info->mpi_type, ioUtilGH->ioproc,
-                      2*myproc+IOTAGBASE, pughGH->PUGH_COMM_WORLD));
-
+                                2*myproc+IOTAGBASE+1, pughGH->PUGH_COMM_WORLD));
+    /* send data */
+    if (outgoing > 0)
+    {
+      CACTUS_MPI_ERROR (MPI_Send (outme, outgoing, info->mpi_type,
+                                  ioUtilGH->ioproc, 2*myproc+IOTAGBASE,
+                                  pughGH->PUGH_COMM_WORLD));
 #ifdef IOFLEXIO_DEBUG
-    printf ("Sent %d data points\n", outgoing);
-    fflush (stdout);
+      printf ("Processor %d sent %d data points\n", myproc, outgoing);
 #endif
-
-  } else if (myproc == ioUtilGH->ioproc) {
-
-    int *lsz = (int *) malloc (dim * sizeof (int));
-
-    /* First calculate the local size and reserve the chunk */
-    if (ioUtilGH->ioproc_every >= nprocs) {
-
-      /* One output file ... the local chunk size is the global size */
-      for (i = 0; i < dim; i++)
-        lsz [i] = geom [2*dim + i];
     }
+  }
+  else if (myproc == ioUtilGH->ioproc)
+  {
+    if (ioUtilGH->unchunked)
+    {
+      local_size = (int *) malloc (dim * sizeof (int));
 
-    /* Dump data held on output processor */
+      /* first calculate the local size and reserve the chunk */
+      if (ioUtilGH->ioproc_every >= nprocs)
+      {
+        /* one output file ... the local chunk size is the global size */
+        for (i = 0; i < dim; i++)
+        {
+          local_size[i] = geom[2*dim + i];
+        }
+      }
 
-    if (ioUtilGH->unchunked)
-      IOreserveChunk (iof, info->flexio_type, dim, lsz);
+      IOreserveChunk (file, info->flexio_type, dim, local_size);
 
-    free (lsz);
+      free (local_size);
+    }
 
-    /* first the data then the attributes */
-    IOFlexIO_procDump (iof, GH, vindex, outme, geom, info->flexio_type);
+    /* write the data first then the attributes */
+    ProcDump (file, GH, vindex, outme, geom, info->flexio_type);
     /* delay adding attributes for unchunked files
        until all chunks were written (otherwise attributes get lost !) */
     if (! ioUtilGH->unchunked)
-      IOFlexIO_AddCommonAttributes (GH, vindex, timelevel, &geom [2*dim], iof);
-
-    /* Dump data from all other processors */
-
-    for (i = myproc+1; i < myproc+ioUtilGH->ioproc_every && i < nprocs; i++) {
-
-      /* Allocate temp to the right size (based on GH->ub [i] et..) */
-      void *tmpd;
-      int incoming;
+    {
+      AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
+    }
 
-      /* Receive bounds information */
+    /* dump data from all other processors */
+    for (i = myproc+1; i < myproc+ioUtilGH->ioproc_every && i < nprocs; i++)
+    {
+      /* receive geometry */
       CACTUS_MPI_ERROR (MPI_Recv (geom, 3*dim, PUGH_MPI_INT4, i,2*i+IOTAGBASE+1,
-                        pughGH->PUGH_COMM_WORLD, &ms));
+                                  pughGH->PUGH_COMM_WORLD, &ms));
 
-      incoming = geom [dim];
-      for (j = 1; j < dim; j++)
-        incoming *= geom [dim + j];
+      incoming = 1;
+      for (j = 0; j < dim; j++)
+      {
+        incoming *= geom[dim + j];
+      }
 
-      tmpd = malloc (incoming * info->element_size);
-      CACTUS_MPI_ERROR (MPI_Recv (tmpd, incoming, info->mpi_type, i,
-                        2*i+IOTAGBASE, pughGH->PUGH_COMM_WORLD, &ms));
+      /* receive data */
+      if (incoming > 0)
+      {
+        tmpd = malloc (incoming * info->element_size);
+        CACTUS_MPI_ERROR (MPI_Recv (tmpd, incoming, info->mpi_type, i,
+                                    2*i+IOTAGBASE,pughGH->PUGH_COMM_WORLD,&ms));
+      }
+      else
+      {
+        tmpd = NULL;
+      }
 
-      IOFlexIO_procDump (iof, GH, vindex, tmpd, geom, info->flexio_type);
-      free (tmpd);
+      /* write data */
+      ProcDump (file, GH, vindex, tmpd, geom, info->flexio_type);
 
-    } /* End loop over processors */
+      if (tmpd)
+      {
+        free (tmpd);
+      }
+    } /* end loop over processors */
 
     /* now add the attributes for unchunked files */
     if (ioUtilGH->unchunked)
-      IOFlexIO_AddCommonAttributes (GH, vindex, timelevel, &geom [2*dim], iof);
-
-  } /* End myproc = 0 */
+    {
+      AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
+    }
+  }
 
+  /* wait for every processor to catch up */
   CCTK_Barrier (GH);
 #endif /* MPI */
 
   if (free_outme)
+  {
     free (outme);
+  }
 
   free (geom);
+
+  return (retval);
 }
 
 
 /*@@
-   @routine    IOFlexIO_AddCommonAttributes
+   @routine    AddCommonAttributes
    @date       July 1998
    @author     Paul Walker
-   @desc 
-   Add "Common" attributes, these are the GF name, the current date,
-   simulation time, origin, bounding box, gridspacings (both downsampled
-   and evolution), global grid size, number of processors and iteration
-   number. Note that the datestamp should be turned of if you are byte 
-   comparing two output files.
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
-   @hdate Wed Sep  2 10:15:22 1998 @hauthor Tom Goodale
-   @hdesc Abstracted WRITE_ATTRIBUTE to merge in Szu-Wen's Panda changes. 
-   @hdate Apr 16 1999 @hauthor Thomas Radke
-   @hdesc Added attributes groupname, grouptype, ntimelevels,
-          and current timelevel to be stored
-   @hdate May 02 1999 @hauthor Thomas Radke
-   @hdesc Made chunked attribute nioprocs common so that it can be found by
-          the recombiner even for unchunked datasets (eg. SCALARs)
-   @hdate May 05 1999 @hauthor Thomas Radke
-   @hdesc Added "unchunked" attribute to distinguish between chunked/unchunked
-          output files
-   @endhistory 
+   @desc
+               Add "Common" attributes, these are the GF name, the current date,
+               simulation time, origin, bounding box, and gridspacings (both
+               downsampled and evolution).
+   @enddesc
+
+   @calls
+
+   @history
+   @hdate      Wed Sep  2 10:15:22 1998
+   @hauthor    Tom Goodale
+   @hdesc      Abstracted WRITE_ATTRIBUTE to merge in Szu-Wen's Panda changes.
+   @hdate      Apr 16 1999
+   @hauthor    Thomas Radke
+   @hdesc      Added attributes groupname, grouptype, ntimelevels,
+               and current timelevel to be stored
+   @hdate      May 02 1999
+   @hauthor    Thomas Radke
+   @hdesc      Made chunked attribute nioprocs common so that it can be found by
+               the recombiner even for unchunked datasets (eg. SCALARs)
+   @hdate      May 05 1999
+   @hauthor    Thomas Radke
+   @hdesc      Added "unchunked" attribute to distinguish between
+               chunked/unchunked output files
+   @endhistory
 @@*/
-static void IOFlexIO_AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
-                                          CCTK_INT4 *gsz, IOFile iof)
+static void AddCommonAttributes (const cGH *GH, int vindex, int timelevel,
+                                 CCTK_INT4 *gsz, IOFile file)
 {
-  DECLARE_CCTK_PARAMETERS
-  int dim, vdim;
-  CCTK_REAL *attr_real; /* buffer for writing doubles to IEEEIO */
-  CCTK_INT4 attr_int4;   /* buffer for writing an int to IEEEIO */
-  char *name, *gname;
+  int i, vdim;
+  CCTK_REAL *dtmp;
+  CCTK_INT4 itmp;
+  char *name;
   ioGH *ioUtilGH;
   char coord_system_name[20];
+  DECLARE_CCTK_PARAMETERS
 
 
-  /* Get the handle for IO extensions */
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  /* get the handle for IO extensions */
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
 
   /* get the dimension of the variable */
   vdim = CCTK_GroupDimFromVarI (vindex);
 
   name = CCTK_FullName (vindex);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "name", FLEXIO_CHAR,
-                       strlen (name) + 1, name));
+  FLEXIO_ERROR (IOwriteAttribute (file, "name", FLEXIO_CHAR,
+                                  strlen (name) + 1, name));
   free (name);
 
-  gname = CCTK_GroupNameFromVarI (vindex);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "groupname", FLEXIO_CHAR,
-                       strlen (gname) + 1, gname));
-  free (gname);
+  name = CCTK_GroupNameFromVarI (vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "groupname", FLEXIO_CHAR,
+                                  strlen (name) + 1, name));
+  free (name);
 
-  attr_int4 = CCTK_GroupTypeFromVarI (vindex);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "grouptype", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = CCTK_GroupTypeFromVarI (vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "grouptype", FLEXIO_INT4, 1, &itmp));
 
-  attr_int4 = CCTK_NumTimeLevelsFromVarI (vindex);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "ntimelevels", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = CCTK_NumTimeLevelsFromVarI (vindex);
+  FLEXIO_ERROR (IOwriteAttribute (file, "ntimelevels", FLEXIO_INT4, 1, &itmp));
 
-  attr_int4 = timelevel;
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "timelevel", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = timelevel;
+  FLEXIO_ERROR (IOwriteAttribute (file, "timelevel", FLEXIO_INT4, 1, &itmp));
 
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "time", FLEXIO_REAL, 1,&GH->cctk_time));
+  FLEXIO_ERROR (IOwriteAttribute (file, "time", FLEXIO_REAL, 1,&GH->cctk_time));
 
   /* attributes describing the underlying grid
      These are only stored for CCTK_GF variables if they are associated
@@ -560,164 +588,157 @@ static void IOFlexIO_AddCommonAttributes (const cGH *GH, int vindex, int timelev
   if (CCTK_GroupTypeFromVarI (vindex) == CCTK_GF &&
       CCTK_CoordSystemHandle (coord_system_name) >= 0)
   {
-    attr_real = (CCTK_REAL *) malloc (3 * vdim * sizeof (CCTK_REAL));
-    for (dim = 0; dim < vdim; dim++)
+    dtmp = (CCTK_REAL *) malloc (3 * vdim * sizeof (CCTK_REAL));
+    for (i = 0; i < vdim; i++)
     {
-      CCTK_CoordRange (GH, &attr_real[dim], &attr_real[dim + vdim], dim + 1,
+      CCTK_CoordRange (GH, &dtmp[i], &dtmp[i + vdim], i + 1,
                        NULL, coord_system_name);
-      attr_real [dim + 2*vdim] = GH->cctk_delta_space [dim] *
-                                 ioUtilGH->downsample [dim];
+      dtmp[i + 2*vdim] = GH->cctk_delta_space[i] * ioUtilGH->downsample[i];
     }
 
-    CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "origin", FLEXIO_REAL, vdim,
-                         attr_real));
-    CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "min_ext", FLEXIO_REAL, vdim,
-                         attr_real));
-    CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "max_ext", FLEXIO_REAL, vdim,
-                         attr_real + vdim));
-    CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "delta", FLEXIO_REAL, vdim,
-                         attr_real + 2*vdim));
-
-    if (ioUtilGH->downsample [0] > 1 ||
-        ioUtilGH->downsample [1] > 1 ||
-        ioUtilGH->downsample [2] > 1)
+    FLEXIO_ERROR (IOwriteAttribute (file, "origin", FLEXIO_REAL, vdim, dtmp));
+    FLEXIO_ERROR (IOwriteAttribute (file, "min_ext", FLEXIO_REAL, vdim, dtmp));
+    FLEXIO_ERROR (IOwriteAttribute (file, "max_ext", FLEXIO_REAL, vdim,
+                                    dtmp + vdim));
+    FLEXIO_ERROR (IOwriteAttribute (file, "delta", FLEXIO_REAL, vdim,
+                                    dtmp + 2*vdim));
+
+    if (ioUtilGH->downsample[0] > 1 ||
+        ioUtilGH->downsample[1] > 1 ||
+        ioUtilGH->downsample[2] > 1)
     {
-      CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "evolution_delta", FLEXIO_REAL,
-                           vdim, GH->cctk_delta_space));
+      FLEXIO_ERROR (IOwriteAttribute (file, "evolution_delta", FLEXIO_REAL,
+                                      vdim, GH->cctk_delta_space));
     }
 
-    free (attr_real);
+    free (dtmp);
   }
 
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "global_size", FLEXIO_INT4,
-                       vdim, gsz));
+  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4, vdim, gsz));
 
-  attr_int4 = CCTK_nProcs (GH);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "nprocs", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = CCTK_nProcs (GH);
+  FLEXIO_ERROR (IOwriteAttribute (file, "nprocs", FLEXIO_INT4, 1, &itmp));
 
-  attr_int4 = ioUtilGH->ioproc_every;
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "ioproc_every", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = ioUtilGH->ioproc_every;
+  FLEXIO_ERROR (IOwriteAttribute (file, "ioproc_every", FLEXIO_INT4, 1, &itmp));
 
-  attr_int4 = ioUtilGH->unchunked;
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "unchunked", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = ioUtilGH->unchunked;
+  FLEXIO_ERROR (IOwriteAttribute (file, "unchunked", FLEXIO_INT4, 1, &itmp));
 
-  attr_int4 = GH->cctk_iteration;
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "iteration", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = GH->cctk_iteration;
+  FLEXIO_ERROR (IOwriteAttribute (file, "iteration", FLEXIO_INT4, 1, &itmp));
 }
 
 
 /*@@
-  @routine IOFlexIO_AddChunkAttributes
-  @author  Paul Walker 
-  @date    Feb 1997
+  @routine     AddChunkAttributes
+  @author      Paul Walker
+  @date        Feb 1997
   @desc
-  This routine adds chunk attributes to a data set. That is,
-  is adds attributes to the chunks of data passed to the
-  io processors. 
+               This routine adds chunk attributes to a data set. That is,
+               is adds attributes to the chunks of data passed to the
+               I/O processors.
   @enddesc
-  @history 
-  @hauthor Gabrielle Allen
-  @hdate Jul 10 1998
-  @hdesc Added the name of the grid function 
-  @hdate Wed Sep  2 10:08:31 1998 @hauthor Tom Goodale
-  @hdesc Abstracted WRITE_ATTRIBUTE to merge in Szu-Wen's Panda calls.
-  @endhistory 
+  @history
+  @hauthor     Gabrielle Allen
+  @hdate       Jul 10 1998
+  @hdesc       Added the name of the grid function
+  @hdate       Wed Sep  2 10:08:31 1998
+  @hauthor     Tom Goodale
+  @hdesc       Abstracted WRITE_ATTRIBUTE to merge in Szu-Wen's Panda calls.
+  @endhistory
 @@*/
-static void IOFlexIO_AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
-                                         IOFile iof)
+static void AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom,
+                                IOFile file)
 {
   int vdim;
   char *name;
-  CCTK_INT4 attr_int4;
+  CCTK_INT4 itmp;
 
 
   /* there is nothing to do for a serial run */
   if (CCTK_nProcs (GH) == 1)
+  {
     return;
+  }
 
   /* get the dimension of the variable */
   vdim = CCTK_GroupDimFromVarI (vindex);
 
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "chunk_origin", FLEXIO_INT4,
-                       vdim, &geom [0]));
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "subchunk_lb", FLEXIO_INT4,
-                       vdim, &geom [0]));
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "global_size", FLEXIO_INT4,
-                       vdim, &geom [2*vdim]));
+  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_origin", FLEXIO_INT4, vdim,
+                                  &geom[0]));
+  FLEXIO_ERROR (IOwriteAttribute (file, "subchunk_lb", FLEXIO_INT4, vdim,
+                                  &geom[0]));
+  FLEXIO_ERROR (IOwriteAttribute (file, "global_size", FLEXIO_INT4, vdim,
+                                  &geom[2*vdim]));
 
-  attr_int4 = GH->cctk_iteration;
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "chunk_dataset", FLEXIO_INT4,
-                       1, &attr_int4));
+  itmp = GH->cctk_iteration;
+  FLEXIO_ERROR (IOwriteAttribute (file, "chunk_dataset", FLEXIO_INT4, 1,&itmp));
 
   name = CCTK_FullName (vindex);
-  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "name", FLEXIO_CHAR,
-                       strlen (name)+1, name));
+  FLEXIO_ERROR (IOwriteAttribute (file, "name", FLEXIO_CHAR,
+                                  strlen (name)+1, name));
   free (name);
-
 }
 
 
 /*@@
-  @routine IOFlexIO_getDumpData
-  @author  Paul Walker 
+  @routine GetDumpData
+  @author  Paul Walker
   @date    Feb 1997
   @desc
   Bounds and data to be output, takes into account downsampling
   @enddesc
-  @history 
+  @history
   @hauthor Gabrielle Allen @hdate Oct 5 1998
   @hdesc Made into subroutine
-  @endhistory 
+  @endhistory
   @var     GH
   @vdesc   Identifies grid hierachy
   @vtype   const cGH *
   @vio     in
-  @vcomment 
-  @endvar 
+  @vcomment
+  @endvar
   @var     vindex
   @vdesc   index of the variable to dump
   @vtype   int
   @vio     in
-  @vcomment 
-  @endvar 
+  @vcomment
+  @endvar
   @var     timelevel
   @vdesc   timelevel of the variable to dump
   @vtype   int
   @vio     in
-  @vcomment 
-  @endvar 
+  @vcomment
+  @endvar
   @var     outme
   @vdesc   data segment to output
   @vtype   void **
   @vio     out
   @vcomment This is just GF->data if there is no downsampling
-  @endvar 
+  @endvar
   @var     free_outme
   @vdesc   Specifies whether or not to free the storage associated with outme
   @vtype   int *
   @vio     out
   @vcomment 1: free storage; 0: don't free storage
-  @endvar 
+  @endvar
   @var     geom
   @vdesc   bounds, local size and global shape of the output
-  @vtype   CCTK_INT4 [3*dim]
+  @vtype   CCTK_INT4[3*dim]
   @vio     out
-  @vcomment geom [0*dim..1*dim-1] lower bounds and geom[3],geom[4],geom[5] 
-            geom [1*dim..2*dim-1] local size of data to send
-            geom [2*dim..3*dim-1] global shape
-  @endvar 
+  @vcomment geom[0*dim..1*dim-1] lower bounds and geom[3],geom[4],geom[5]
+            geom[1*dim..2*dim-1] local size of data to send
+            geom[2*dim..3*dim-1] global shape
+  @endvar
   @var     element_size
   @vdesc   the size of an element of variable
   @vtype   int
   @vio     in
   @endvar
 @@*/
-static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
-              void **outme, int *free_outme, CCTK_INT4 *geom, int element_size)
+static int GetDumpData (const cGH *GH, int vindex, int timelevel, void **outme,
+                        int *free_outme, CCTK_INT4 *geom, int element_size)
 {
   DECLARE_CCTK_PARAMETERS
   int i, myproc, do_downsample, dim;
@@ -739,7 +760,7 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
   myproc = CCTK_MyProc (GH);
 
   /* get GH extensions for IO */
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  ioUtilGH = (ioGH *) GH->extensions[CCTK_GHExtensionHandle ("IO")];
 
   /* get the pGExtras pointer as a shortcut */
   extras = ((pGA ***)PUGH_pGH (GH)->variables)[vindex][timelevel]->extras;
@@ -749,7 +770,7 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
 
   do_downsample = 0;
   for (i = 0; i < dim; i++)
-    do_downsample |= ioUtilGH->downsample [i] > 1;
+    do_downsample |= ioUtilGH->downsample[i] > 1;
 
   /* All the downsampling code is hard-coded for 3D data.
      For other-dimensional variables we print a warning
@@ -769,84 +790,84 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
   if (! do_downsample)
   {
 
-    if (ioUtilGH->out_single) 
+    if (ioUtilGH->out_single)
     {
       single_ptr = (CCTK_REAL4 *) malloc (extras->npoints*sizeof (CCTK_REAL4));
 
       for (i = 0; i < extras->npoints; i++)
       {
-        single_ptr [i] = (CCTK_REAL4) ((CCTK_REAL *) data) [i];
+        single_ptr[i] = (CCTK_REAL4) ((CCTK_REAL *) data)[i];
       }
 
       *outme = single_ptr;
       *free_outme = 1;
-    } 
-    else 
+    }
+    else
     {
       *outme = data;
       *free_outme = 0;
     }
 
-    for (i = 0; i < dim; i++) 
+    for (i = 0; i < dim; i++)
     {
-      geom [i + 0*dim] = extras->lb [myproc][i];;     /* the bounds */
-      geom [i + 1*dim] = extras->lnsize [i];          /* the sizes */
-      geom [i + 2*dim] = extras->nsize [i];           /* the global space */
+      geom[i + 0*dim] = extras->lb[myproc][i];;     /* the bounds */
+      geom[i + 1*dim] = extras->lnsize[i];          /* the sizes */
+      geom[i + 2*dim] = extras->nsize[i];           /* the global space */
     }
 
   } else {
 
     /* NOTE: the following downsampling code is hard-coded for 3D data */
 
-    int start [3], end [3];
+    int start[3], end[3];
     int j, k, l;
 
     /* Downsampling code ... */
     for (i = 0; i < 3; i++) {
-      geom [i + 6] = extras->nsize [i] / ioUtilGH->downsample [i];
-      if (extras->nsize [i] % ioUtilGH->downsample [i])
-        geom [i + 6]++;
+      geom[i + 6] = extras->nsize[i] / ioUtilGH->downsample[i];
+      if (extras->nsize[i] % ioUtilGH->downsample[i])
+        geom[i + 6]++;
     }
 
-    if (verbose) 
+    if (verbose)
       CCTK_VInfo (CCTK_THORNSTRING, "Downsampled sizes (%d, %d, %d) -> "
                   "(%d, %d, %d)",
-                  extras->nsize [0], extras->nsize [1], extras->nsize [2],
-                  (int) geom [6], (int) geom [7], (int) geom [8]);
-    
+                  extras->nsize[0], extras->nsize[1], extras->nsize[2],
+                  (int) geom[6], (int) geom[7], (int) geom[8]);
+
     /* Now figure out the local downsampling */
     /* The local starts are the lb modded into the downsample */
     for (i = 0; i < 3; i++) {
-      geom [i]   = extras->lb [myproc][i] / ioUtilGH->downsample [i];
-      start [i] = geom [i] * ioUtilGH->downsample [i];
-      if (start [i] <
-          extras->lb [myproc][i] + extras->ownership [PUGH_NO_STAGGER][0][i]) {
-        start [i] += ioUtilGH->downsample [i];
-        geom [i] ++;
+      geom[i]   = extras->lb[myproc][i] / ioUtilGH->downsample[i];
+      start[i] = geom[i] * ioUtilGH->downsample[i];
+      if (start[i] <
+          extras->lb[myproc][i] + extras->ownership[PUGH_NO_STAGGER][0][i]) {
+        start[i] += ioUtilGH->downsample[i];
+        geom[i] ++;
       }
-      end [i]   = ((extras->lb [myproc][i] +
-                    extras->ownership [PUGH_NO_STAGGER][1][i] - 1) /
-                   ioUtilGH->downsample [i]) * ioUtilGH->downsample [i];
-      geom [i+3] = (end [i] - start [i]) / ioUtilGH->downsample [i] + 1;
+      end[i]   = ((extras->lb[myproc][i] +
+                    extras->ownership[PUGH_NO_STAGGER][1][i] - 1) /
+                   ioUtilGH->downsample[i]) * ioUtilGH->downsample[i];
+      geom[i+3] = (end[i] - start[i]) / ioUtilGH->downsample[i] + 1;
     }
 
     if (verbose) {
       CCTK_VInfo (CCTK_THORNSTRING, "Downsample ranges (%d, %d, %d) -> "
-                  "(%d, %d, %d)", start [0], start [1], start [2],
-                  end [0], end [1], end [2]);
+                  "(%d, %d, %d)", start[0], start[1], start[2],
+                  end[0], end[1], end[2]);
       CCTK_VInfo (CCTK_THORNSTRING, "Local size/bound  (%d, %d, %d)  "
                   "(%d, %d, %d)",
-                  (int) geom [3], (int) geom [4], (int) geom [5],
-                  (int) geom [0], (int) geom [1], (int) geom [2]);
+                  (int) geom[3], (int) geom[4], (int) geom[5],
+                  (int) geom[0], (int) geom[1], (int) geom[2]);
     }
 
     /* compute local ranges */
     for (i = 0; i < 3; i++) {
-      start [i] -= extras->lb [myproc][i];
-      end [i] -= extras->lb [myproc][i];
+      start[i] -= extras->lb[myproc][i];
+      end[i] -= extras->lb[myproc][i];
     }
 
-    *outme = malloc (geom [3] * geom [4] * geom [5] * element_size);
+    *outme = malloc (geom[3] * geom[4] * geom[5] * element_size);
     *free_outme = 1;
 
     /* I hate it to repeat the loops for each case label
@@ -855,18 +876,18 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
     switch (CCTK_VarTypeI (vindex)) {
       case CCTK_VARIABLE_CHAR:
         char_ptr = (CCTK_BYTE *) *outme;
-        for (k = start [2]; k <= end [2]; k += ioUtilGH->downsample [2])
-          for (j = start [1]; j <= end [1]; j += ioUtilGH->downsample [1])
-            for (i = start [0]; i <= end [0]; i += ioUtilGH->downsample [0])
-              char_ptr [l++] = ((CCTK_BYTE *) data) [DATINDEX (extras, i, j, k)];
+        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
+          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
+            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
+              char_ptr[l++] = ((CCTK_BYTE *) data)[DATINDEX (extras, i, j, k)];
         break;
 
       case CCTK_VARIABLE_INT:
         int_ptr = (CCTK_INT *) *outme;
-        for (k = start [2]; k <= end [2]; k += ioUtilGH->downsample [2])
-          for (j = start [1]; j <= end [1]; j += ioUtilGH->downsample [1])
-            for (i = start [0]; i <= end [0]; i += ioUtilGH->downsample [0])
-              int_ptr [l++] = ((CCTK_INT *) data) [DATINDEX (extras, i, j, k)];
+        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
+          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
+            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
+              int_ptr[l++] = ((CCTK_INT *) data)[DATINDEX (extras, i, j, k)];
         break;
 
       case CCTK_VARIABLE_REAL:
@@ -874,15 +895,15 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
           single_ptr = (CCTK_REAL4 *) *outme;
         else
           real_ptr = (CCTK_REAL *) *outme;
-        for (k = start [2]; k <= end [2]; k += ioUtilGH->downsample [2])
-          for (j = start [1]; j <= end [1]; j += ioUtilGH->downsample [1])
-            for (i = start [0]; i <= end [0]; i += ioUtilGH->downsample [0])
+        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
+          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
+            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
               if (ioUtilGH->out_single)
-                single_ptr [l++] = (CCTK_REAL4)
-                      (((CCTK_REAL *) data) [DATINDEX (extras, i, j, k)]);
+                single_ptr[l++] = (CCTK_REAL4)
+                      (((CCTK_REAL *) data)[DATINDEX (extras, i, j, k)]);
               else
-                real_ptr [l++] =
-                      ((CCTK_REAL *) data) [DATINDEX (extras, i, j, k)];
+                real_ptr[l++] =
+                      ((CCTK_REAL *) data)[DATINDEX (extras, i, j, k)];
         break;
 
 #if 0
@@ -892,57 +913,60 @@ static void IOFlexIO_getDumpData (const cGH *GH, int vindex, int timelevel,
           single_ptr = (CCTK_REAL4 *) *outme;
         else
           cmplx_ptr = (CCTK_COMPLEX *) *outme;
-        for (k = start [2]; k <= end [2]; k += ioUtilGH->downsample [2])
-          for (j = start [1]; j <= end [1]; j += ioUtilGH->downsample [1])
-            for (i = start [0]; i <= end [0]; i += ioUtilGH->downsample [0])
+        for (k = start[2]; k <= end[2]; k += ioUtilGH->downsample[2])
+          for (j = start[1]; j <= end[1]; j += ioUtilGH->downsample[1])
+            for (i = start[0]; i <= end[0]; i += ioUtilGH->downsample[0])
               if (ioUtilGH->out_single) {
-                single_ptr [l++] = (CCTK_REAL4)
-                       (((CCTK_REAL *) data) [0 + DATINDEX (extras, i, j, k)]);
-                single_ptr [l++] = (CCTK_REAL4)
-                       (((CCTK_REAL *) data) [1 + DATINDEX (extras, i, j, k)]);
+                single_ptr[l++] = (CCTK_REAL4)
+                       (((CCTK_REAL *) data)[0 + DATINDEX (extras, i, j, k)]);
+                single_ptr[l++] = (CCTK_REAL4)
+                       (((CCTK_REAL *) data)[1 + DATINDEX (extras, i, j, k)]);
               } else
-                cmplx_ptr [l++] =
-                       ((CCTK_COMPLEX *) data) [DATINDEX (extras, i, j, k)];
+                cmplx_ptr[l++] =
+                       ((CCTK_COMPLEX *) data)[DATINDEX (extras, i, j, k)];
         break;
 #endif
 
       default:
-        CCTK_WARN (1, "Unsupported variable type in IOFlexIO_getDumpData");
-        return;
+        CCTK_WARN (1, "Unsupported variable type in GetDumpData");
+        return (-1);
     }
   }
 
 #ifdef IOFLEXIO_DEBUG
-  printf ("Lower bound: %d", (int) geom [0]);
+  printf ("Lower bound: %d", (int) geom[0]);
   for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom [i]);
+    printf (" %d", (int) geom[i]);
   printf ("\n");
-  printf ("Chunk size : %d", (int) geom [1*dim + 0]);
+  printf ("Chunk size : %d", (int) geom[1*dim + 0]);
   for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom [1*dim + i]);
+    printf (" %d", (int) geom[1*dim + i]);
   printf ("\n");
-  printf ("Global size: %d", (int) geom [2*dim + 0]);
+  printf ("Global size: %d", (int) geom[2*dim + 0]);
   for (i = 1; i < dim; i++)
-    printf (" %d", (int) geom [2*dim + i]);
+    printf (" %d", (int) geom[2*dim + i]);
   printf ("\n");
 #endif
+
+  return (0);
 }
 
 
 /*@@
-  @routine IOFlexIO_eachProcDump
-  @author  Paul Walker 
-  @date    Feb 1997
+  @routine     EachProcDump
+  @author      Paul Walker
+  @date        Feb 1997
   @desc
-  Dump data from each processor
+               Dump data from each processor
   @enddesc
-  @history 
-  @hauthor Gabrielle Allen @hdate Oct 5 1998
-  @hdesc Made into subroutine
-  @endhistory 
+  @history
+  @hauthor     Gabrielle Allen
+  @hdate       Oct 5 1998
+  @hdesc       Made into subroutine
+  @endhistory
 @@*/
-static void IOFlexIO_eachProcDump (const cGH *GH, int vindex, int timelevel,
-                void *outme, CCTK_INT4 *geom, IOFile iof, int flexio_type) 
+static void EachProcDump (const cGH *GH, int vindex, int timelevel, void *outme,
+                          CCTK_INT4 *geom, IOFile file, int flexio_type)
 {
   int i, dim, *chunk_dims;
 
@@ -952,68 +976,71 @@ static void IOFlexIO_eachProcDump (const cGH *GH, int vindex, int timelevel,
 
   chunk_dims = (int *) malloc (dim * sizeof (int));
 
-  /* So set up the local shape. */
+  /* set up the local shape */
   for (i = 0; i < dim; i++)
-    chunk_dims [i] = geom [dim + i];
+  {
+    chunk_dims[i] = geom[dim + i];
+  }
 
-  /* Dump the data */
-  CACTUS_IEEEIO_ERROR (IOwrite (iof, flexio_type, dim, chunk_dims, outme));
-  
-  /* Add attributes for global space */
-  IOFlexIO_AddCommonAttributes (GH, vindex, timelevel, &geom [2*dim], iof);
-  
-  /* Add chunk attributes */
-  IOFlexIO_AddChunkAttributes (GH, vindex, geom, iof);
+  /* dump the data */
+  FLEXIO_ERROR (IOwrite (file, flexio_type, dim, chunk_dims, outme));
+
+  /* add attributes for global space */
+  AddCommonAttributes (GH, vindex, timelevel, &geom[2*dim], file);
+
+  /* add chunk attributes */
+  AddChunkAttributes (GH, vindex, geom, file);
 
   free (chunk_dims);
 }
 
 
 /*@@
-  @routine IOFlexIO_procDump
-  @author  Paul Walker 
-  @date    Feb 1997
+  @routine     ProcDump
+  @author      Paul Walker
+  @date        Feb 1997
   @desc
-  Dump data 
+               Dump data
   @enddesc
-  @history 
-  @hauthor Gabrielle Allen @hdate Oct 5 1998
-  @hdesc Made into subroutine
-  @endhistory 
+  @history
+  @hauthor     Gabrielle Allen
+  @hdate       Oct 5 1998
+  @hdesc       Made into subroutine
+  @endhistory
 @@*/
 #ifdef CCTK_MPI
-static void IOFlexIO_procDump (IOFile iof, const cGH *GH, int vindex, void *outme,
-                               CCTK_INT4 *geom, int flexio_type)
+static void ProcDump (IOFile file, const cGH *GH, int vindex, void *outme,
+                      CCTK_INT4 *geom, int flexio_type)
 {
   ioGH *ioUtilGH;
-  int i, dim, *chunk_dims, *chunk_origin;      /* Chunk dim and origin */
+  int i, dim, *chunk_dims, *chunk_origin;
 
 
-  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
 
   /* get the dimension of the variable */
   dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
 
   chunk_origin = (int *) malloc (2*dim * sizeof (int));
   chunk_dims = chunk_origin + dim;
-  for (i = 0; i < dim; i++) {
-    chunk_origin [i] = geom [i];
-    chunk_dims [i] = geom [dim + i];
+  for (i = 0; i < dim; i++)
+  {
+    chunk_origin[i] = geom[i];
+    chunk_dims[i] = geom[dim + i];
   }
 
-  if (ioUtilGH->unchunked) {
-    
-    /* Unchunked data */
-    CACTUS_IEEEIO_ERROR (IOwriteChunk (iof, chunk_dims, chunk_origin, outme));
-    
-  } else {
-    
-    /* Chunked data */
-    CACTUS_IEEEIO_ERROR (IOwrite (iof, flexio_type, dim, chunk_dims, outme));
-
-    /* Write chunk attributes */
-    IOFlexIO_AddChunkAttributes (GH, vindex, geom, iof);
+  if (ioUtilGH->unchunked)
+  {
+    /* unchunked data */
+    FLEXIO_ERROR (IOwriteChunk (file, chunk_dims, chunk_origin, outme));
+  }
+  else
+  {
+    /* uhunked data */
+    FLEXIO_ERROR (IOwrite (file, flexio_type, dim, chunk_dims, outme));
 
+    /* write chunk attributes */
+    AddChunkAttributes (GH, vindex, geom, file);
   }
 
   free (chunk_origin);