CarpetIOHDF5: some code cleanup

darcs-hash:20041130150933-3fd61-b07a8e91c055082ff3ddebccf11a07d368c7b47c.gz

11 files changed, 1259 insertions, 2332 deletions

diff --git a/Carpet/CarpetIOHDF5/schedule.ccl b/Carpet/CarpetIOHDF5/schedule.ccl
index 5ef822470..588abcc0b 100644
--- a/Carpet/CarpetIOHDF5/schedule.ccl
+++ b/Carpet/CarpetIOHDF5/schedule.ccl
@@ -3,18 +3,18 @@
 
 storage: next_output_iteration next_output_time this_iteration
 
-schedule CarpetIOHDF5Startup at STARTUP after IOUtil_Startup
+schedule CarpetIOHDF5_Startup at STARTUP after IOUtil_Startup
 {
   LANG: C
 } "Startup routine"
 
-schedule CarpetIOHDF5Init at BASEGRID
+schedule CarpetIOHDF5_Init at BASEGRID
 {
   LANG: C
   OPTIONS: global
 } "Initialisation routine"
 
-schedule CarpetIOHDF5ReadData at INITIAL
+schedule CarpetIOHDF5_ReadData at INITIAL
 {
   LANG: C
   OPTIONS: level
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5.hh b/Carpet/CarpetIOHDF5/src/CarpetIOHDF5.hh
index e8c9c4651..6fbe76f11 100644
--- a/Carpet/CarpetIOHDF5/src/iohdf5.hh
+++ b/Carpet/CarpetIOHDF5/src/CarpetIOHDF5.hh
@@ -10,7 +10,6 @@
 
 #include "carpet.hh"
 
-#include "iohdf5.h"
 #include "CactusBase/IOUtil/src/ioutil_Utils.h"
 
 /* some macros for HDF5 group names */
@@ -137,6 +136,44 @@
         }
 
 
+/* CarpetIOHDF5 GH extension structure */
+typedef struct
+{
+  /* default number of times to output */
+  int out_every_default;
+
+  /* number of times to output for each variable */
+  CCTK_INT *out_every;
+
+  /* the last iteration output for each variable */
+  int *out_last;
+
+  /* list of variables to output */
+  char *out_vars;
+
+  /* I/O request description list (for all variables) */
+  ioRequest **requests;
+
+  /* directory in which to output */
+  char *out_dir;
+
+  /* timer array for checkpointing/recovery */
+  // int timers[IOHDF5_NUM_TIMERS];
+
+  /* flag to indicate request for timer output */
+  // int print_timing_info;
+
+  /* ring buffer for list of successfully created cp files */
+  int    cp_filename_index;
+  char **cp_filename_list;
+
+  /* iteration number of the last checkpoint */
+  int last_checkpoint_iteration;
+
+  /* hdf5 datatype for stupid complex variables; to be set at run time */
+  hid_t HDF5_COMPLEX, HDF5_COMPLEX8, HDF5_COMPLEX16, HDF5_COMPLEX32;
+  
+} CarpetIOHDF5GH;
 
 
 namespace CarpetIOHDF5 {
@@ -148,18 +185,9 @@ namespace CarpetIOHDF5 {
   extern vector<bool> do_truncate; // [var]
   extern vector<vector<vector<int> > > last_output; // [ml][rl][var]
   
-  void* SetupGH (tFleshConfig* const fc,
-		 const int convLevel, cGH* const cctkGH);
-  
-  int OutputGH (const cGH* const cctkGH);
   int WriteVar (const cGH* const cctkGH, const hid_t writer, const ioRequest* request,
 		   const int called_from_checkpoint);
-  int OutputVarAs (const cGH* const cctkGH, const char* const varname,
-		   const char* const alias);
 
-  int TimeToOutput (const cGH* const cctkGH, const int vindex);
-  int TriggerOutput (const cGH* const cctkGH, const int vindex);
-  
   int InputGH (const cGH* const cctkGH);
   int ReadVar (const cGH* const cctkGH, const int vindex,
 	       const hid_t currdataset, vector<ibset> &regions_read, 
@@ -169,9 +197,6 @@ namespace CarpetIOHDF5 {
 
   // auxiliary functions defined in iohdf5utils.cc
 
-  bool CheckForVariable (const char* const varlist, const int vindex);
-  void SetFlag (int index, const char* optstring, void* arg);
-  
   void WriteAttribute (const hid_t dataset, const char* name, int value);
   void WriteAttribute (const hid_t dataset, const char* name, const int* values, int nvalues);
   void WriteAttribute (const hid_t dataset, const char* name, double value);
@@ -191,7 +216,25 @@ namespace CarpetIOHDF5 {
   int GetnDatasets (const hid_t reader);
   void GetDatasetName (const hid_t reader, const int _index, char* name);
 
-  hid_t h5DataType(const cGH* const cctkGH, int cctk_type);
+  hid_t h5DataType (const cGH* const cctkGH, int cctk_type,
+                    int single_precision);
+
+extern "C" {
+
+// scheduled routines
+int CarpetIOHDF5_Startup (void);
+int CarpetIOHDF5_Init (const cGH* const);
+int CarpetIOHDF5_ReadData (const cGH* const);
+int CarpetIOHDF5_CloseFile (void);
+int CarpetIOHDF5_InitialDataCheckpoint (const cGH* const);
+int CarpetIOHDF5_EvolutionCheckpoint (const cGH* const);
+int CarpetIOHDF5_TerminationCheckpoint (const cGH* const);
+
+// routines registered for recovery
+int CarpetIOHDF5_Recover (cGH* cgh, const char *basefilename, int called_from);
+int CarpetIOHDF5_RecoverParameters (void);
+
+} // extern "C"
 
 } // namespace CarpetIOHDF5
 
diff --git a/Carpet/CarpetIOHDF5/src/Checkpoint.cc b/Carpet/CarpetIOHDF5/src/Checkpoint.cc
index 86808fc00..bcc04b174 100644
--- a/Carpet/CarpetIOHDF5/src/Checkpoint.cc
+++ b/Carpet/CarpetIOHDF5/src/Checkpoint.cc
@@ -19,7 +19,7 @@
 #include "cctk_Version.h"
 
 extern "C" {
-static const char* rcsid = "$Header: /home/cvs/carpet/Carpet/CarpetIOHDF5/src/Checkpoint.cc,v 1.4 2004/11/03 10:24:42 tradke Exp $";
+static const char* rcsid = "$Header:$";
 CCTK_FILEVERSION(Carpet_CarpetIOHDF5_Checkpoint_cc);
 }
 
@@ -34,8 +34,7 @@ CCTK_FILEVERSION(Carpet_CarpetIOHDF5_Checkpoint_cc);
 
 #include "carpet.hh"
 
-#include "iohdf5.hh"
-#include "iohdf5GH.h"
+#include "CarpetIOHDF5.hh"
 
 
 namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5.cc b/Carpet/CarpetIOHDF5/src/Output.cc
index 5904217b9..c29a7c1b6 100644
--- a/Carpet/CarpetIOHDF5/src/iohdf5.cc
+++ b/Carpet/CarpetIOHDF5/src/Output.cc
@@ -17,8 +17,8 @@
 #include "cctk_Parameters.h"
 
 extern "C" {
-  static const char* rcsid = "$Header: /home/cvs/carpet/Carpet/CarpetIOHDF5/src/iohdf5.cc,v 1.42 2004/10/08 13:30:18 cott Exp $";
-  CCTK_FILEVERSION(Carpet_CarpetIOHDF5_iohdf5_cc);
+  static const char* rcsid = "$Header:$";
+  CCTK_FILEVERSION(Carpet_CarpetIOHDF5_Output_cc);
 }
 
 #include "CactusBase/IOUtil/src/ioGH.h"
@@ -32,11 +32,11 @@ extern "C" {
 
 #include "carpet.hh"
 
-#include "iohdf5.hh"
-#include "iohdf5GH.h"
+#include "CarpetIOHDF5.hh"
 
 
-namespace CarpetIOHDF5 {
+namespace CarpetIOHDF5
+{
 
 using namespace std;
 using namespace Carpet;
@@ -47,91 +47,93 @@ vector<bool> do_truncate;     // [var]
 vector<vector<vector<int> > > last_output; // [ml][rl][var]
 
 
+// registered GH extension setup routine
+static void* SetupGH (tFleshConfig* const fleshconfig,
+                      const int convLevel, cGH* const cctkGH);
+
+// callbacks for CarpetIOHDF5's I/O method
+static int OutputGH (const cGH* const cctkGH);
+static int OutputVarAs (const cGH* const cctkGH, const char* const varname,
+                        const char* const alias);
+static int TimeToOutput (const cGH* const cctkGH, const int vindex);
+static int TriggerOutput (const cGH* const cctkGH, const int vindex);
+
+// add attributes to an HDF5 dataset
 static void AddAttributes (const cGH *const cctkGH, const char *fullname,
                            int vdim, int refinementlevel, int timelevel,
                            ibset::const_iterator bbox, hid_t dataset);
 
-void CarpetIOHDF5Startup (void)
+// callback for I/O parameter parsing routine
+static void SetFlag (int vindex, const char* optstring, void* arg);
+
+
+int CarpetIOHDF5_Startup (void)
 {
   DECLARE_CCTK_PARAMETERS
 
 
-  CCTK_RegisterBanner ("AMR 3D HDF5 I/O provided by CarpetIOHDF5");
+  CCTK_RegisterBanner ("AMR HDF5 I/O provided by CarpetIOHDF5");
 
-  int GHExtension = CCTK_RegisterGHExtension ("CarpetIOHDF5");
-  CCTK_RegisterGHExtensionSetupGH (GHExtension, SetupGH);
+  // initial I/O parameter check
+  vector<bool> flags (CCTK_NumVars ());
 
-  int IOMethod = CCTK_RegisterIOMethod ("IOHDF5");
-  CCTK_RegisterIOMethodOutputGH (IOMethod, OutputGH);
-  CCTK_RegisterIOMethodOutputVarAs (IOMethod, OutputVarAs);
-  CCTK_RegisterIOMethodTimeToOutput (IOMethod, TimeToOutput);
-  CCTK_RegisterIOMethodTriggerOutput (IOMethod, TriggerOutput);
-
-  /* initial I/O parameter check */
-  int numvars = CCTK_NumVars ();
-  vector<bool> flags(numvars);
-
-  if (CCTK_TraverseString (out3D_vars, SetFlag, &flags,CCTK_GROUP_OR_VAR) < 0)
+  if (CCTK_TraverseString (out3D_vars, SetFlag, &flags, CCTK_GROUP_OR_VAR) < 0)
   {
     CCTK_VWarn (strict_io_parameter_check ? 0 : 1,
                 __LINE__, __FILE__, CCTK_THORNSTRING,
                 "error while parsing parameter 'IOHDF5::out3D_vars'");
   }
 
+  const int GHExtension = CCTK_RegisterGHExtension ("CarpetIOHDF5");
+  CCTK_RegisterGHExtensionSetupGH (GHExtension, SetupGH);
+
+  const int IOMethod = CCTK_RegisterIOMethod ("IOHDF5");
+  CCTK_RegisterIOMethodOutputGH (IOMethod, OutputGH);
+  CCTK_RegisterIOMethodOutputVarAs (IOMethod, OutputVarAs);
+  CCTK_RegisterIOMethodTimeToOutput (IOMethod, TimeToOutput);
+  CCTK_RegisterIOMethodTriggerOutput (IOMethod, TriggerOutput);
+
+  // register CarpetIOHDF5's recovery routine
   if (IOUtil_RegisterRecover ("CarpetIOHDF5 recovery", Recover) < 0)
   {
     CCTK_WARN (1, "Failed to register " CCTK_THORNSTRING " recovery routine");
   }
-}
-
-
-
-int CarpetIOHDF5Init (const cGH* const cctkGH)
-{
-  DECLARE_CCTK_ARGUMENTS;
-
-  *this_iteration = -1;
-  *next_output_iteration = 0;
-  *next_output_time = cctk_time;
 
   return (0);
 }
 
 
-
-void* SetupGH (tFleshConfig* const fc,
-               const int convLevel, cGH* const cctkGH)
+static void* SetupGH (tFleshConfig* const fleshconfig,
+                      const int convLevel, cGH* const cctkGH)
 {
+  DECLARE_CCTK_ARGUMENTS;
   DECLARE_CCTK_PARAMETERS;
 
   CarpetIOHDF5GH* myGH;
 
   const void *dummy;
-  dummy = &fc;
+  dummy = &fleshconfig;
   dummy = &convLevel;
   dummy = &cctkGH;
   dummy = &dummy;
 
+  const int numvars = CCTK_NumVars ();
+
   // Truncate all files if this is not a restart
-  do_truncate.resize(CCTK_NumVars(), true);
+  do_truncate.resize(numvars, true);
 
   // No iterations have yet been output
   last_output.resize(mglevels);
-  for (int ml=0; ml<mglevels; ++ml) {
+  for (int ml=0; ml<mglevels; ++ml)
+  {
     last_output.at(ml).resize(maxreflevels);
-    for (int rl=0; rl<maxreflevels; ++rl) {
-      last_output.at(ml).at(rl).resize(CCTK_NumVars(), INT_MIN);
+    for (int rl=0; rl<maxreflevels; ++rl)
+    {
+      last_output.at(ml).at(rl).resize(numvars, INT_MIN);
     }
   }
 
-  // We register only once, ergo we get only one handle.  We store
-  // that statically, so there is no need to pass anything to
-  // Cactus.
-
-  /* allocate a new GH extension structure */
-
-  CCTK_INT numvars = CCTK_NumVars ();
-
+  // allocate a new GH extension structure
   myGH            = (CarpetIOHDF5GH*) malloc (sizeof (CarpetIOHDF5GH));
   myGH->out_last  = (int *) malloc (numvars * sizeof (int));
   myGH->requests  = (ioRequest **) calloc (numvars, sizeof (ioRequest *));
@@ -142,19 +144,16 @@ void* SetupGH (tFleshConfig* const fc,
 
   for (int i = 0; i < numvars; i++)
   {
-    myGH->out_last [i] = -1;
+    myGH->out_last[i] = -1;
   }
 
-  myGH->open_output_files = NULL;
-
   // Now set hdf5 complex datatypes
-  // Stolen from Thomas Radke
   HDF5_ERROR (myGH->HDF5_COMPLEX =
-            H5Tcreate (H5T_COMPOUND, sizeof (CCTK_COMPLEX)));
+              H5Tcreate (H5T_COMPOUND, sizeof (CCTK_COMPLEX)));
   HDF5_ERROR (H5Tinsert (myGH->HDF5_COMPLEX, "real",
-                       offsetof (CCTK_COMPLEX, Re), HDF5_REAL));
+                         offsetof (CCTK_COMPLEX, Re), HDF5_REAL));
   HDF5_ERROR (H5Tinsert (myGH->HDF5_COMPLEX, "imag",
-                       offsetof (CCTK_COMPLEX, Im), HDF5_REAL));
+                         offsetof (CCTK_COMPLEX, Im), HDF5_REAL));
 #ifdef CCTK_REAL4
   HDF5_ERROR (myGH->HDF5_COMPLEX8 =
               H5Tcreate (H5T_COMPOUND, sizeof (CCTK_COMPLEX8)));
@@ -179,37 +178,217 @@ void* SetupGH (tFleshConfig* const fc,
   HDF5_ERROR (H5Tinsert (myGH->HDF5_COMPLEX32, "imag",
                          offsetof (CCTK_COMPLEX32, Im), HDF5_REAL16));
 #endif
+
   return (myGH);
 }
 
 
-int OutputGH (const cGH* const cctkGH)
+int CarpetIOHDF5_Init (const cGH* const cctkGH)
 {
-  for (int vindex=0; vindex<CCTK_NumVars(); ++vindex)
+  DECLARE_CCTK_ARGUMENTS
+
+  *this_iteration = -1;
+  *next_output_iteration = 0;
+  *next_output_time = cctk_time;
+
+  return (0);
+}
+
+
+static int OutputGH (const cGH* const cctkGH)
+{
+  for (int vindex = CCTK_NumVars () - 1; vindex >= 0; vindex--)
   {
-    if (TimeToOutput(cctkGH, vindex))
+    if (TimeToOutput (cctkGH, vindex))
     {
-      TriggerOutput(cctkGH, vindex);
+      TriggerOutput (cctkGH, vindex);
     }
   }
-  return 0;
+
+  return (0);
+}
+
+
+static int TimeToOutput (const cGH* const cctkGH, const int vindex)
+{
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+
+  const int numvars = CCTK_NumVars();
+  assert (vindex>=0 && vindex<numvars);
+
+  if (CCTK_GroupTypeFromVarI (vindex) != CCTK_GF && ! do_global_mode)
+  {
+    return 0;
+  }
+
+  const char *myoutcriterion = CCTK_EQUALS (out3D_criterion, "default") ?
+                               out_criterion : out3D_criterion;
+
+  if (CCTK_EQUALS (myoutcriterion, "never"))
+  {
+    return (0);
+  }
+
+  // check whether to output at this iteration
+  bool output_this_iteration;
+
+  if (CCTK_EQUALS (myoutcriterion, "iteration"))
+  {
+    int myoutevery = out3D_every;
+    if (myoutevery == -2)
+    {
+      myoutevery = out_every;
+    }
+    if (myoutevery <= 0)
+    {
+      // output is disabled
+      output_this_iteration = false;
+    }
+    else if (cctk_iteration == *this_iteration)
+    {
+      // we already decided to output this iteration
+      output_this_iteration = true;
+    }
+    else if (cctk_iteration >= *next_output_iteration)
+    {
+      // it is time for the next output
+      output_this_iteration = true;
+      *next_output_iteration = cctk_iteration + myoutevery;
+      *this_iteration = cctk_iteration;
+    }
+    else
+    {
+      // we want no output at this iteration
+      output_this_iteration = false;
+    }
+  }
+  else if (CCTK_EQUALS (myoutcriterion, "divisor"))
+  {
+    int myoutevery = out3D_every;
+    if (myoutevery == -2)
+    {
+      myoutevery = out_every;
+    }
+    if (myoutevery <= 0)
+    {
+      // output is disabled
+      output_this_iteration = false;
+    }
+    else if ((cctk_iteration % myoutevery) == 0)
+    {
+      // we already decided to output this iteration
+      output_this_iteration = true;
+    }
+    else
+    {
+      // we want no output at this iteration
+      output_this_iteration = false;
+    }
+  }
+  else if (CCTK_EQUALS (myoutcriterion, "time"))
+  {
+    CCTK_REAL myoutdt = out3D_dt;
+    if (myoutdt == -2)
+    {
+      myoutdt = out_dt;
+    }
+    if (myoutdt < 0)
+    {
+      // output is disabled
+      output_this_iteration = false;
+    }
+    else if (myoutdt == 0)
+    {
+      // output all iterations
+      output_this_iteration = true;
+    }
+    else if (cctk_iteration == *this_iteration)
+    {
+      // we already decided to output this iteration
+      output_this_iteration = true;
+    }
+    else if (cctk_time / cctk_delta_time
+               >= *next_output_time / cctk_delta_time - 1.0e-12)
+    {
+      // it is time for the next output
+      output_this_iteration = true;
+      *next_output_time = cctk_time + myoutdt;
+      *this_iteration = cctk_iteration;
+    }
+    else
+    {
+      // we want no output at this iteration
+      output_this_iteration = false;
+    }
+  }
+  else
+  {
+    assert (0);
+  } // select output criterion
+
+  if (! output_this_iteration)
+  {
+    return 0;
+  }
+
+  vector<bool> flags(numvars, false);
+
+  CCTK_TraverseString (out3D_vars, SetFlag, &flags, CCTK_GROUP_OR_VAR);
+
+  if (! flags.at(vindex))
+  {
+    // This variable should not be output
+    return 0;
+  }
+
+  if (last_output.at(mglevel).at(reflevel).at(vindex) == cctk_iteration)
+  {
+    // Has already been output during this iteration
+    char* varname = CCTK_FullName(vindex);
+    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Skipping output for variable \"%s\", because this variable "
+                "has already been output during the current iteration -- "
+                "probably via a trigger during the analysis stage",
+                varname);
+    free (varname);
+    return 0;
+  }
+
+  assert (last_output.at(mglevel).at(reflevel).at(vindex) < cctk_iteration);
+
+  // Should be output during this iteration
+  return 1;
 }
 
 
-int OutputVarAs (const cGH* const cctkGH, const char* const varname,
-                 const char* const alias)
+static int TriggerOutput (const cGH* const cctkGH, const int vindex)
+{
+  char *fullname = CCTK_FullName (vindex);
+  const char *varname = CCTK_VarName (vindex);
+  const int retval = OutputVarAs (cctkGH, fullname, varname);
+  free (fullname);
+
+  last_output.at(mglevel).at(reflevel).at(vindex) = cctkGH->cctk_iteration;
+
+  return (retval);
+}
+
+
+static int OutputVarAs (const cGH* const cctkGH, const char* const fullname,
+                        const char* const alias)
 {
   DECLARE_CCTK_ARGUMENTS
   DECLARE_CCTK_PARAMETERS
 
   int numvars = CCTK_NumVars ();
-  vector<bool> flags (numvars);
+  vector<bool> flags (numvars, false);
 
-  if (CCTK_TraverseString (varname, SetFlag, &flags, CCTK_VAR) < 0)
+  if (CCTK_TraverseString (fullname, SetFlag, &flags, CCTK_VAR) < 0)
   {
     CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "error while parsing variable name '%s' (alias name '%s')",
-                varname, alias);
+                fullname, alias);
     return (-1);
   }
 
@@ -228,7 +407,7 @@ int OutputVarAs (const cGH* const cctkGH, const char* const varname,
   {
     CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "Cannot output variable '%s' because it has no storage",
-                varname);
+                fullname);
     return (0);
   }
 
@@ -287,7 +466,7 @@ int OutputVarAs (const cGH* const cctkGH, const char* const varname,
   {
     CCTK_VInfo (CCTK_THORNSTRING,
                 "Writing variable '%s' on mglevel %d reflevel %d",
-                varname, mglevel, reflevel);
+                fullname, mglevel, reflevel);
   }
   WriteVar (cctkGH, writer, request, 0);
 
@@ -336,24 +515,10 @@ int WriteVar (const cGH* const cctkGH, const hid_t writer,
 
   // If the user requested so, select single precision data output
   hid_t memdatatype, filedatatype;
-  HDF5_ERROR (memdatatype = filedatatype = h5DataType (cctkGH, group.vartype));
-  if (out_single_precision && ! called_from_checkpoint)
-  {
-    if (group.vartype == CCTK_VARIABLE_REAL)
-    {
-      HDF5_ERROR (filedatatype = h5DataType (cctkGH, CCTK_VARIABLE_REAL4));
-    }
-    else if (group.vartype == CCTK_VARIABLE_COMPLEX)
-    {
-      HDF5_ERROR (filedatatype = h5DataType (cctkGH, CCTK_VARIABLE_COMPLEX8));
-    }
-#ifdef CCTK_INT2
-    else if (group.vartype == CCTK_VARIABLE_INT)
-    {
-      HDF5_ERROR (filedatatype = h5DataType (cctkGH, CCTK_VARIABLE_INT2));
-    }
-#endif
-  }
+  HDF5_ERROR (memdatatype = h5DataType (cctkGH, group.vartype, 0));
+  HDF5_ERROR (filedatatype =
+              h5DataType (cctkGH, group.vartype,
+                          out_single_precision && ! called_from_checkpoint));
 
   const int myproc = CCTK_MyProc (cctkGH);
 
@@ -574,718 +739,40 @@ static void AddAttributes (const cGH *const cctkGH, const char *fullname,
   }
   WriteAttribute (dataset, "origin", origin, vdim);
   WriteAttribute (dataset, "delta", delta, vdim);
-#ifdef WRITE_ADDITIONAL_ATTRIBUTES
-  WriteAttribute (dataset, "min_ext", min_ext, vdim);
-  WriteAttribute (dataset, "max_ext", max_ext, vdim);
-  WriteAttribute (dataset, "level_timestep", cctk_iteration / reflevelfact);
-  WriteAttribute (dataset, "persistence", maxreflevelfact / reflevelfact);
-#endif
-
-  WriteAttribute (dataset, "time", cctk_time);
-  WriteAttribute (dataset, "timestep", cctk_iteration);
-
-#ifdef WRITE_ADDITIONAL_ATTRIBUTES
-  const int time_refinement = reflevelfact;
-  const vect<int, dim> spatial_refinement = reflevelfact;
-  WriteAttribute (dataset, "time_refinement", time_refinement);
-  WriteAttribute (dataset, "spatial_refinement", spatial_refinement, vdim);
-  WriteAttribute (dataset, "grid_placement_refinement", spatial_refinement, vdim);
-#endif
 
   vect<int, dim> iorigin = bbox->lower() / bbox->stride();
   WriteAttribute (dataset, "iorigin", &iorigin[0], vdim);
 
+  WriteAttribute (dataset, "time", cctk_time);
+  WriteAttribute (dataset, "timestep", cctk_iteration);
+
   // Legacy arguments
   WriteAttribute (dataset, "name", fullname);
-
-#ifdef WRITE_ADDITIONAL_ATTRIBUTES
-  // Group arguments
-  const int gindex = CCTK_GroupIndexFromVar (fullname);
-
-  WriteAttribute (dataset, "group_version", 1);
-  WriteAttribute (dataset, "group_fullname", fullname);
-  WriteAttribute (dataset, "group_varname",
-                  CCTK_VarName (CCTK_VarIndex (fullname)));
-  {
-    char * groupname = CCTK_GroupName (gindex);
-    assert (groupname);
-    WriteAttribute (dataset, "group_groupname", groupname);
-    free (groupname);
-  }
-  const char *group_grouptype = NULL;
-  switch (CCTK_GroupTypeI (gindex))
-  {
-    case CCTK_GF:     group_grouptype = "CCTK_GF"; break;
-    case CCTK_ARRAY:  group_grouptype = "CCTK_ARRAY"; break;
-    case CCTK_SCALAR: group_grouptype = "CCTK_SCALAR"; break;
-  }
-  assert (group_grouptype);
-  WriteAttribute (dataset, "group_grouptype", group_grouptype);
-  WriteAttribute (dataset, "group_dim", vdim);
-#endif
   WriteAttribute (dataset, "group_timelevel", -timelevel);
-#ifdef WRITE_ADDITIONAL_ATTRIBUTES
-  WriteAttribute (dataset, "group_numtimelevels", CCTK_NumTimeLevelsI(gindex));
 
   // Cactus arguments
-  WriteAttribute (dataset, "cctk_version", 1);
-  WriteAttribute (dataset, "cctk_dim", cctk_dim);
-  WriteAttribute (dataset, "cctk_iteration", cctk_iteration);
-  WriteAttribute (dataset, "cctk_gsh", cctk_gsh, dim);
-  WriteAttribute (dataset, "cctk_lsh", cctk_lsh, dim);
-  WriteAttribute (dataset, "cctk_lbnd", cctk_lbnd, dim);
-  WriteAttribute (dataset, "cctk_delta_time", cctk_delta_time);
-  WriteAttribute (dataset, "cctk_delta_space", cctk_delta_space, dim);
-  WriteAttribute (dataset, "cctk_origin_space", cctk_origin_space, dim);
-  WriteAttribute (dataset, "cctk_levfac", cctk_levfac, dim);
-  WriteAttribute (dataset, "cctk_levoff", cctk_levoff, dim);
-  WriteAttribute (dataset, "cctk_levoffdenom", cctk_levoffdenom, dim);
-  WriteAttribute (dataset, "cctk_timefac", cctk_timefac);
-  WriteAttribute (dataset, "cctk_convlevel", cctk_convlevel);
-  WriteAttribute (dataset, "cctk_convfac", cctk_convfac);
-  WriteAttribute (dataset, "cctk_time", cctk_time);
-#endif
   WriteAttribute (dataset, "cctk_bbox", cctk_bbox, 2*vdim);
   WriteAttribute (dataset, "cctk_nghostzones", cctk_nghostzones, vdim);
 
   // Carpet arguments
-#ifdef WRITE_ADDITIONAL_ATTRIBUTES
-  WriteAttribute (dataset, "carpet_version", 1);
-  WriteAttribute (dataset, "carpet_dim", dim);
-  WriteAttribute (dataset, "carpet_basemglevel", basemglevel);
-  WriteAttribute (dataset, "carpet_mglevels", mglevels);
-  WriteAttribute (dataset, "carpet_mgface", mgfact);
-  WriteAttribute (dataset, "carpet_reflevels", reflevels);
-  WriteAttribute (dataset, "carpet_reffact", reffact);
-  WriteAttribute (dataset, "carpet_map", Carpet::map);
-  WriteAttribute (dataset, "carpet_maps", maps);
-  WriteAttribute (dataset, "carpet_component", component);
-  WriteAttribute (dataset, "carpet_components",
-                  vhh.at(Carpet::map)->components(refinementlevel));
-#endif
   WriteAttribute (dataset, "carpet_mglevel", mglevel);
   WriteAttribute (dataset, "carpet_reflevel", refinementlevel);
 }
 
 
-int TimeToOutput (const cGH* const cctkGH, const int vindex)
+static void SetFlag (int vindex, const char* optstring, void* arg)
 {
-  DECLARE_CCTK_ARGUMENTS;
-  DECLARE_CCTK_PARAMETERS;
-
-  assert (vindex>=0 && vindex<CCTK_NumVars());
-
-  switch (CCTK_GroupTypeFromVarI (vindex))
-  {
-    case CCTK_SCALAR:
-    case CCTK_ARRAY:
-      if (! do_global_mode)
-      {
-        return 0;
-      }
-      break;
-    case CCTK_GF:
-      // do nothing
-      break;
-    default:
-      assert (0);
-  }
-
-  // check whether to output at this iteration
-  bool output_this_iteration;
-
-  const char *myoutcriterion = out3D_criterion;
-  if (CCTK_EQUALS(myoutcriterion, "default"))
-  {
-    myoutcriterion = out_criterion;
-  }
-
-  if (CCTK_EQUALS (myoutcriterion, "never"))
-  {
-    // Never output
-    output_this_iteration = false;
-  }
-  else if (CCTK_EQUALS (myoutcriterion, "iteration"))
-  {
-    int myoutevery = out3D_every;
-    if (myoutevery == -2)
-    {
-      myoutevery = out_every;
-    }
-    if (myoutevery <= 0)
-    {
-      // output is disabled
-      output_this_iteration = false;
-    }
-    else if (cctk_iteration == *this_iteration)
-    {
-      // we already decided to output this iteration
-      output_this_iteration = true;
-    }
-    else if (cctk_iteration >= *next_output_iteration)
-    {
-      // it is time for the next output
-      output_this_iteration = true;
-      *next_output_iteration = cctk_iteration + myoutevery;
-      *this_iteration = cctk_iteration;
-    }
-    else
-    {
-      // we want no output at this iteration
-      output_this_iteration = false;
-    }
-  }
-  else if (CCTK_EQUALS (myoutcriterion, "divisor"))
-  {
-    int myoutevery = out3D_every;
-    if (myoutevery == -2)
-    {
-      myoutevery = out_every;
-    }
-    if (myoutevery <= 0)
-    {
-      // output is disabled
-      output_this_iteration = false;
-    }
-    else if ((cctk_iteration % myoutevery) == 0)
-    {
-      // we already decided to output this iteration
-      output_this_iteration = true;
-    }
-    else
-    {
-      // we want no output at this iteration
-      output_this_iteration = false;
-    }
-  }
-  else if (CCTK_EQUALS (myoutcriterion, "time"))
-  {
-    CCTK_REAL myoutdt = out3D_dt;
-    if (myoutdt == -2)
-    {
-      myoutdt = out_dt;
-    }
-    if (myoutdt < 0)
-    {
-      // output is disabled
-      output_this_iteration = false;
-    }
-    else if (myoutdt == 0)
-    {
-      // output all iterations
-      output_this_iteration = true;
-    }
-    else if (cctk_iteration == *this_iteration)
-    {
-      // we already decided to output this iteration
-      output_this_iteration = true;
-    }
-    else if (cctk_time / cctk_delta_time
-               >= *next_output_time / cctk_delta_time - 1.0e-12)
-    {
-      // it is time for the next output
-      output_this_iteration = true;
-      *next_output_time = cctk_time + myoutdt;
-      *this_iteration = cctk_iteration;
-    }
-    else
-    {
-      // we want no output at this iteration
-      output_this_iteration = false;
-    }
-  }
-  else
-  {
-    assert (0);
-  } // select output criterion
-
-  if (! output_this_iteration)
-  {
-    return 0;
-  }
-
-  if (! CheckForVariable(out3D_vars, vindex))
-  {
-    // This variable should not be output
-    return 0;
-  }
-
-  if (last_output.at(mglevel).at(reflevel).at(vindex) == cctk_iteration)
-  {
-    // Has already been output during this iteration
-    char* varname = CCTK_FullName(vindex);
-    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Skipping output for variable \"%s\", because this variable "
-                "has already been output during the current iteration -- "
-                "probably via a trigger during the analysis stage",
-                varname);
-    free (varname);
-    return 0;
-  }
-
-  assert (last_output.at(mglevel).at(reflevel).at(vindex) < cctk_iteration);
-
-  // Should be output during this iteration
-  return 1;
-}
-
-
-int TriggerOutput (const cGH* const cctkGH, const int vindex)
-{
-  assert (vindex>=0 && vindex<CCTK_NumVars());
-
-  char *fullname = CCTK_FullName (vindex);
-  const char *varname = CCTK_VarName (vindex);
-  const int retval = OutputVarAs (cctkGH, fullname, varname);
-  free (fullname);
-
-  last_output.at(mglevel).at(reflevel).at(vindex) = cctkGH->cctk_iteration;
-
-  return retval;
-}
-
-
-int ReadVar (const cGH* const cctkGH, const int vindex,
-             const hid_t dataset, vector<ibset> &regions_read,
-             const int called_from_recovery)
-{
-  DECLARE_CCTK_PARAMETERS;
-
-  const int group = CCTK_GroupIndexFromVarI (vindex);
-  assert (group>=0 && group<(int)Carpet::arrdata.size());
-  char *fullname = CCTK_FullName (vindex);
-  const int n0 = CCTK_FirstVarIndexI(group);
-  assert (n0>=0 && n0<CCTK_NumVars());
-  const int var = vindex - n0;
-  assert (var>=0 && var<CCTK_NumVars());
-  int tl = 0;
-
-  bool did_read_something = false;
-
-  // Stuff needed for Recovery
-
-  void *h5data = NULL;
-
-  if (verbose)
-  {
-    CCTK_VInfo (CCTK_THORNSTRING, "  reading '%s'", fullname);
-  }
-
-  // Check for storage
-  if (! CCTK_QueryGroupStorageI(cctkGH, group))
-  {
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Cannot input variable \"%s\" because it has no storage",
-                fullname);
-    free (fullname);
-    return 0;
-  }
-
-  const int grouptype = CCTK_GroupTypeI(group);
-  if ((grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY) && reflevel > 0)
+  if (optstring)
   {
+    char *fullname = CCTK_FullName (vindex);
+    CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Option string '%s' will be ignored for HDF5 output of "
+                "variable '%s'", optstring, fullname);
     free (fullname);
-    return 0;
   }
-
-  const int gpdim = CCTK_GroupDimI(group);
-
-
-  int intbuffer[2 + 2*dim];
-  int &group_timelevel = intbuffer[0],
-      &amr_level       = intbuffer[1];
-  int *amr_origin      = &intbuffer[2],
-      *amr_dims        = &intbuffer[2+dim];
-
-  if (CCTK_MyProc(cctkGH)==0)
-  {
-    // get dataset dimensions
-    hid_t dataspace;
-    HDF5_ERROR (dataspace = H5Dget_space (dataset));
-    int rank = (int) H5Sget_simple_extent_ndims (dataspace);
-    assert (0 < rank && rank <= dim);
-    assert ((grouptype == CCTK_SCALAR ? gpdim+1 : gpdim) == rank);
-    vector<hsize_t> shape(rank);
-    HDF5_ERROR (H5Sget_simple_extent_dims (dataspace, &shape[0], NULL));
-    HDF5_ERROR (H5Sclose (dataspace));
-
-    for (int i = 0; i < dim; i++)
-    {
-      amr_dims[i]   = 1;
-      amr_origin[i] = 0;
-    }
-    int datalength = 1;
-    for (int i = 0; i < rank; i++)
-    {
-      datalength *= shape[i];
-      amr_dims[i] = shape[rank-i-1];
-    }
-
-    const int cctkDataType = CCTK_VarTypeI(vindex);
-    const hid_t datatype = h5DataType(cctkGH,cctkDataType);
-
-    //cout << "datalength: " << datalength << " rank: " << rank << "\n";
-    //cout << shape[0] << " " << shape[1] << " " << shape[2] << "\n";
-
-    // to do: read in an allocate with correct datatype
-
-    h5data = malloc (CCTK_VarTypeSize (cctkDataType) * datalength);
-    HDF5_ERROR (H5Dread (dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT,
-                         h5data));
-
-    ReadAttribute (dataset, "level", amr_level);
-    ReadAttribute (dataset, "iorigin", amr_origin, rank);
-
-    if(called_from_recovery)
-    {
-      ReadAttribute(dataset,"group_timelevel", group_timelevel);
-    }
-  } // MyProc == 0
-
-  MPI_Bcast (intbuffer, sizeof (intbuffer) / sizeof (*intbuffer), MPI_INT, 0, dist::comm);
-
-#ifdef CARPETIOHDF5_DEBUG
-  cout << "amr_level: " << amr_level << " reflevel: " << reflevel << endl;
-#endif
-
-  if (amr_level == reflevel)
-  {
-    // Traverse all components on all levels
-    BEGIN_MAP_LOOP (cctkGH, grouptype)
-    {
-      BEGIN_COMPONENT_LOOP (cctkGH, grouptype)
-      {
-        did_read_something = true;
-
-        ggf<dim>* ff = 0;
-
-        assert (var < (int)arrdata.at(group).at(Carpet::map).data.size());
-        ff = (ggf<dim>*)arrdata.at(group).at(Carpet::map).data.at(var);
-
-        if(called_from_recovery) tl = group_timelevel;
-
-        gdata<dim>* const data = (*ff) (tl, reflevel, component, mglevel);
-
-        // Create temporary data storage on processor 0
-        vect<int,dim> str = vect<int,dim>(maxreflevelfact/reflevelfact);
-
-        if(grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY)
-          str = vect<int,dim> (1);
-
-        vect<int,dim> lb = vect<int,dim>::ref(amr_origin) * str;
-        vect<int,dim> ub
-          = lb + (vect<int,dim>::ref(amr_dims) - 1) * str;
-
-        gdata<dim>* const tmp = data->make_typed (vindex);
-
-
-        cGroup cgdata;
-        int ierr = CCTK_GroupData(group,&cgdata);
-        assert(ierr==0);
-        //cout << "lb_before: " << lb << endl;
-        //cout << "ub_before: " << ub << endl;
-        if (cgdata.disttype == CCTK_DISTRIB_CONSTANT) {
-#ifdef CARPETIOHDF5_DEBUG
-          cout << "CCTK_DISTRIB_CONSTANT: " << fullname << endl;
-#endif
-          assert(grouptype == CCTK_ARRAY || grouptype == CCTK_SCALAR);
-          if (grouptype == CCTK_SCALAR)
-          {
-            lb[0] = arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component);
-            ub[0] = arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component);
-            for(int i=1;i<dim;i++)
-            {
-              lb[i]=0;
-              ub[i]=0;
-            }
-          }
-          else
-          {
-            const int newlb = lb[gpdim-1] +
-              (ub[gpdim-1]-lb[gpdim-1]+1)*
-              (arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component));
-            const int newub = ub[gpdim-1] +
-              (ub[gpdim-1]-lb[gpdim-1]+1)*
-              (arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component));
-            lb[gpdim-1] = newlb;
-            ub[gpdim-1] = newub;
-          }
-#ifdef CARPETIOHDF5_DEBUG
-          cout << "lb: " << lb << endl;
-          cout << "ub: " << ub << endl;
-#endif
-        }
-        const bbox<int,dim> ext(lb,ub,str);
-
-#ifdef CARPETIOHDF5_DEBUG
-        cout << "ext: " << ext << endl;
-#endif
-
-        if (CCTK_MyProc(cctkGH)==0)
-        {
-          tmp->allocate (ext, 0, h5data);
-        }
-        else
-        {
-          tmp->allocate (ext, 0);
-        }
-
-        // Initialise with what is found in the file -- this does
-        // not guarantee that everything is initialised.
-        const bbox<int,dim> overlap = tmp->extent() & data->extent();
-        regions_read.at(Carpet::map) |= overlap;
-
-#ifdef CARPETIOHDF5_DEBUG
-        cout << "working on component: " << component << endl;
-        cout << "tmp->extent " << tmp->extent() << endl;
-        cout << "data->extent " << data->extent() << endl;
-        cout << "overlap " << overlap << endl;
-        cout << "-----------------------------------------------------" << endl;
-#endif
-
-        // FIXME: is this barrier really necessary ??
-        MPI_Barrier(MPI_COMM_WORLD);
-
-        // Copy into grid function
-        for (comm_state<dim> state; !state.done(); state.step())
-        {
-          data->copy_from (state, tmp, overlap);
-        }
-
-
-        // Delete temporary copy
-        delete tmp;
-
-
-      } END_COMPONENT_LOOP;
-
-      if (called_from_recovery)
-      {
-        arrdata.at(group).at(Carpet::map).tt->set_time(reflevel,mglevel,
-                (CCTK_REAL) ((cctkGH->cctk_time - cctk_initial_time)
-                             / (delta_time * mglevelfact)) );
-      }
-
-
-    } END_MAP_LOOP;
-
-  } // if amr_level == reflevel
-
-  free (h5data);
-  free (fullname);
-
-  return did_read_something;
-}
-
-
-static int InputVarAs (const cGH* const cctkGH, const int vindex,
-                       const char* const alias)
-{
-  DECLARE_CCTK_PARAMETERS;
-
-  char *fullname = CCTK_FullName (vindex);
-  const int group = CCTK_GroupIndexFromVarI (vindex);
-  assert (group>=0 && group<(int)Carpet::arrdata.size());
-
-  int want_dataset = 0;
-  bool did_read_something = false;
-  int ndatasets = 0;
-  hid_t dataset = 0;
-
-  char datasetname[1024];
-
-  // Check for storage
-  if (! CCTK_QueryGroupStorageI(cctkGH, group))
-  {
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Cannot input variable \"%s\" because it has no storage",
-                fullname);
-    free (fullname);
-    return 0;
-  }
-
-  const int grouptype = CCTK_GroupTypeI(group);
-  const int rl = grouptype==CCTK_GF ? reflevel : 0;
-  //cout << "want level " << rl << " reflevel " << reflevel << endl;
-
-  // Find the input directory
-  const char* const myindir = in3D_dir;
-
-  // Invent a file name
-  ostringstream filenamebuf;
-  filenamebuf << myindir << "/" << alias << in3D_extension;
-  string filenamestr = filenamebuf.str();
-  const char * const filename = filenamestr.c_str();
-
-  hid_t reader = -1;
-
-  // Read the file only on the root processor
-  if (CCTK_MyProc(cctkGH)==0)
-  {
-    // Open the file
-    if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Opening file \"%s\"", filename);
-    reader = H5Fopen (filename, H5F_ACC_RDONLY, H5P_DEFAULT);
-    if (reader<0)
-    {
-      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Could not open file \"%s\" for reading", filename);
-    }
-    assert (reader>=0);
-    // get the number of datasets in the file
-    ndatasets=GetnDatasets(reader);
-  }
-
-  vector<ibset> regions_read(Carpet::maps);
-
-  // Broadcast number of datasets
-  MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm);
-  assert (ndatasets>=0);
-
-
-  for (int datasetid=0; datasetid<ndatasets; ++datasetid)
-  {
-    if (verbose)
-    {
-      CCTK_VInfo (CCTK_THORNSTRING, "Handling dataset #%d", datasetid);
-    }
-
-    // Read data
-    if (CCTK_MyProc(cctkGH)==0)
-    {
-      GetDatasetName(reader,datasetid,datasetname);
-      //         cout << datasetname << "\n";
-
-      HDF5_ERROR (dataset = H5Dopen (reader, datasetname));
-    }
-
-
-#if 0
-    int amr_level;
-    int amr_origin[dim];
-    int amr_dims[dim];
-#endif
-
-    if (CCTK_MyProc(cctkGH)==0)
-    {
-      // Read data
-      char * name;
-      ReadAttribute (dataset, "name", name);
-      //        cout << "dataset name is " << name << endl;
-      if (verbose && name)
-      {
-        CCTK_VInfo (CCTK_THORNSTRING, "Dataset name is \"%s\"", name);
-      }
-      want_dataset = name && CCTK_EQUALS(name, fullname);
-      free (name);
-    } // myproc == 0
-
-    MPI_Bcast (&want_dataset, 1, MPI_INT, 0, dist::comm);
-
-    if(want_dataset)
-    {
-      did_read_something = ReadVar(cctkGH,vindex,dataset,regions_read,0);
-    } // want_dataset
-
-  } // loop over datasets
-
-  // Close the file
-  if (CCTK_MyProc(cctkGH)==0)
-  {
-    if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Closing file");
-    HDF5_ERROR (H5Fclose(reader));
-    reader=-1;
-  }
-
-  // Was everything initialised?
-  if (did_read_something)
-  {
-    for (int m=0; m<Carpet::maps; ++m)
-    {
-      dh<dim>& thedd = *arrdata.at(group).at(m).dd;
-      ibset all_exterior;
-      for (size_t c=0; c<thedd.boxes.at(rl).size(); ++c)
-      {
-        all_exterior |= thedd.boxes.at(rl).at(c).at(mglevel).exterior;
-      }
-      if (regions_read.at(m) != all_exterior)
-      {
-#ifdef CARPETIOHDF5_DEBUG
-        cout << "read: " << regions_read.at(m) << endl
-             << "want: " << all_exterior << endl;
-#endif
-        CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                    "Variable \"%s\" could not be initialised from file -- the file may be missing data",
-                    fullname);
-      }
-    }
-  } // if did_read_something
-  //        CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,"stop!");
-
-  return did_read_something ? 0 : -1;
-}
-
-
-int CarpetIOHDF5ReadData (const cGH* const cctkGH)
-{
-  int retval = 0;
-  DECLARE_CCTK_PARAMETERS
-
-
-  int numvars = CCTK_NumVars ();
-  vector<bool> flags (numvars);
-
-  if (CCTK_TraverseString (in3D_vars, SetFlag, &flags, CCTK_GROUP_OR_VAR) < 0)
-  {
-    CCTK_VWarn (strict_io_parameter_check ? 0 : 1,
-                __LINE__, __FILE__, CCTK_THORNSTRING,
-                "error while parsing parameter 'IOHDF5::in3D_vars'");
-  }
-
-  for (int vindex = 0; vindex < numvars && retval == 0; vindex++)
-  {
-    if (flags.at (vindex))
-    {
-      retval = InputVarAs (cctkGH, vindex, CCTK_VarName (vindex));
-    }
-  }
-
-  return (retval);
+  vector<bool>& flags = *(vector<bool>*)arg;
+  flags.at(vindex) = true;
 }
 
 
-
-#if 0
-/** returns the number of recovered variables */
-int Recover (cGH* const cctkGH, const char *basefilename,
-             const int called_from)
-{
-  assert (cctkGH);
-  assert (basefilename);
-  assert (called_from == CP_INITIAL_DATA
-          || called_from == CP_EVOLUTION_DATA
-          || called_from == CP_RECOVER_PARAMETERS
-          || called_from == CP_RECOVER_DATA
-          || called_from == FILEREADER_DATA);
-
-  // the other modes are not supported yet
-  assert (called_from == FILEREADER_DATA);
-
-  const ioGH * const iogh = (const ioGH *) CCTK_GHExtension (cctkGH, "IO");
-  assert (iogh);
-
-  int num_vars_read = 0;
-  assert (iogh->do_inVars);
-  for (int n=0; n<CCTK_NumVars(); ++n) {
-    if (iogh->do_inVars[n]) {
-      const int ierr = InputVarAs (cctkGH, vindex, basefilename);
-      if (! ierr) {
-        ++ num_vars_read;
-      }
-    }
-  }
-
-  return num_vars_read;
-}
-#endif
-
 } // namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/Recover.cc b/Carpet/CarpetIOHDF5/src/Recover.cc
index 2203879df..fbf6b349c 100644
--- a/Carpet/CarpetIOHDF5/src/Recover.cc
+++ b/Carpet/CarpetIOHDF5/src/Recover.cc
@@ -20,7 +20,7 @@
 
 extern "C" {
 static const char* rcsid = "$Header:$";
-CCTK_FILEVERSION(Carpet_CarpetIOHDF5_iohdf5chckpt_recover_cc);
+CCTK_FILEVERSION(Carpet_CarpetIOHDF5_Recover_cc);
 }
 
 #include "CactusBase/IOUtil/src/ioGH.h"
@@ -34,8 +34,7 @@ CCTK_FILEVERSION(Carpet_CarpetIOHDF5_iohdf5chckpt_recover_cc);
 
 #include "carpet.hh"
 
-#include "iohdf5.hh"
-#include "iohdf5GH.h"
+#include "CarpetIOHDF5.hh"
 
 /* some macros for HDF5 group names */
 #define METADATA_GROUP "Parameters and Global Attributes"
@@ -90,6 +89,9 @@ static int OpenFile (const char *basefilename, file_t *file, int called_from);
 static int RecoverVariables (cGH* cctkGH, file_t *file);
 static herr_t ReadMetadata (hid_t group, const char *objectname, void *arg);
 
+// callback for I/O parameter parsing routine
+static void SetFlag (int vindex, const char* optstring, void* arg);
+
 
 // Register with the Cactus Recovery Interface
 int CarpetIOHDF5_RecoverParameters (void)
@@ -399,6 +401,407 @@ int Recover (cGH* cctkGH, const char *basefilename, int called_from)
 }
 
 
+int ReadVar (const cGH* const cctkGH, const int vindex,
+             const hid_t dataset, vector<ibset> &regions_read,
+             const int called_from_recovery)
+{
+  DECLARE_CCTK_PARAMETERS;
+
+  const int group = CCTK_GroupIndexFromVarI (vindex);
+  assert (group>=0 && group<(int)Carpet::arrdata.size());
+  char *fullname = CCTK_FullName (vindex);
+  const int n0 = CCTK_FirstVarIndexI(group);
+  assert (n0>=0 && n0<CCTK_NumVars());
+  const int var = vindex - n0;
+  assert (var>=0 && var<CCTK_NumVars());
+  int tl = 0;
+
+  bool did_read_something = false;
+
+  // Stuff needed for Recovery
+
+  void *h5data = NULL;
+
+  if (verbose)
+  {
+    CCTK_VInfo (CCTK_THORNSTRING, "  reading '%s'", fullname);
+  }
+
+  // Check for storage
+  if (! CCTK_QueryGroupStorageI(cctkGH, group))
+  {
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Cannot input variable \"%s\" because it has no storage",
+                fullname);
+    free (fullname);
+    return 0;
+  }
+
+  const int grouptype = CCTK_GroupTypeI(group);
+  if ((grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY) && reflevel > 0)
+  {
+    free (fullname);
+    return 0;
+  }
+
+  const int gpdim = CCTK_GroupDimI(group);
+
+
+  int intbuffer[2 + 2*dim];
+  int &group_timelevel = intbuffer[0],
+      &amr_level       = intbuffer[1];
+  int *amr_origin      = &intbuffer[2],
+      *amr_dims        = &intbuffer[2+dim];
+
+  if (CCTK_MyProc(cctkGH)==0)
+  {
+    // get dataset dimensions
+    hid_t dataspace;
+    HDF5_ERROR (dataspace = H5Dget_space (dataset));
+    int rank = (int) H5Sget_simple_extent_ndims (dataspace);
+    assert (0 < rank && rank <= dim);
+    assert ((grouptype == CCTK_SCALAR ? gpdim+1 : gpdim) == rank);
+    vector<hsize_t> shape(rank);
+    HDF5_ERROR (H5Sget_simple_extent_dims (dataspace, &shape[0], NULL));
+    HDF5_ERROR (H5Sclose (dataspace));
+
+    for (int i = 0; i < dim; i++)
+    {
+      amr_dims[i]   = 1;
+      amr_origin[i] = 0;
+    }
+    int datalength = 1;
+    for (int i = 0; i < rank; i++)
+    {
+      datalength *= shape[i];
+      amr_dims[i] = shape[rank-i-1];
+    }
+
+    const int cctkDataType = CCTK_VarTypeI(vindex);
+    const hid_t datatype = h5DataType (cctkGH, cctkDataType, 0);
+
+    //cout << "datalength: " << datalength << " rank: " << rank << "\n";
+    //cout << shape[0] << " " << shape[1] << " " << shape[2] << "\n";
+
+    // to do: read in an allocate with correct datatype
+
+    h5data = malloc (CCTK_VarTypeSize (cctkDataType) * datalength);
+    HDF5_ERROR (H5Dread (dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT,
+                         h5data));
+
+    ReadAttribute (dataset, "level", amr_level);
+    ReadAttribute (dataset, "iorigin", amr_origin, rank);
+
+    if(called_from_recovery)
+    {
+      ReadAttribute(dataset,"group_timelevel", group_timelevel);
+    }
+  } // MyProc == 0
+
+  MPI_Bcast (intbuffer, sizeof (intbuffer) / sizeof (*intbuffer), MPI_INT, 0, dist::comm);
+
+#ifdef CARPETIOHDF5_DEBUG
+  cout << "amr_level: " << amr_level << " reflevel: " << reflevel << endl;
+#endif
+
+  if (amr_level == reflevel)
+  {
+    // Traverse all components on all levels
+    BEGIN_MAP_LOOP (cctkGH, grouptype)
+    {
+      BEGIN_COMPONENT_LOOP (cctkGH, grouptype)
+      {
+        did_read_something = true;
+
+        ggf<dim>* ff = 0;
+
+        assert (var < (int)arrdata.at(group).at(Carpet::map).data.size());
+        ff = (ggf<dim>*)arrdata.at(group).at(Carpet::map).data.at(var);
+
+        if(called_from_recovery) tl = group_timelevel;
+
+        gdata<dim>* const data = (*ff) (tl, reflevel, component, mglevel);
+
+        // Create temporary data storage on processor 0
+        vect<int,dim> str = vect<int,dim>(maxreflevelfact/reflevelfact);
+
+        if(grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY)
+          str = vect<int,dim> (1);
+
+        vect<int,dim> lb = vect<int,dim>::ref(amr_origin) * str;
+        vect<int,dim> ub
+          = lb + (vect<int,dim>::ref(amr_dims) - 1) * str;
+
+        gdata<dim>* const tmp = data->make_typed (vindex);
+
+
+        cGroup cgdata;
+        int ierr = CCTK_GroupData(group,&cgdata);
+        assert(ierr==0);
+        //cout << "lb_before: " << lb << endl;
+        //cout << "ub_before: " << ub << endl;
+        if (cgdata.disttype == CCTK_DISTRIB_CONSTANT) {
+#ifdef CARPETIOHDF5_DEBUG
+          cout << "CCTK_DISTRIB_CONSTANT: " << fullname << endl;
+#endif
+          assert(grouptype == CCTK_ARRAY || grouptype == CCTK_SCALAR);
+          if (grouptype == CCTK_SCALAR)
+          {
+            lb[0] = arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component);
+            ub[0] = arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component);
+            for(int i=1;i<dim;i++)
+            {
+              lb[i]=0;
+              ub[i]=0; 
+            }
+          }
+          else
+          {
+            const int newlb = lb[gpdim-1] + 
+              (ub[gpdim-1]-lb[gpdim-1]+1)*
+              (arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component));
+            const int newub = ub[gpdim-1] +
+              (ub[gpdim-1]-lb[gpdim-1]+1)*
+              (arrdata.at(group).at(Carpet::map).hh->processors.at(reflevel).at(component));
+            lb[gpdim-1] = newlb;
+            ub[gpdim-1] = newub;
+          }
+#ifdef CARPETIOHDF5_DEBUG
+          cout << "lb: " << lb << endl;
+          cout << "ub: " << ub << endl;
+#endif
+        }
+        const bbox<int,dim> ext(lb,ub,str);
+
+#ifdef CARPETIOHDF5_DEBUG
+        cout << "ext: " << ext << endl;
+#endif
+    
+        if (CCTK_MyProc(cctkGH)==0)
+        {
+          tmp->allocate (ext, 0, h5data);
+        }
+        else
+        {
+          tmp->allocate (ext, 0);
+        }
+
+        // Initialise with what is found in the file -- this does
+        // not guarantee that everything is initialised.
+        const bbox<int,dim> overlap = tmp->extent() & data->extent();
+        regions_read.at(Carpet::map) |= overlap;
+
+#ifdef CARPETIOHDF5_DEBUG
+        cout << "working on component: " << component << endl;
+        cout << "tmp->extent " << tmp->extent() << endl;
+        cout << "data->extent " << data->extent() << endl;
+        cout << "overlap " << overlap << endl;
+        cout << "-----------------------------------------------------" << endl;
+#endif
+
+        // FIXME: is this barrier really necessary ??
+        MPI_Barrier(MPI_COMM_WORLD);
+
+        // Copy into grid function
+        for (comm_state<dim> state; !state.done(); state.step())
+        {
+          data->copy_from (state, tmp, overlap);
+        }
+
+
+        // Delete temporary copy
+        delete tmp;
+
+
+      } END_COMPONENT_LOOP;
+
+      if (called_from_recovery)
+      {
+        arrdata.at(group).at(Carpet::map).tt->set_time(reflevel,mglevel,
+                (CCTK_REAL) ((cctkGH->cctk_time - cctk_initial_time)
+                             / (delta_time * mglevelfact)) );
+      }
+
+
+    } END_MAP_LOOP;
+
+  } // if amr_level == reflevel
+
+  free (h5data);
+  free (fullname);
+
+  return did_read_something;
+}
+
+
+static int InputVarAs (const cGH* const cctkGH, const int vindex,
+                       const char* const alias)
+{
+  DECLARE_CCTK_PARAMETERS;
+
+  char *fullname = CCTK_FullName (vindex);
+  const int group = CCTK_GroupIndexFromVarI (vindex);
+  assert (group>=0 && group<(int)Carpet::arrdata.size());
+
+  int want_dataset = 0;
+  bool did_read_something = false;
+  int ndatasets = 0;
+  hid_t dataset = 0;
+
+  char datasetname[1024];
+
+  // Check for storage
+  if (! CCTK_QueryGroupStorageI(cctkGH, group))
+  {
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Cannot input variable \"%s\" because it has no storage",
+                fullname);
+    free (fullname);
+    return 0;
+  }
+
+  const int grouptype = CCTK_GroupTypeI(group);
+  const int rl = grouptype==CCTK_GF ? reflevel : 0;
+  //cout << "want level " << rl << " reflevel " << reflevel << endl;
+
+  // Find the input directory
+  const char* const myindir = in3D_dir;
+
+  // Invent a file name
+  ostringstream filenamebuf;
+  filenamebuf << myindir << "/" << alias << in3D_extension;
+  string filenamestr = filenamebuf.str();
+  const char * const filename = filenamestr.c_str();
+
+  hid_t reader = -1;
+
+  // Read the file only on the root processor
+  if (CCTK_MyProc(cctkGH)==0)
+  {
+    // Open the file
+    if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Opening file \"%s\"", filename);
+    reader = H5Fopen (filename, H5F_ACC_RDONLY, H5P_DEFAULT);
+    if (reader<0)
+    {
+      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Could not open file \"%s\" for reading", filename);
+    }
+    assert (reader>=0);
+    // get the number of datasets in the file
+    ndatasets=GetnDatasets(reader);
+  }
+
+  vector<ibset> regions_read(Carpet::maps);
+
+  // Broadcast number of datasets
+  MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm);
+  assert (ndatasets>=0);
+
+
+  for (int datasetid=0; datasetid<ndatasets; ++datasetid)
+  {
+    if (verbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING, "Handling dataset #%d", datasetid);
+    }
+
+    // Read data
+    if (CCTK_MyProc(cctkGH)==0)
+    {
+      GetDatasetName(reader,datasetid,datasetname);
+      //         cout << datasetname << "\n";
+
+      HDF5_ERROR (dataset = H5Dopen (reader, datasetname));
+    }
+
+    if (CCTK_MyProc(cctkGH)==0)
+    {
+      // Read data 
+      char * name;
+      ReadAttribute (dataset, "name", name);
+      //        cout << "dataset name is " << name << endl;
+      if (verbose && name)
+      { 
+        CCTK_VInfo (CCTK_THORNSTRING, "Dataset name is \"%s\"", name);
+      }
+      want_dataset = name && CCTK_EQUALS(name, fullname);
+      free (name);
+    } // myproc == 0
+  
+    MPI_Bcast (&want_dataset, 1, MPI_INT, 0, dist::comm);
+
+    if(want_dataset)
+    {
+      did_read_something = ReadVar(cctkGH,vindex,dataset,regions_read,0);
+    } // want_dataset
+
+  } // loop over datasets
+  
+  // Close the file
+  if (CCTK_MyProc(cctkGH)==0)
+  {
+    if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Closing file");
+    HDF5_ERROR (H5Fclose(reader));
+    reader=-1;
+  }
+  
+  // Was everything initialised?
+  if (did_read_something)
+  {
+    for (int m=0; m<Carpet::maps; ++m)
+    {
+      dh<dim>& thedd = *arrdata.at(group).at(m).dd;
+      ibset all_exterior;
+      for (size_t c=0; c<thedd.boxes.at(rl).size(); ++c)
+      {
+        all_exterior |= thedd.boxes.at(rl).at(c).at(mglevel).exterior;
+      }
+      if (regions_read.at(m) != all_exterior)
+      {
+#ifdef CARPETIOHDF5_DEBUG
+        cout << "read: " << regions_read.at(m) << endl
+             << "want: " << all_exterior << endl;
+#endif
+        CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "Variable \"%s\" could not be initialised from file -- the file may be missing data",
+                    fullname);
+      }
+    }
+  } // if did_read_something
+  //        CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,"stop!");
+  
+  return did_read_something ? 0 : -1;
+}
+
+
+int CarpetIOHDF5_ReadData (const cGH* const cctkGH)
+{
+  int retval = 0;
+  DECLARE_CCTK_PARAMETERS
+
+
+  int numvars = CCTK_NumVars ();
+  vector<bool> flags (numvars);
+
+  if (CCTK_TraverseString (in3D_vars, SetFlag, &flags, CCTK_GROUP_OR_VAR) < 0)
+  {
+    CCTK_VWarn (strict_io_parameter_check ? 0 : 1,
+                __LINE__, __FILE__, CCTK_THORNSTRING,
+                "error while parsing parameter 'IOHDF5::in3D_vars'");
+  }
+
+  for (int vindex = 0; vindex < numvars && retval == 0; vindex++)
+  {
+    if (flags.at (vindex))
+    {
+      retval = InputVarAs (cctkGH, vindex, CCTK_VarName (vindex));
+    }
+  }
+
+  return (retval);
+}
+
+
 static herr_t ReadMetadata (hid_t group, const char *objectname, void *arg)
 {
   file_t *file = (file_t *) arg;
@@ -522,4 +925,19 @@ static int RecoverVariables (cGH* cctkGH, file_t *file)
 }
 
 
+static void SetFlag (int vindex, const char* optstring, void* arg)
+{
+  if (optstring)
+  {
+    char *fullname = CCTK_FullName (vindex);
+    CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Option string '%s' will be ignored for HDF5 input of "
+                "variable '%s'", optstring, fullname);
+    free (fullname);
+  }
+  vector<bool>& flags = *(vector<bool>*)arg;
+  flags.at(vindex) = true;
+}
+
+
 } // namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/Utils.cc b/Carpet/CarpetIOHDF5/src/Utils.cc
new file mode 100644
index 000000000..37a7104c4
--- /dev/null
+++ b/Carpet/CarpetIOHDF5/src/Utils.cc
@@ -0,0 +1,515 @@
+#include <assert.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <algorithm>
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+#include <hdf5.h>
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+extern "C" {
+static const char* rcsid = "$Header:$";
+CCTK_FILEVERSION(Carpet_CarpetIOHDF5_Utils_cc);
+}
+
+#include "CactusBase/IOUtil/src/ioGH.h"
+#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
+
+#include "bbox.hh"
+#include "data.hh"
+#include "gdata.hh"
+#include "ggf.hh"
+#include "vect.hh"
+
+#include "carpet.hh"
+
+#include "CarpetIOHDF5.hh"
+
+
+namespace CarpetIOHDF5 {
+
+using namespace std;
+using namespace Carpet;
+
+
+
+void WriteAttribute (const hid_t dataset, const char* const name, const int value)
+{
+  WriteAttribute (dataset, name, &value, 1);
+}
+
+void WriteAttribute (const hid_t dataset, const char* const name, const int* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  hsize_t shape[1];
+  shape[0] = nvalues;
+  const hid_t dataspace = nvalues==1 ? H5Screate (H5S_SCALAR) : H5Screate_simple (1, shape, NULL);
+  assert (dataspace>=0);
+  
+  const hid_t datatype = H5T_NATIVE_INT;
+  
+  const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
+  assert (attribute>=0);
+  herr = H5Awrite (attribute, datatype, values);
+  assert (!herr);
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+}
+
+
+
+void WriteAttribute (const hid_t dataset, const char* const name, const double value)
+{
+  WriteAttribute (dataset, name, &value, 1);
+}
+
+void WriteAttribute (const hid_t dataset, const char* const name, const double* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  hsize_t shape[1];
+  shape[0] = nvalues;
+  const hid_t dataspace = nvalues==1 ? H5Screate (H5S_SCALAR) : H5Screate_simple (1, shape, NULL);
+  assert (dataspace>=0);
+  
+  const hid_t datatype = H5T_NATIVE_DOUBLE;
+  
+  const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
+  assert (attribute>=0);
+  herr = H5Awrite (attribute, datatype, values);
+  assert (!herr);
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+}
+
+
+
+void WriteAttribute (const hid_t dataset, const char* const name, const char value)
+{
+  WriteAttribute (dataset, name, &value, 1);
+}
+
+void WriteAttribute (const hid_t dataset, const char* const name, const char* const values)
+{
+  WriteAttribute (dataset, name, values, strlen(values));
+}
+
+void WriteAttribute (const hid_t dataset, const char* const name, const char* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  const hid_t dataspace = H5Screate (H5S_SCALAR);
+  assert (dataspace>=0);
+  
+  const hid_t datatype = H5Tcopy (H5T_C_S1);
+  assert (datatype>=0);
+  herr = H5Tset_size (datatype, nvalues);
+  assert (!herr);
+  
+  const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
+  assert (attribute>=0);
+  herr = H5Awrite (attribute, datatype, values);
+  assert (!herr);
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  herr = H5Tclose (datatype);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+}
+
+
+
+int ReadAttribute (const hid_t dataset, const char* const name, int& value)
+{
+  return ReadAttribute (dataset, name, &value, 1);
+}
+
+int ReadAttribute (const hid_t dataset, const char* name, int* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  const hid_t attribute = H5Aopen_name (dataset, name);
+  if (attribute<0) return attribute;
+  
+  const hid_t dataspace = H5Aget_space (attribute);
+  assert (dataspace>=0);
+  
+  //    cout << "reading int attribute " << name << endl;
+
+  hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
+  hsize_t shape[1];
+  if (rank==0) {
+    shape[0] = 1;
+  } else if (rank==1) {
+    herr = H5Sget_simple_extent_dims (dataspace, shape, NULL);
+    assert (herr >= 0);
+  } else {
+    assert (0);
+  }
+  const int length = shape[0];
+  
+  const hid_t datatype = H5Aget_type (attribute);
+  assert (datatype>=0);
+
+  assert(H5Tequal(datatype, H5T_NATIVE_INT));
+  
+  vector<int> values1(length);
+  
+  herr = H5Aread (attribute, datatype, &values1.at(0));
+  assert (!herr);
+  
+  for (int i=0; i<min(length, nvalues); ++i) {
+    values[i] = values1[i];
+  }
+
+  herr = H5Tclose (datatype);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+  
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  return length;
+}
+
+
+
+int ReadAttribute (const hid_t dataset, const char* const name, double& value)
+{
+  return ReadAttribute (dataset, name, &value, 1);
+}
+
+int ReadAttribute (const hid_t dataset, const char* const name, double* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  const hid_t attribute = H5Aopen_name (dataset, name);
+  if (attribute<0) return attribute;
+  
+  const hid_t dataspace = H5Aget_space (attribute);
+  assert (dataspace>=0);
+  
+  hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
+  hsize_t shape[1];
+  if (rank==0) {
+    shape[0] = 1;
+  } else if (rank==1) {
+    rank = H5Sget_simple_extent_dims (dataspace, shape, NULL);
+    assert (rank == 1);
+  } else {
+    assert (0);
+  }
+  const int length = shape[0];
+  
+  const hid_t datatype = H5Aget_type (attribute);
+  assert (datatype>=0);
+  assert(H5Tequal(datatype, H5T_NATIVE_DOUBLE));
+  
+  vector<double> values1(length);
+  
+  herr = H5Aread (attribute, datatype, &values1.at(0));
+  assert (!herr);
+  
+  for (int i=0; i<min(length, nvalues); ++i) {
+    values[i] = values1[i];
+  }
+  
+  herr = H5Tclose (datatype);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+  
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  return length;
+}
+
+
+
+int ReadAttribute (const hid_t dataset, const char* const name, char& value)
+{
+  return ReadAttribute (dataset, name, &value, 1);
+}
+
+int ReadAttribute (const hid_t dataset, const char* const name, char*& values)
+{
+  assert (dataset>=0);
+  assert (name);
+  
+  herr_t herr;
+  
+  const hid_t attribute = H5Aopen_name (dataset, name);
+  if (attribute<0) return attribute;
+  
+  const hid_t dataspace = H5Aget_space (attribute);
+  assert (dataspace>=0);
+  
+  hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
+  assert (rank==0);
+  
+  const hid_t datatype = H5Aget_type (attribute);
+  assert (datatype>=0);
+  assert (H5Tget_class (datatype) == H5T_STRING);
+  const int length = H5Tget_size (datatype);
+  assert (length>=0);
+  
+  values = (char*) malloc (length+1);
+  assert (values);
+  
+  herr = H5Aread (attribute, datatype, values);
+  assert (!herr);
+  values[length] = '\0';
+  
+  herr = H5Tclose (datatype);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+  
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  return length;
+}
+
+int ReadAttribute (const hid_t dataset, const char* const name, char* const values, const int nvalues)
+{
+  assert (dataset>=0);
+  assert (name);
+  assert (values);
+  assert (nvalues>=0);
+  
+  herr_t herr;
+  
+  const hid_t attribute = H5Aopen_name (dataset, name);
+  if (attribute<0) return attribute;
+  
+  const hid_t dataspace = H5Aget_space (attribute);
+  assert (dataspace>=0);
+  
+  hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
+  assert (rank==0);
+  
+  const hid_t datatype = H5Aget_type (attribute);
+  assert (datatype>=0);
+  assert(H5Tget_class (datatype) == H5T_STRING); 
+  const int length = H5Tget_size (datatype);
+  assert (length>=0);
+  
+  vector<char> values1(length);
+  
+  herr = H5Aread (attribute, datatype, &values1.at(0));
+  assert (!herr);
+  
+  for (int i=0; i<min(length, nvalues); ++i) {
+    values[i] = values1[i];
+  }
+  
+  herr = H5Tclose (datatype);
+  assert (!herr);
+  
+  herr = H5Sclose (dataspace);
+  assert (!herr);
+  
+  herr = H5Aclose (attribute);
+  assert (!herr);
+  
+  return length;
+}
+
+herr_t DatasetCounter(hid_t group_id, const char *member_name, void *operator_data)
+  /* Counts datasets. Used by GetnDatasets; straight from John Shalf's FlexIO library */
+{
+  int *count = (int*)operator_data;
+  H5G_stat_t objinfo;
+  // request info about the type of objects in root group
+  if(H5Gget_objinfo(group_id,member_name,1 /* follow links */,&objinfo)<0) {
+    return 0; 
+  }
+  // only count objects that are datasets (not subgroups)
+  if(objinfo.type==H5G_DATASET) {
+    (*count)++;
+  }
+return 0;
+}
+
+
+int GetnDatasets(const hid_t reader)
+{
+  //this is straight from John Shalf's FlexIO library
+
+  int count=0;
+  int idx=0;
+  while(H5Giterate(reader, /* hid_t loc_id, */
+		     "/", /*const char *name, */
+		     &idx, /* int *idx, */
+		     DatasetCounter,
+		     &count)<0){}
+  return count;
+}
+
+struct H5IO_getname_t {
+  //this is straight from John Shalf's FlexIO library
+  int index,count;
+  char *name;
+};
+
+
+herr_t GetName(hid_t group_id, const char *member_name, void *operator_data)
+{
+  //this is straight from John Shalf's FlexIO library
+  H5IO_getname_t *getn = (H5IO_getname_t*)operator_data;
+  // check type first (only respond if it is a dataset)
+  H5G_stat_t objinfo;
+  // request info about the type of objects in root group
+  if(H5Gget_objinfo(group_id,
+		      member_name,
+		    1 /* follow links */,
+		      &objinfo)<0) return 0; // error (probably bad symlink)
+  // only count objects that are datasets (not subgroups)
+  if(objinfo.type!=H5G_DATASET)
+    return 0; // do not increment count if it isn't a dataset.
+  if(getn->index==getn->count){
+    strcpy(getn->name,member_name);
+    return 1; // success
+  }
+  getn->count++;
+  return 0;
+}
+
+
+void GetDatasetName(const hid_t reader, const int _index,  char *name) {
+  //this is straight from John Shalf's FlexIO library
+  H5IO_getname_t getn;
+  int idx=_index;
+  getn.index=_index; getn.name=name; getn.count=_index;
+  while(H5Giterate(reader, /* hid_t loc_id, */
+		     "/", /*const char *name, */
+		     &idx, /* int *idx, */
+		     GetName,
+		     &getn)<0){}
+}
+
+hid_t h5DataType (const cGH* const cctkGH, int cctk_type, int single_precision)
+{
+  hid_t retval;  
+
+  CarpetIOHDF5GH *myGH;
+  myGH = (CarpetIOHDF5GH *) CCTK_GHExtension (cctkGH, "CarpetIOHDF5");
+
+  // this is adapted from Thomas Radke's IOHDF5Util. Thanks, Thomas!
+
+  switch (cctk_type)
+  {
+    case CCTK_VARIABLE_CHAR:      retval = HDF5_CHAR; break;
+
+    case CCTK_VARIABLE_INT:       retval = HDF5_INT;
+#ifdef CCTK_INT2
+                                  if (single_precision)
+                                  {
+                                    retval = HDF5_INT2;
+                                  }
+#endif
+                                  break;
+
+    case CCTK_VARIABLE_REAL:      retval = HDF5_REAL;
+#ifdef CCTK_REAL4
+                                  if (single_precision)
+                                  {
+                                    retval = HDF5_REAL4;
+                                  }
+#endif
+                                  break;
+
+    case CCTK_VARIABLE_COMPLEX:   retval = myGH->HDF5_COMPLEX;
+#ifdef CCTK_REAL4
+                                  if (single_precision)
+                                  {
+                                    retval = retval = myGH->HDF5_COMPLEX8;
+                                  }
+#endif
+                                  break;
+
+#ifdef CCTK_INT1
+    case CCTK_VARIABLE_INT1:      retval = HDF5_INT1; break;
+#endif
+#ifdef CCTK_INT2
+    case CCTK_VARIABLE_INT2:      retval = HDF5_INT2; break;
+#endif
+#ifdef CCTK_INT4
+    case CCTK_VARIABLE_INT4:      retval = HDF5_INT4; break;
+#endif
+#ifdef CCTK_INT8
+    case CCTK_VARIABLE_INT8:      retval = HDF5_INT8; break;
+#endif
+#ifdef CCTK_REAL4
+    case CCTK_VARIABLE_REAL4:     retval = HDF5_REAL4; break;
+    case CCTK_VARIABLE_COMPLEX8:  retval = myGH->HDF5_COMPLEX8; break;
+#endif
+#ifdef CCTK_REAL8
+    case CCTK_VARIABLE_REAL8:     retval = HDF5_REAL8; break;
+    case CCTK_VARIABLE_COMPLEX16: retval = myGH->HDF5_COMPLEX16; break;
+#endif
+#ifdef CCTK_REAL16
+    case CCTK_VARIABLE_REAL16:    retval = HDF5_REAL16; break;
+    case CCTK_VARIABLE_COMPLEX32: retval = myGH->HDF5_COMPLEX32; break;
+#endif
+
+    default: CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                         "Unsupported CCTK variable datatype %d", cctk_type);
+             retval = -1;
+  }
+
+  return (retval);
+}
+
+
+} // namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5.h b/Carpet/CarpetIOHDF5/src/iohdf5.h
deleted file mode 100644
index 28cea9b57..000000000
--- a/Carpet/CarpetIOHDF5/src/iohdf5.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/* $Header:$ */
-
-#ifndef CARPETIOHDF5_H
-#define CARPETIOHDF5_H
-
-#ifdef __cplusplus
-namespace CarpetIOHDF5 {
-  extern "C" {
-#endif
-    
-#ifdef USE_CHRISTIANS_ROUTINES
-#include "cctk_Arguments.h"
-
-int CarpetIOHDF5Startup (void);
-void CarpetIOHDF5Init (CCTK_ARGUMENTS);
-void CarpetIOHDF5ReadData (CCTK_ARGUMENTS);
-void CarpetIOHDF5_EvolutionCheckpoint (const cGH*);
-void CarpetIOHDF5_InitialDataCheckpoint (const cGH*);
-
-#else
-
-/* Scheduled functions */
-void CarpetIOHDF5Startup (void);
-int CarpetIOHDF5Init (const cGH* const);
-int CarpetIOHDF5ReadData (const cGH* const);
-int CarpetIOHDF5_InitialDataCheckpoint (const cGH* const);
-int CarpetIOHDF5_EvolutionCheckpoint (const cGH* const);
-int CarpetIOHDF5_TerminationCheckpoint (const cGH* const);
-void CarpetIOHDF5EvolutionCheckpoint (const cGH* const);
-void CarpetIOHDF5InitialDataCheckpoint (const cGH* const);
-    
-#endif
-
-int CarpetIOHDF5_Recover (cGH* cgh, const char *basefilename, int called_from);
-
-int CarpetIOHDF5_RecoverParameters (void);
-int CarpetIOHDF5_CloseFile (void);
-
-#ifdef __cplusplus
-  } /* extern "C" */
-} /* namespace CarpetIOHDF5 */
-#endif
-
-#endif /* !defined(CARPETIOHDF5_H) */
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5GH.h b/Carpet/CarpetIOHDF5/src/iohdf5GH.h
deleted file mode 100644
index 0516201c2..000000000
--- a/Carpet/CarpetIOHDF5/src/iohdf5GH.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// This was adopted from Thomas Radke's IOHDF5 thorn.
-// Thanks, Thomas!
-
-#ifndef _CARPETIOHDF5_IOHDF5GH_H_
-#define _CARPETIOHDF5_IOHDF5GH_H_ 1
-
-#include "StoreNamedData.h"
-
-/* I took basically everything in this file from Thomas' IOHDF5; much of
-   below is still unused.. */
-
-/* CARPET IOHDF5 GH extension structure */
-typedef struct
-{
-  /* default number of times to output */
-  int out_every_default;
-
-  /* number of times to output for each variable */
-  CCTK_INT *out_every;
-
-  /* the last iteration output for each variable */
-  int *out_last;
-
-  /* list of variables to output */
-  char *out_vars;
-
-  /* I/O request description list (for all variables) */
-  ioRequest **requests;
-
-  /* directory in which to output */
-  char *out_dir;
-
-  /* filename database for opened files */
-  pNamedData *open_output_files;
-
-  /* timer array for checkpointing/recovery */
-  // int timers[IOHDF5_NUM_TIMERS];
-
-  /* flag to indicate request for timer output */
-  // int print_timing_info;
-
-  /* ring buffer for list of successfully created cp files */
-  int    cp_filename_index;
-  char **cp_filename_list;
-
-  /* iteration number of the last checkpoint */
-  int last_checkpoint_iteration;
-
-  /* hdf5 datatype for stupid complex variables; to be set at run time */
-  hid_t HDF5_COMPLEX, HDF5_COMPLEX8, HDF5_COMPLEX16, HDF5_COMPLEX32;
-  
-
-} CarpetIOHDF5GH;
-
-
-#endif  
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5chckpt_recover.cc b/Carpet/CarpetIOHDF5/src/iohdf5chckpt_recover.cc
deleted file mode 100644
index 035d05f4c..000000000
--- a/Carpet/CarpetIOHDF5/src/iohdf5chckpt_recover.cc
+++ /dev/null
@@ -1,914 +0,0 @@
-#include <assert.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include <algorithm>
-#include <fstream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include <hdf5.h>
-
-#include "cctk.h"
-#include "cctk_Parameters.h"
-#include "cctk_Version.h"
-
-extern "C" {
-  static const char* rcsid = "$Header:$";
-  CCTK_FILEVERSION(Carpet_CarpetIOHDF5_iohdf5chckpt_recover_cc);
-}
-
-#include "CactusBase/IOUtil/src/ioGH.h"
-#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
-
-#include "bbox.hh"
-#include "data.hh"
-#include "gdata.hh"
-#include "ggf.hh"
-#include "vect.hh"
-
-#include "carpet.hh"
-
-#include "iohdf5.hh"
-#include "iohdf5GH.h"
-
-/* some macros for HDF5 group names */
-#define PARAMETERS_GLOBAL_ATTRIBUTES_GROUP "Parameters and Global Attributes"
-#define ALL_PARAMETERS "All Parameters"
-
-
-
-namespace CarpetIOHDF5 {
-  
-  using namespace std;
-  using namespace Carpet;
-
-  // linked list for reading in the checkpoint file
-
-  list<string> datasetnamelist;
-  
-  int Checkpoint (const cGH* const cctkGH, int called_from);
-  int DumpParametersGHExtentions (const cGH *cctkGH, int all, hid_t writer);  
-  
-  int RecoverParameters (hid_t reader);
-  int RecoverGHextensions (cGH* cctkGH, hid_t reader);
-  int RecoverVariables (cGH* cctkGH, hid_t reader);
-
-  void CarpetIOHDF5InitialDataCheckpoint( const cGH* const cgh){
-    
-    DECLARE_CCTK_PARAMETERS;
-    
-    if (checkpoint && checkpoint_ID)
-      {
-	CCTK_INFO ("---------------------------------------------------------");
-	CCTK_INFO ("Dumping initial data checkpoint");
-	CCTK_INFO ("---------------------------------------------------------");
-	
-	Checkpoint (cgh, CP_INITIAL_DATA);
-      }
-  } // CarpetIOHDF5_InitialDataCheckpoint
-
-
-  void CarpetIOHDF5EvolutionCheckpoint( const cGH* const cgh){
-    
-    DECLARE_CCTK_PARAMETERS;
-
-    // Test checkpoint_every, and adjust it. This is necessary until 
-    // recovery is more flexible.
-    int every_full = pow(2.0,maxreflevels-1);
-    if (checkpoint_every>0 && (checkpoint_every % every_full) != 0) {
-      int every = (checkpoint_every / every_full + 1) * every_full;
-      ostringstream outevery;
-      outevery << every;
-      CCTK_ParameterSet("checkpoint_every","IOUtil",
-			outevery.str().c_str());
-      CCTK_VInfo (CCTK_THORNSTRING,"I have adjusted your checkpoint_every to %i.",every);
-      
-    }
-
-      if (checkpoint &&
-      ((checkpoint_every > 0 && cgh->cctk_iteration % checkpoint_every == 0) ||
-       checkpoint_next))
-      {
-	CCTK_INFO ("---------------------------------------------------------");
-	CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint at "
-		    "iteration %d", cgh->cctk_iteration);
-	CCTK_INFO ("---------------------------------------------------------");
-
-	Checkpoint (cgh, CP_EVOLUTION_DATA);
-
-       	if (checkpoint_next)
-	{
-	  CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no");
-	}
-      }
-  } // CarpetIOHDF5_EvolutionCheckpoint
-
-
-  int CarpetIOHDF5RecoverParameters(void){
-    // Register with the Cactus Recovery Interface
-    return (IOUtil_RecoverParameters (CarpetIOHDF5_Recover, ".h5", "HDF5"));
-  }
-
-  int CarpetIOHDF5_Recover (cGH* cctkGH, const char *basefilename, int called_from) {
-    
-    DECLARE_CCTK_PARAMETERS;
-
-    int result=0;
-    int myproc=-1;
-
-    CarpetIOHDF5GH *myGH;
-
-    
-    static hid_t reader=0; //this thing absolutely needs to be static!!!
-
-    myGH = NULL;
-
-    myproc = CCTK_MyProc (cctkGH);
-    
-  
-    if (called_from == CP_RECOVER_PARAMETERS) {
-      // Okay, let's see what we can do about the parameters
-    
-       // Invent a file name
-      ostringstream filenamebuf;
-
-      //      if(CCTK_nProcs(cctkGH) == 1)
-      filenamebuf << recover_dir << "/" << basefilename << ".h5";
-      //else
-      //	filenamebuf << recover_dir << "/" << basefilename << ".file_0.h5";
-
-      string filenamestr = filenamebuf.str();
-      const char * const filename = filenamestr.c_str();
-
-      if (myproc == 0) {
-	// First, open the file
-	if (verbose) 
-	  CCTK_VInfo(CCTK_THORNSTRING, "Opening Checkpoint file %s for recovery",filename);
-	reader = H5Fopen (filename, H5F_ACC_RDONLY, H5P_DEFAULT);
-	if (reader<0) {
-	  CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-		    "Could not recover from \"%s\"", filename);
-	}
-      } // myproc == 0
-    }
-    else {
-	/* This is the case for CP_RECOVER_DATA.
-	   CCTK_RECOVER_PARAMETERS must have been called before
-	   and set up the file info structure. */
-	if (myproc == 0) {
-	  assert(reader>=0);
-	}
-    }
-  
-    if (called_from == CP_RECOVER_PARAMETERS)
-      {
-	return (RecoverParameters (reader));
-      }
-  
-    if (called_from == CP_RECOVER_DATA) {
-      CCTK_INT4 numberofmgtimes=0;
-
-      CCTK_VInfo(CCTK_THORNSTRING,"Starting to recover data on reflevel %d!!!",reflevel);
-
-      if (myproc == 0) {
-	
-	// Use refinement levels parameter from checkpointing file?
-	if (use_reflevels_from_checkpoint && reflevel==0) {
-	  
-	  herr_t herr;
-	  
-	  hid_t group = H5Gopen (reader, PARAMETERS_GLOBAL_ATTRIBUTES_GROUP);
-	  assert(group >= 0);
-	  hid_t dataset = H5Dopen (group, ALL_PARAMETERS);
-	  assert(dataset>= 0);
-	  
-	  int reflevels_chkpt;
-	
-	  ReadAttribute (dataset, "carpet_reflevels", reflevels_chkpt);
-      
-	  herr = H5Dclose(dataset);
-	  assert(!herr);
-	  herr = H5Gclose(group);
-	  assert(!herr);
-      
-	  ostringstream reflevels_str;
-	  reflevels_str << reflevels_chkpt;
-	  CCTK_ParameterSet ("refinement_levels","CarpetRegrid",reflevels_str.str().c_str());
-	  	 
-	  CCTK_VInfo (CCTK_THORNSTRING, "Using %i reflevels read from checkpoint file. Ignoring value in parameter file.", reflevels_chkpt);
-	  
-	}     
-	
-	/* we need all the times on the individual levels */
-	// these are a bit messy to extract
-
-	// Actually, we do have to do this only once
-
-	  hid_t group = H5Gopen (reader, PARAMETERS_GLOBAL_ATTRIBUTES_GROUP);
-	  assert(group>=0);
-	  hid_t dataset = H5Dopen (group, ALL_PARAMETERS);
-	  assert(dataset >= 0);
-	  hid_t attr = H5Aopen_name (dataset, "numberofmgtimes");
-	  assert(attr >= 0);
-	  hid_t atype = H5Aget_type (attr);
-	  if(H5Tequal(atype, H5T_NATIVE_INT)) {
-	    herr_t herr = H5Aread(attr, atype, &numberofmgtimes);
-	    assert(!herr);
-	    herr = H5Aclose(attr);
-	    assert(numberofmgtimes==mglevels);
-	    char buffer[100];
-	    for(int lcv=0;lcv<numberofmgtimes;lcv++) {
-	      sprintf(buffer,"mgleveltimes %d",lcv);
-	      attr = H5Aopen_name(dataset, buffer);
-	      assert (attr>=0);
-	      atype = H5Aget_type (attr);
-	      assert (atype>=0);
-	      herr = H5Aread (attr, atype, &leveltimes.at(lcv).at(0));
-	      assert(!herr);
-	      herr = H5Aclose(attr);
-	      assert(!herr);
-	    }  
-	    herr = H5Dclose(dataset);
-	    assert(!herr);
-	    herr = H5Gclose(group);
-	    assert(!herr);
-	  } else {
-	    CCTK_WARN(0,"BAD BAD BAD! Can't read leveltimes!!");
-	  }
-
-      } // myproc == 0  	
-
-
-      // communicate the time on all the mglevels and reflevels
-
-      int mpierr = MPI_Bcast (&numberofmgtimes, 1, CARPET_MPI_INT4, 0,MPI_COMM_WORLD);
-      assert(!mpierr);
-
-
-      for(int i=0;i<numberofmgtimes;i++) {
-	mpierr = MPI_Bcast (&(leveltimes.at(i).at(0)), reflevels, CARPET_MPI_REAL, 0, MPI_COMM_WORLD);
-	assert(!mpierr);
-      }
-
-      if (verbose) {
-	cout << "leveltimes: " << leveltimes << endl;
-      }
-
-      cctkGH->cctk_time = leveltimes.at(mglevel).at(reflevel);
-
-      result += RecoverGHextensions(cctkGH,reader);
-
-	  
-      if (verbose) {
-	cout << "reflevel: " << reflevel << endl;
-      }
-
-      result += RecoverVariables (cctkGH,reader);
-
-
-      CCTK_VInfo (CCTK_THORNSTRING,
-		  "Restarting simulation at iteration %d (physical time %g)",
-		  cctkGH->cctk_iteration, (double) cctkGH->cctk_time);
-
-    } // called_from == CP_RECOVER_DATA
-  
-    if (reflevel==maxreflevels) {	
-      if(myproc == 0) {
-	H5Fclose(reader);
-      }
-    }
-
-    return (result);
-  } // CarpetIOHDF5_Recover
-
-  
-  int RecoverVariables (cGH* cctkGH, hid_t reader) {
-
-    DECLARE_CCTK_PARAMETERS;
-
-    int retval = 0;
-    int myproc = CCTK_MyProc (cctkGH);
-    char * name;
-
-    int ndatasets;
-
-    int varindex; 
-
-    double datasettime;
-    double leveltime;
-    static double totaltime;
-
-    hid_t dataset;
-    herr_t herr;
-
-    list<string> refleveldatasetnamelist;
-
-    if (reflevel==0) {
-      totaltime = 0;
-    }
-
-    leveltime = MPI_Wtime();
-
-
-    if(myproc==0) {
-      ndatasets=GetnDatasets(reader);
-      assert (ndatasets>=0);
-    }
-
-    // Broadcast number of datasets
-    MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm);
-
-    assert ((ndatasets)>=0);
-    
-    //if (verbose && reflevel==0) cout << "ndatasets: " << ndatasets << endl;
-    if ( reflevel == 0) {
-      cout << "ndatasets: " << ndatasets << endl;
-    }
-
-    if (reflevel==0) {
-      for (int currdataset=0;currdataset<ndatasets+1;currdataset++){
-	char datasetname[256];
-	if (myproc==0) {
-	  GetDatasetName(reader,currdataset,datasetname);
-	  datasetnamelist.push_back(datasetname);
-	} //myproc = 0
-	else {
-	  datasetnamelist.push_back("blah");
-	}
-      }
-    } 
-
-    cout << "I have " << datasetnamelist.size() << endl;
-
-    double comparetime = MPI_Wtime();
-      
-    if(myproc==0) {
-      //      for(currdataset=datasetnamelist.begin();
-      //	  currdataset!=datasetnamelist.end();
-      //	  ++currdataset) {
-
-      list<string>::iterator currdataset;
-      currdataset=datasetnamelist.begin();
-       while(currdataset!=datasetnamelist.end()) {
-	char tempstr[10];
-	sprintf(tempstr,"rl=%d m=",reflevel);
-	if ( (*currdataset).find(tempstr) < (*currdataset).size() ) {
-	  refleveldatasetnamelist.push_back((*currdataset).c_str());
-	  currdataset = datasetnamelist.erase(currdataset);
-	} else {
-	  ++currdataset;
-	} // if ...
-      } // while ...
-    } // if(myproc==0)
-
-
-
-   
-    static long reflevelnamenum; // static keeps this thing local
-
-    if(myproc==0) {
-	reflevelnamenum=refleveldatasetnamelist.size();
-      }
-
-    MPI_Bcast (&reflevelnamenum, 1, MPI_LONG, 0, dist::comm);
-    assert ((reflevelnamenum)>=0);
-    
-    // fill bogus namelist for non-IO cpus
-    if(myproc !=0) {
-      for(long i=0;i < reflevelnamenum;i++) {
-	refleveldatasetnamelist.push_back("blah");
-      }
-    }
-
-    comparetime = MPI_Wtime() - comparetime;
-    cout << "Time for string comparison: " << comparetime << endl;
-    cout << "I have for this reflevel " << refleveldatasetnamelist.size() << endl;
-
-    list<string>::iterator currdataset;
-
-      currdataset=refleveldatasetnamelist.begin();
- 
-      while(currdataset!=refleveldatasetnamelist.end()) {
-
-	//	cout << "name: " << (*currdataset).c_str() << endl;
-
-      if (myproc==0) {
-	dataset = H5Dopen (reader, (*currdataset).c_str());
-	assert(dataset);
-	// Read data
-	ReadAttribute (dataset, "name", name);
-	varindex = CCTK_VarIndex(name);
-      }
-
-      MPI_Bcast (&varindex, 1, MPI_INT, 0, dist::comm);
-
-      name = CCTK_FullName(varindex);
-
-      if (verbose) {
-	cout << name << "  rl: " << reflevel << endl;
-      }
-      vector<ibset> regions_read(Carpet::maps);
-
-      assert (varindex>=0 && varindex<CCTK_NumVars());
-      const int group = CCTK_GroupIndexFromVarI (varindex);
-      const int grouptype = CCTK_GroupTypeI(group);
-
-      int did_read_something = ReadVar(cctkGH,name,dataset,regions_read,1);
-
-      MPI_Bcast (&did_read_something, 1, MPI_INT, 0, dist::comm);
-
-      if (did_read_something) {
-      	currdataset = refleveldatasetnamelist.erase(currdataset);
-      } else {
-	++currdataset;
-      }
-
-      if(myproc==0) {
-	herr = H5Dclose(dataset);
-	assert(!herr);
-      }
-      free(name);
-
-    }  // for (currdataset ... )
-    leveltime = MPI_Wtime() - leveltime;
-    totaltime = totaltime + leveltime;
-
-    cout << "Timers: leveltime: " << leveltime << " totaltime: " << totaltime << endl; 
-
-    return retval;
-  }
-   
-  
-  int RecoverGHextensions (cGH *cctkGH, hid_t reader)
-  {
-    const int myproc = CCTK_MyProc(cctkGH);
-    CCTK_INT4 int4Buffer[3];
-    CCTK_REAL realBuffer;
-    CCTK_REAL realBuffer2;
-    CCTK_INT4 intbuffer;
-
-    int mpierr = 0;
-
-    if (myproc==0)
-      {
-		
-	// First open group and dataset
-	hid_t group = H5Gopen (reader, PARAMETERS_GLOBAL_ATTRIBUTES_GROUP);
-	assert(group>=0);
-	hid_t dataset = H5Dopen (group, ALL_PARAMETERS);
-	assert(dataset >= 0);
-
-	ReadAttribute(dataset,"GH$iteration",int4Buffer[0]);
-	ReadAttribute(dataset,"main loop index",int4Buffer[1]);
-	ReadAttribute(dataset,"carpet_global_time",realBuffer);
-	//	ReadAttribute(dataset,"carpet_reflevels",int4Buffer[2]);
-	ReadAttribute(dataset,"carpet_delta_time",realBuffer2);
-
-	herr_t herr = H5Dclose(dataset);
-	assert(!herr);
-	herr = H5Gclose(group);
-	assert(!herr);
-	
-      }
-  /* Broadcast the GH extensions to all processors */
-  /* NOTE: We have to use MPI_COMM_WORLD here
-     because PUGH_COMM_WORLD is not yet set up at parameter recovery time.
-     We also assume that PUGH_MPI_INT4 is a compile-time defined datatype. */
-
-    mpierr = MPI_Bcast (int4Buffer, 3, CARPET_MPI_INT4, 0,MPI_COMM_WORLD);
-    assert(!mpierr);
-    mpierr = MPI_Bcast (int4Buffer, 3, CARPET_MPI_INT4, 0,MPI_COMM_WORLD);
-    assert(!mpierr);
-    mpierr = MPI_Bcast (&realBuffer, 1, CARPET_MPI_REAL,0,MPI_COMM_WORLD);
-    assert(!mpierr);
-    mpierr = MPI_Bcast (&realBuffer2, 1, CARPET_MPI_REAL,0,MPI_COMM_WORLD);
-    assert(!mpierr);
-
-    global_time = (CCTK_REAL) realBuffer;
-    delta_time = (CCTK_REAL) realBuffer2;
-    //    reflevels = (int) int4Buffer[2];
-    cctkGH->cctk_iteration = (int) int4Buffer[0];
-    CCTK_SetMainLoopIndex ((int) int4Buffer[1]);
-
-
-    return (0);
-
-  } // RecoverGHExtensions
-
-  int RecoverParameters(hid_t reader){
-
-    DECLARE_CCTK_PARAMETERS; 
-    
-    int myproc, retval;
-    char *parameters;
-    CCTK_INT4 parameterSize;
-
-    hid_t group,dataset;
-    herr_t herr;
-
-    int mpierr;
-
-    myproc = CCTK_MyProc (NULL);
-
-    if (myproc == 0){
-      CCTK_VInfo (CCTK_THORNSTRING, "Recovering parameters from checkpoint ");
-      
-      group = H5Gopen (reader, PARAMETERS_GLOBAL_ATTRIBUTES_GROUP);
-      assert(group >= 0);
-      dataset = H5Dopen (group, ALL_PARAMETERS);
-      assert(dataset>= 0);
-
-      parameterSize = H5Dget_storage_size (dataset);
-      parameters = (char *) malloc (parameterSize);
-      herr = H5Dread (dataset, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL,
-                             H5P_DEFAULT, parameters);
-      assert(!herr);
-      herr = H5Dclose(dataset);
-      assert(!herr);
-      herr = H5Gclose(group);
-      assert(!herr);
-
-      if(verbose) {
-	CCTK_VInfo (CCTK_THORNSTRING, "\n%s\n",parameters);
-      }
-      
-      CCTK_VInfo(CCTK_THORNSTRING, "Successfully recovered parameters!");
-    } // myproc == 0
-
-    /* Broadcast the parameter buffer size to all processors */
-    /* NOTE: We have to use MPI_COMM_WORLD here
-       because CARPET_COMM_WORLD is not yet set up at parameter recovery time.
-       We also assume that CARPET_MPI_INT4 is a compile-time defined datatype. */
-    mpierr = MPI_Bcast (&parameterSize, 1, CARPET_MPI_INT4, 0,
-			MPI_COMM_WORLD);
-    assert(!mpierr);
-
-    if (parameterSize > 0) {
-      if (myproc) {
-	parameters = (char*) malloc (parameterSize + 1);
-      }
-	
-      mpierr = MPI_Bcast (parameters, parameterSize + 1, CARPET_MPI_CHAR,
-			  0,MPI_COMM_WORLD);
-      assert(!mpierr);
-      
-      IOUtil_SetAllParameters (parameters);
-    
-      free (parameters);
-    }
-  
-    /* return positive value for success otherwise negative */
-    retval = (parameterSize > 0 ? 1 : -1);
-
-    return (retval);
-
-  } // RecoverParameters
-
-
-
-  int Checkpoint (const cGH* const cctkGH, int called_from)
-  {
-    DECLARE_CCTK_ARGUMENTS;
-    DECLARE_CCTK_PARAMETERS;
-
-    herr_t herr;
-    
-    int retval = 0;
-
-    const ioGH *ioUtilGH;
-    
-    ioRequest *request;
-
-    CarpetIOHDF5GH *myGH;
-    myGH = (CarpetIOHDF5GH *) CCTK_GHExtension (cctkGH, "CarpetIOHDF5");
-
-    // check if CarpetIOHDF5 was registered as I/O method
-    if (myGH == NULL) {
-      CCTK_WARN (0, "No CarpetIOHDF5 I/O methods registered");
-      return -1;
-    }
-    
-    int const myproc = CCTK_MyProc (cctkGH);
-    
-    
-    
-    // Invent a file name
-    ostringstream cp_tempname_buf;
-    ostringstream cp_filename_buf;
-    
-    switch (called_from)
-    {
-    case CP_INITIAL_DATA:
-      cp_tempname_buf << checkpoint_dir << "/tmp_" << checkpoint_ID_file << ".it_" << cctkGH->cctk_iteration << ".h5";
-      cp_filename_buf << checkpoint_dir << "/" << checkpoint_ID_file << ".it_" << cctkGH->cctk_iteration << ".h5";
-      break;
-      
-    case CP_EVOLUTION_DATA:
-      cp_tempname_buf << checkpoint_dir << "/tmp_" << checkpoint_file << ".it_" << cctkGH->cctk_iteration << ".h5";
-      cp_filename_buf << checkpoint_dir << "/" << checkpoint_file << ".it_" << cctkGH->cctk_iteration << ".h5";
-      break;
-      
-    case CP_RECOVER_DATA:
-    case CP_RECOVER_PARAMETERS:
-    case FILEREADER_DATA:
-      cp_tempname_buf << recover_dir << "/tmp_" << recover_file << ".h5";
-      cp_filename_buf << recover_dir << "/" << recover_file << ".h5";
-      break;
-      
-    default:
-      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "IOUtil_PrepareFilename: Unknown calling mode %d",
-                  called_from);
-      break;
-    }
-    
-    string const cp_tempname_str = cp_tempname_buf.str();
-    string const cp_filename_str = cp_filename_buf.str();
-    char const * const cp_tempname = cp_tempname_str.c_str();
-    char const * const cp_filename = cp_filename_str.c_str();
-    
-    
-    
-    hid_t writer = -1;
-
-    if (myproc == 0) {
-
-      if (verbose) {
-	CCTK_VInfo (CCTK_THORNSTRING, "Creating temporary checkpoint file '%s'", cp_tempname);
-      }
-
-      CCTK_VInfo (CCTK_THORNSTRING, "Creating temporary checkpoint file '%s'", cp_tempname);
-      
-      CCTK_VInfo (CCTK_THORNSTRING, "Verbose = %d", verbose);
-
-      writer = H5Fcreate (cp_tempname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
-      assert (writer>=0);
-      herr = H5Fclose (writer);
-      assert (!herr);
-      writer = -1;
-
-      // Now open the file
-
-      writer = H5Fopen (cp_tempname, H5F_ACC_RDWR, H5P_DEFAULT);
-      assert (writer>=0);
-
-      // Dump all parameters and GHExtentions
-      retval += DumpParametersGHExtentions (cctkGH, 1, writer);
-      assert(!retval);
-
-    } // myproc == 0 
-
-
-    // now dump the grid variables on all mglevels, reflevels, maps and components
-    BEGIN_MGLEVEL_LOOP(cctkGH) {
-
-      BEGIN_REFLEVEL_LOOP(cctkGH) {
-
-	if (verbose)
-	  {
-	    CCTK_INFO ("Dumping Grid Variables ...");
-	  }
-	for (int group = CCTK_NumGroups () - 1; group >= 0; group--)
-	  {
-	    /* only dump groups which have storage assigned */
-
-	    if (CCTK_QueryGroupStorageI (cctkGH, group) <= 0)
-	      {
-		continue;
-	      }
-
-	    const int grouptype = CCTK_GroupTypeI(group);
-
-	    /* scalars and grid arrays only have 1 reflevel: */
-	    if ( (grouptype != CCTK_GF) && (reflevel != 0) ) {
-	      continue;
-	    }
-	    /* now check if there is any memory allocated
-	       for GFs and GAs. GSs should always have 
-	       memory allocated and there is at this point
-	       no CCTK function to check this :/
-	    */
-
-	    if ( (grouptype == CCTK_GF) || (grouptype == CCTK_ARRAY)){
-	      const int gpdim = CCTK_GroupDimI(group);
-	      int gtotalsize=1;
-	      vector<int> tlsh(gpdim);
-	      assert(!CCTK_GrouplshGI(cctkGH,gpdim,&tlsh[0],group));
-	      for(int i=0;i<gpdim;i++) {
-		gtotalsize=tlsh[i];		  
-	      }
-	      if(gtotalsize == 0){
-		if (verbose) CCTK_VInfo(CCTK_THORNSTRING, 
-		    "Group %s is zero-sized. No checkpoint info written",CCTK_GroupName(group));
-		continue;
-	      }
-	    }
-	    
-	    /* get the number of allocated timelevels */
-            cGroup gdata;
-	    CCTK_GroupData (group, &gdata);
-	    gdata.numtimelevels = 0;
-	    gdata.numtimelevels = CCTK_GroupStorageIncrease (cctkGH, 1, &group,
-							     &gdata.numtimelevels,NULL);
-	    
-	    int first_vindex = CCTK_FirstVarIndexI (group);
-
-	    /* get the default I/O request for this group */
-	    request = IOUtil_DefaultIORequest (cctkGH, first_vindex, 1);
-	    
-	    /* disable checking for old data objects, disable datatype conversion
-	       and downsampling */
-	    request->check_exist = 0;
-	    request->hdatatype = gdata.vartype;
-	    for (request->hdim = 0; request->hdim < request->vdim; request->hdim++)
-	      {
-		request->downsample[request->hdim] = 1;
-	      }
-	    
-	    /* loop over all variables in this group */
-	    for (request->vindex = first_vindex;
-		 request->vindex < first_vindex + gdata.numvars;
-		 request->vindex++)
-	      {
-		/* loop over all timelevels of this variable */
-		for (request->timelevel = 0;
-		     request->timelevel < gdata.numtimelevels;
-		     request->timelevel++)
-		  {
-		    if (verbose)
-		      {
-			char *fullname = CCTK_FullName (request->vindex);
-			if (fullname != NULL) {
-			    CCTK_VInfo (CCTK_THORNSTRING, "  %s (timelevel %d)",
-					fullname, request->timelevel);
-			    free (fullname);
-			}
-			else {
-			  CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
-				     "Invalid variable with varindex %d", request->vindex);
-			}
-		      }
-		    // write the var
-		    
-		    if (grouptype == CCTK_ARRAY || grouptype == CCTK_GF || grouptype == CCTK_SCALAR)
-		      {
-			char* fullname = CCTK_FullName (request->vindex);
-			if (verbose)
-			  CCTK_VInfo (CCTK_THORNSTRING,"%s: reflevel: %d map: %d component: %d grouptype: %d ",
-				      fullname,reflevel,Carpet::map,component,grouptype);
-			free(fullname);
-		    retval += WriteVar(cctkGH,writer,request,1);
-		      }
-		    else
-		      {
-			char *fullname = CCTK_FullName (request->vindex);
-			CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-				    "Invalid group type %d for variable '%s'", grouptype, fullname);
-			free(fullname);
-			retval = -1;
-		      }
-		    
-		  }
-	      } /* end of loop over all variables */
-	  } /* end of loop over all groups */
-      } END_REFLEVEL_LOOP;
-      
-    } END_MGLEVEL_LOOP;
-  
-
-    if (myproc==0) {
-      // Close the file
-      herr = H5Fclose(writer);
-      assert(!herr);
-    }
-    
-    if (retval == 0) {
-      if (myproc==0) {
-	if (rename (cp_tempname, cp_filename))
-	  {
-	    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-			"Could not rename temporary checkpoint file '%s' to '%s'",
-			cp_tempname, cp_filename);
-	    retval = -1;
-	  }
-	else {
-	  if (myGH->cp_filename_list[myGH->cp_filename_index]) {
-	    if (checkpoint_keep > 0) {
-	      remove (myGH->cp_filename_list[myGH->cp_filename_index]);
-	    }
-	    free (myGH->cp_filename_list[myGH->cp_filename_index]);
-	  }
-	  myGH->cp_filename_list[myGH->cp_filename_index] = strdup (cp_filename);
-	  myGH->cp_filename_index = (myGH->cp_filename_index+1) % abs (checkpoint_keep);
-	} // else
-      } // myproc == 0
-    } // retval == 0
-
-    return retval;
-
-  } // Checkpoint
-
-  
-  int DumpParametersGHExtentions (const cGH *cctkGH, int all, hid_t writer)
-  {
-    // large parts of this routine were taken from
-    // Thomas Radke's IOHDF5Util. Thanks Thomas!
-
-    DECLARE_CCTK_PARAMETERS;
-
-    char *parameters;
-    hid_t group, dataspace, dataset;
-    hsize_t size;
-    herr_t herr;
-    
-    CCTK_INT4 itmp;
-    CCTK_REAL dtmp;
-    const char *version;
-    ioGH *ioUtilGH;
-  
-    if (verbose) {
-      CCTK_INFO ("Dumping Parameters and GH Extentions...");
-    }
-  
-    /* get the parameter string buffer */
-    parameters = IOUtil_GetAllParameters (cctkGH, all);
-  
-    if (parameters)
-    {
-      size = strlen (parameters) + 1;
-      group = H5Gcreate (writer, PARAMETERS_GLOBAL_ATTRIBUTES_GROUP, 0);
-      assert(group>=0);
-      dataspace = H5Screate_simple (1, &size, NULL);
-      assert(dataspace>=0);
-      dataset = H5Dcreate (group, ALL_PARAMETERS, H5T_NATIVE_UCHAR,
-                                       dataspace, H5P_DEFAULT);
-      assert(dataset>=0);
-      herr = H5Dwrite (dataset, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL,
-                            H5P_DEFAULT, parameters);
-      assert(!herr);
-
-      // now dump the GH Extentions
-
-      /* get the handle for IOUtil extensions */
-      ioUtilGH = (ioGH *) CCTK_GHExtension (cctkGH, "IO");
-      
-      itmp = CCTK_MainLoopIndex ();
-      WriteAttribute(dataset,"main loop index",itmp);
-      
-      itmp = cctkGH->cctk_iteration;
-      WriteAttribute(dataset,"GH$iteration",itmp);
-      
-      itmp = ioUtilGH->ioproc_every;
-      WriteAttribute(dataset,"GH$ioproc_every",itmp);
-      
-      itmp = CCTK_nProcs (cctkGH);
-      WriteAttribute(dataset,"GH$nprocs",itmp);
-      
-      dtmp = cctkGH->cctk_time;
-      WriteAttribute(dataset,"GH$time", dtmp);
-
-      dtmp = global_time;
-      WriteAttribute(dataset,"carpet_global_time", dtmp);
-
-      itmp = reflevels;
-      WriteAttribute(dataset,"carpet_reflevels", itmp);
-
-      dtmp = delta_time;
-      WriteAttribute(dataset,"carpet_delta_time", dtmp);
-
-      version = CCTK_FullVersion();
-      WriteAttribute(dataset,"Cactus version", version);
-
-      /* finally, we need all the times on the individual refinement levels */
-
-      const int numberofmgtimes=mglevels;
-      WriteAttribute(dataset,"numberofmgtimes",numberofmgtimes);
-      for(int i=0;i < numberofmgtimes;i++) {
-	char buffer[100];
-	sprintf(buffer,"mgleveltimes %d",i);
-	WriteAttribute(dataset,buffer,(double *) &leveltimes.at(i).at(0), reflevels);
-      }
-
-      herr = H5Dclose (dataset);
-      assert(!herr);
-      herr = H5Sclose (dataspace);
-      assert(!herr);
-      herr = H5Gclose (group);
-      assert(!herr);
-  
-      free (parameters);
-    }
-
-    return 0;
-  } // DumpParametersGHExtentions
-  
-
-    
-  
-} // namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/iohdf5utils.cc b/Carpet/CarpetIOHDF5/src/iohdf5utils.cc
deleted file mode 100644
index 8d25b7f7b..000000000
--- a/Carpet/CarpetIOHDF5/src/iohdf5utils.cc
+++ /dev/null
@@ -1,521 +0,0 @@
-#include <assert.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include <algorithm>
-#include <fstream>
-#include <sstream>
-#include <vector>
-
-#include <hdf5.h>
-
-#include "cctk.h"
-#include "cctk_Parameters.h"
-
-extern "C" {
-  static const char* rcsid = "$Header:$";
-  CCTK_FILEVERSION(Carpet_CarpetIOHDF5_iohdf5utils_cc);
-}
-
-#include "CactusBase/IOUtil/src/ioGH.h"
-#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
-
-#include "bbox.hh"
-#include "data.hh"
-#include "gdata.hh"
-#include "ggf.hh"
-#include "vect.hh"
-
-#include "carpet.hh"
-
-#include "iohdf5.hh"
-#include "iohdf5GH.h"
-
-
-
-namespace CarpetIOHDF5 {
-  
-  using namespace std;
-  using namespace Carpet;
-  
-  
-  
-  bool CheckForVariable (const char* const varlist, const int vindex)
-  {
-    const int numvars = CCTK_NumVars();
-    assert (vindex>=0 && vindex<numvars);
-    
-    vector<bool> flags(numvars);
-    
-    CCTK_TraverseString (varlist, SetFlag, &flags, CCTK_GROUP_OR_VAR);
-    
-    return flags.at(vindex);
-  }
-  
-  void SetFlag (int vindex, const char* optstring, void* arg)
-  {
-    if (optstring)
-    {
-      char *fullname = CCTK_FullName (vindex);
-      CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Option string '%s' will be ignored for HDF5 output of "
-                  "variable '%s'", optstring, fullname);
-      free (fullname);
-    }
-    vector<bool>& flags = *(vector<bool>*)arg;
-    flags.at(vindex) = true;
-  }
-  
-  
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const int value)
-  {
-    WriteAttribute (dataset, name, &value, 1);
-  }
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const int* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    hsize_t shape[1];
-    shape[0] = nvalues;
-    const hid_t dataspace = nvalues==1 ? H5Screate (H5S_SCALAR) : H5Screate_simple (1, shape, NULL);
-    assert (dataspace>=0);
-    
-    const hid_t datatype = H5T_NATIVE_INT;
-    
-    const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
-    assert (attribute>=0);
-    herr = H5Awrite (attribute, datatype, values);
-    assert (!herr);
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-  }
-  
-  
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const double value)
-  {
-    WriteAttribute (dataset, name, &value, 1);
-  }
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const double* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    hsize_t shape[1];
-    shape[0] = nvalues;
-    const hid_t dataspace = nvalues==1 ? H5Screate (H5S_SCALAR) : H5Screate_simple (1, shape, NULL);
-    assert (dataspace>=0);
-    
-    const hid_t datatype = H5T_NATIVE_DOUBLE;
-    
-    const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
-    assert (attribute>=0);
-    herr = H5Awrite (attribute, datatype, values);
-    assert (!herr);
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-  }
-  
-  
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const char value)
-  {
-    WriteAttribute (dataset, name, &value, 1);
-  }
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const char* const values)
-  {
-    WriteAttribute (dataset, name, values, strlen(values));
-  }
-  
-  void WriteAttribute (const hid_t dataset, const char* const name, const char* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    const hid_t dataspace = H5Screate (H5S_SCALAR);
-    assert (dataspace>=0);
-    
-    const hid_t datatype = H5Tcopy (H5T_C_S1);
-    assert (datatype>=0);
-    herr = H5Tset_size (datatype, nvalues);
-    assert (!herr);
-    
-    const hid_t attribute = H5Acreate (dataset, name, datatype, dataspace, H5P_DEFAULT);
-    assert (attribute>=0);
-    herr = H5Awrite (attribute, datatype, values);
-    assert (!herr);
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    herr = H5Tclose (datatype);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-  }
-  
-  
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, int& value)
-  {
-    return ReadAttribute (dataset, name, &value, 1);
-  }
-  
-  int ReadAttribute (const hid_t dataset, const char* name, int* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    const hid_t attribute = H5Aopen_name (dataset, name);
-    if (attribute<0) return attribute;
-    
-    const hid_t dataspace = H5Aget_space (attribute);
-    assert (dataspace>=0);
-    
-    //    cout << "reading int attribute " << name << endl;
-
-    hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
-    hsize_t shape[1];
-    if (rank==0) {
-      shape[0] = 1;
-    } else if (rank==1) {
-      herr = H5Sget_simple_extent_dims (dataspace, shape, NULL);
-      assert (herr >= 0);
-    } else {
-      assert (0);
-    }
-    const int length = shape[0];
-    
-    const hid_t datatype = H5Aget_type (attribute);
-    assert (datatype>=0);
-
-    assert(H5Tequal(datatype, H5T_NATIVE_INT));
-    
-    vector<int> values1(length);
-    
-    herr = H5Aread (attribute, datatype, &values1.at(0));
-    assert (!herr);
-    
-    for (int i=0; i<min(length, nvalues); ++i) {
-      values[i] = values1[i];
-    }
-
-    herr = H5Tclose (datatype);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-    
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    return length;
-  }
-  
-  
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, double& value)
-  {
-    return ReadAttribute (dataset, name, &value, 1);
-  }
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, double* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    const hid_t attribute = H5Aopen_name (dataset, name);
-    if (attribute<0) return attribute;
-    
-    const hid_t dataspace = H5Aget_space (attribute);
-    assert (dataspace>=0);
-    
-    hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
-    hsize_t shape[1];
-    if (rank==0) {
-      shape[0] = 1;
-    } else if (rank==1) {
-      rank = H5Sget_simple_extent_dims (dataspace, shape, NULL);
-      assert (rank == 1);
-    } else {
-      assert (0);
-    }
-    const int length = shape[0];
-    
-    const hid_t datatype = H5Aget_type (attribute);
-    assert (datatype>=0);
-    assert(H5Tequal(datatype, H5T_NATIVE_DOUBLE));
-    
-    vector<double> values1(length);
-    
-    herr = H5Aread (attribute, datatype, &values1.at(0));
-    assert (!herr);
-    
-    for (int i=0; i<min(length, nvalues); ++i) {
-      values[i] = values1[i];
-    }
-    
-    herr = H5Tclose (datatype);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-    
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    return length;
-  }
-  
-  
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, char& value)
-  {
-    return ReadAttribute (dataset, name, &value, 1);
-  }
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, char*& values)
-  {
-    assert (dataset>=0);
-    assert (name);
-    
-    herr_t herr;
-    
-    const hid_t attribute = H5Aopen_name (dataset, name);
-    if (attribute<0) return attribute;
-    
-    const hid_t dataspace = H5Aget_space (attribute);
-    assert (dataspace>=0);
-    
-    hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
-    assert (rank==0);
-    
-    const hid_t datatype = H5Aget_type (attribute);
-    assert (datatype>=0);
-    assert (H5Tget_class (datatype) == H5T_STRING);
-    const int length = H5Tget_size (datatype);
-    assert (length>=0);
-    
-    values = (char*) malloc (length+1);
-    assert (values);
-    
-    herr = H5Aread (attribute, datatype, values);
-    assert (!herr);
-    values[length] = '\0';
-    
-    herr = H5Tclose (datatype);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-    
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    return length;
-  }
-  
-  int ReadAttribute (const hid_t dataset, const char* const name, char* const values, const int nvalues)
-  {
-    assert (dataset>=0);
-    assert (name);
-    assert (values);
-    assert (nvalues>=0);
-    
-    herr_t herr;
-    
-    const hid_t attribute = H5Aopen_name (dataset, name);
-    if (attribute<0) return attribute;
-    
-    const hid_t dataspace = H5Aget_space (attribute);
-    assert (dataspace>=0);
-    
-    hsize_t rank = H5Sget_simple_extent_ndims (dataspace);
-    assert (rank==0);
-    
-    const hid_t datatype = H5Aget_type (attribute);
-    assert (datatype>=0);
-    assert(H5Tget_class (datatype) == H5T_STRING); 
-    const int length = H5Tget_size (datatype);
-    assert (length>=0);
-    
-    vector<char> values1(length);
-    
-    herr = H5Aread (attribute, datatype, &values1.at(0));
-    assert (!herr);
-    
-    for (int i=0; i<min(length, nvalues); ++i) {
-      values[i] = values1[i];
-    }
-    
-    herr = H5Tclose (datatype);
-    assert (!herr);
-    
-    herr = H5Sclose (dataspace);
-    assert (!herr);
-    
-    herr = H5Aclose (attribute);
-    assert (!herr);
-    
-    return length;
-  }
-
-  herr_t DatasetCounter(hid_t group_id, const char *member_name, void *operator_data)
-    /* Counts datasets. Used by GetnDatasets; straight from John Shalf's FlexIO library */
-  {
-    int *count = (int*)operator_data;
-    H5G_stat_t objinfo;
-    // request info about the type of objects in root group
-    if(H5Gget_objinfo(group_id,member_name,1 /* follow links */,&objinfo)<0) {
-      return 0; 
-    }
-    // only count objects that are datasets (not subgroups)
-    if(objinfo.type==H5G_DATASET) {
-      (*count)++;
-    }
-  return 0;
-  }
-
-  
-  int GetnDatasets(const hid_t reader)
-  {
-    //this is straight from John Shalf's FlexIO library
-
-    int count=0;
-    int idx=0;
-    while(H5Giterate(reader, /* hid_t loc_id, */
-		     "/", /*const char *name, */
-		     &idx, /* int *idx, */
-		     DatasetCounter,
-		     &count)<0){}
-    return count;
-  }
-
-  struct H5IO_getname_t {
-    //this is straight from John Shalf's FlexIO library
-    int index,count;
-    char *name;
-  };
-
-
-  herr_t GetName(hid_t group_id, const char *member_name, void *operator_data)
-  {
-    //this is straight from John Shalf's FlexIO library
-    H5IO_getname_t *getn = (H5IO_getname_t*)operator_data;
-    // check type first (only respond if it is a dataset)
-    H5G_stat_t objinfo;
-    // request info about the type of objects in root group
-    if(H5Gget_objinfo(group_id,
-		      member_name,
-		    1 /* follow links */,
-		      &objinfo)<0) return 0; // error (probably bad symlink)
-    // only count objects that are datasets (not subgroups)
-    if(objinfo.type!=H5G_DATASET)
-      return 0; // do not increment count if it isn't a dataset.
-    if(getn->index==getn->count){
-      strcpy(getn->name,member_name);
-      return 1; // success
-    }
-    getn->count++;
-    return 0;
-  }
-
-
-  void GetDatasetName(const hid_t reader, const int _index,  char *name) {
-    //this is straight from John Shalf's FlexIO library
-    H5IO_getname_t getn;
-    int idx=_index;
-    getn.index=_index; getn.name=name; getn.count=_index;
-    while(H5Giterate(reader, /* hid_t loc_id, */
-		     "/", /*const char *name, */
-		     &idx, /* int *idx, */
-		     GetName,
-		     &getn)<0){}
-  }
-
-  hid_t h5DataType (const cGH* const cctkGH, int cctk_type) {
-
-    hid_t retval;  
-
-    CarpetIOHDF5GH *myGH;
-    myGH = (CarpetIOHDF5GH *) CCTK_GHExtension (cctkGH, "CarpetIOHDF5");
-  
-    // this is adapted from Thomas Radke's IOHDF5Util. Thanks, Thomas!
-
-    switch (cctk_type)
-    {
-      case CCTK_VARIABLE_CHAR:      retval = HDF5_CHAR; break;
-      case CCTK_VARIABLE_INT:       retval = HDF5_INT; break;
-      case CCTK_VARIABLE_REAL:      retval = HDF5_REAL; break;
-      case CCTK_VARIABLE_COMPLEX:   retval = myGH->HDF5_COMPLEX; break;
-#ifdef CCTK_INT1
-      case CCTK_VARIABLE_INT1:      retval = HDF5_INT1; break;
-#endif
-#ifdef CCTK_INT2
-      case CCTK_VARIABLE_INT2:      retval = HDF5_INT2; break;
-#endif
-#ifdef CCTK_INT4
-      case CCTK_VARIABLE_INT4:      retval = HDF5_INT4; break;
-#endif
-#ifdef CCTK_INT8
-      case CCTK_VARIABLE_INT8:      retval = HDF5_INT8; break;
-#endif
-#ifdef CCTK_REAL4
-      case CCTK_VARIABLE_REAL4:     retval = HDF5_REAL4; break;
-      case CCTK_VARIABLE_COMPLEX8:  retval = myGH->HDF5_COMPLEX8; break;
-#endif
-#ifdef CCTK_REAL8
-      case CCTK_VARIABLE_REAL8:     retval = HDF5_REAL8; break;
-      case CCTK_VARIABLE_COMPLEX16: retval = myGH->HDF5_COMPLEX16; break;
-#endif
-#ifdef CCTK_REAL16
-      case CCTK_VARIABLE_REAL16:    retval = HDF5_REAL16; break;
-      case CCTK_VARIABLE_COMPLEX32: retval = myGH->HDF5_COMPLEX32; break;
-#endif
-  
-      default: CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                           "Unsupported CCTK variable datatype %d", cctk_type);
-               retval = -1;
-    }
-  
-    return (retval);
-  }
-  
-    
-  
-} // namespace CarpetIOHDF5
diff --git a/Carpet/CarpetIOHDF5/src/make.code.defn b/Carpet/CarpetIOHDF5/src/make.code.defn
index 98fdad123..3f15c29af 100644
--- a/Carpet/CarpetIOHDF5/src/make.code.defn
+++ b/Carpet/CarpetIOHDF5/src/make.code.defn
@@ -2,7 +2,7 @@
 # $Header:$
 
 # Source files in this directory
-SRCS = iohdf5.cc iohdf5utils.cc Checkpoint.cc Recover.cc
+SRCS = Output.cc Utils.cc Checkpoint.cc Recover.cc
 
 # Extend CXXFLAGS if HDF5 library was built with LFS support
 ifneq ($(strip $(HDF5_LFS_FLAGS)),)