Added checkpoint/recovery functionality.

Moved a lot of code into IOHDF5Util and inherit from this thorn.

Forgot to 'cvs add' these files at my last commit.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOStreamedHDF5/trunk@45 0888f3d4-9f52-45d2-93bc-d00801ff5e46

3 files changed, 770 insertions, 0 deletions

diff --git a/src/DumpGH.c b/src/DumpGH.c
new file mode 100644
index 0000000..c31c1e1
--- /dev/null
+++ b/src/DumpGH.c
@@ -0,0 +1,305 @@
+ /*@@
+   @file      DumpGH.c
+   @date      Fri Oct 6 2000
+   @author    Thomas Radke
+   @desc 
+              Checkpoint routines scheduled at CCTK_CPINITIAL, CCTK_CHECKPOINT,
+              and CCTK_TERMINATE.
+              They check the IO checkpointing parameters and - if it's time
+              to do so - call the routine which finally creates a checkpoint.
+   @enddesc 
+   @version   $Id$
+ @@*/
+
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "CactusBase/IOUtil/src/ioGH.h"
+#include "CactusPUGH/PUGH/src/include/pugh.h"
+#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
+#include "BetaThorns/IOHDF5Util/src/ioHDF5UtilGH.h"
+#include "ioStreamedHDF5GH.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* the rcs ID and its dummy function to use it */
+static char *rcsid = "$Id$";
+CCTK_FILEVERSION(BetaThorns_IOStreamedHDF5_DumpGH_c)
+
+
+/* local function prototypes */
+static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from);
+
+
+ /*@@
+   @routine    IOStreamedHDF5_InitialDataCheckpoint
+   @date       Fri Oct 6 2000
+   @author     Thomas Radke
+   @desc 
+               This routine is registered at CCTK_CPINITIAL.
+               It checks if initial data should be checkpointed.
+   @enddesc 
+
+   @calls      IOStreamedHDF5_Checkpoint
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
+   @vtype      cGH *
+   @vio        in
+   @endvar
+@@*/
+void IOStreamedHDF5_InitialDataCheckpoint (cGH *GH)
+{
+  DECLARE_CCTK_PARAMETERS
+
+
+  if (checkpoint && checkpoint_ID)
+  {
+    IOStreamedHDF5_Checkpoint (GH, CP_INITIAL_DATA);
+  }
+}
+
+
+ /*@@
+   @routine    IOStreamedHDF5_EvolutionCheckpoint
+   @date       Fri Oct 6 2000
+   @author     Thomas Radke
+   @desc 
+               This routine is registered at CCTK_CHECKPOINT.
+               It periodically checks if it's time to checkpoint evolution data.
+   @enddesc 
+ 
+   @calls      IOStreamedHDF5_Checkpoint
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
+   @vtype      cGH *
+   @vio        in
+   @endvar
+@@*/
+void IOStreamedHDF5_EvolutionCheckpoint (cGH *GH)
+{
+  DECLARE_CCTK_PARAMETERS
+
+
+  if (checkpoint &&
+      ((checkpoint_every > 0 && GH->cctk_iteration % checkpoint_every == 0) ||
+       checkpoint_next))
+  {
+    if (verbose)
+    {
+      CCTK_INFO("------------------------------------------------------------");
+      CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint file at "
+                  "iteration %d", GH->cctk_iteration);
+    }
+    IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA);
+
+    /* reset the 'checkpoint_next' parameter */
+    if (checkpoint_next)
+    {
+      CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no");
+    }
+  }
+}
+
+
+ /*@@
+   @routine    IOStreamedHDF5_TerminationCheckpoint
+   @date       Fri Oct 6 2000
+   @author     Thomas Radke
+   @desc 
+               This routine is registered at CCTK_TERMINATE.
+               It checks if the last iteration should be checkpointed.
+   @enddesc 
+
+   @calls      IOStreamedHDF5_Checkpoint
+ 
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
+   @vtype      cGH *
+   @vio        in
+   @endvar
+@@*/
+void IOStreamedHDF5_TerminationCheckpoint (cGH *GH)
+{
+  DECLARE_CCTK_PARAMETERS
+
+
+  if (checkpoint && checkpoint_on_terminate)
+  {
+    IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA);
+  }
+}
+
+
+/*****************************************************************************/
+/*                             local routines                                */
+/*****************************************************************************/
+ /*@@
+   @routine    IOStreamedHDF5_Checkpoint
+   @date       Fri Oct 6 2000
+   @author     Thomas Radke
+   @desc 
+               The heart of checkpointing.
+               Called by the different wrappers, this routine creates
+               a new checkpoint file and then dumps away using the
+               dump routines from IOHDF5Util.
+   @enddesc 
+
+   @calls      IOHDF5Util_DumpGH
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        called_from
+   @vdesc      flag indicating where this routine was called from
+               (either CHECKPOINT_ID, CHECKPOINT, or filereader)
+   @vtype      int
+   @vio        in
+   @endvar
+   @returntype int
+   @returndesc
+               -1 if checkpoint file could not be created
+               or returncode of @seeroutine IOUtilHDF5_DumpGH 
+   @endreturndesc
+@@*/
+static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from)
+{
+  DECLARE_CCTK_PARAMETERS
+  hid_t file;
+  int old_ioproc;
+  int old_nioprocs;
+  int old_ioproc_every;
+  fd_set readset;
+  struct timeval timeout;
+  ioGH *ioUtilGH;
+  ioStreamedHDF5GH *myGH;
+  CCTK_INT4 retval;
+
+
+  retval = 0;
+  file = -1;
+
+  /* Get the GH extensions for IOUtil and IOStreamedHDF5 */
+  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
+  myGH = (ioStreamedHDF5GH *) CCTK_GHExtension (GH, "IOStreamedHDF5");
+
+  /* check if IOStreamedHDF5 was registered as IO method */
+  if (myGH == NULL)
+  {
+    CCTK_WARN (2, "No IOStreamedHDF5 IO method registered");
+    return (-1);
+  }
+
+  /* start the CP_TOTAL_TIMER timer */
+  if (myGH->print_timing_info)
+  {
+    CCTK_TimerStartI (myGH->timers[CP_TOTAL_TIMER]);
+  }
+
+  /* for now we can only have IO mode "oneproc" */
+  if (ioUtilGH->nioprocs != 1)
+  {
+    CCTK_WARN (2, "Output into multiple chunked files not yet implemented. "
+                  "Switching to IO mode \"oneproc\".");
+  }
+  old_ioproc = ioUtilGH->ioproc;
+  ioUtilGH->ioproc = 0;
+  old_nioprocs = ioUtilGH->nioprocs;
+  ioUtilGH->nioprocs = 1;
+  old_ioproc_every = ioUtilGH->ioproc_every;
+  ioUtilGH->ioproc_every = CCTK_nProcs (GH);
+
+  /* Now open the file */
+  if (CCTK_MyProc (GH) == ioUtilGH->ioproc && myGH->checkpoint_socket >= 0)
+  {
+    if (verbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING, "Creating checkpoint file");
+    }
+
+    file = H5Fcreate ("dummy_name", H5F_ACC_TRUNC, H5P_DEFAULT,
+                      myGH->checkpoint_fapl);
+    if (file < 0)
+    {
+      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Can't create checkpoint file on port %d. "
+                  "Checkpointing is skipped.", checkpoint_port);
+      retval = -1;
+    }
+  }
+  else
+  {
+    retval = -1;
+  }
+
+#ifdef CCTK_MPI
+  /* broadcast the result of the H5Fcreate() call */
+  CACTUS_MPI_ERROR (MPI_Bcast (&retval, 1, PUGH_MPI_INT4, 0,
+                               PUGH_pGH (GH)->PUGH_COMM_WORLD));
+#endif
+
+  if (retval == 0)
+  {
+    retval = IOHDF5Util_DumpGH (GH,
+                                myGH->print_timing_info ? myGH->timers : NULL,
+                                file);
+  }
+
+  if (file >= 0)
+  {
+    if (checkpoint_accept_timeout > 0)
+    {
+      if (verbose)
+      {
+        CCTK_VInfo (CCTK_THORNSTRING, "Waiting %d seconds to send "
+                                      "checkpoint file to port %d",
+                                      checkpoint_accept_timeout,
+                                      checkpoint_port);
+      }
+
+      FD_ZERO (&readset);
+      FD_SET (myGH->checkpoint_socket, &readset);
+      timeout.tv_sec = checkpoint_accept_timeout;
+      timeout.tv_usec = 0;
+      select (myGH->checkpoint_socket + 1, &readset, NULL, NULL, &timeout);
+    }
+    if (verbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING, "Closing checkpoint file on port %d",
+                  checkpoint_port);
+    }
+    IOHDF5_ERROR (H5Fclose (file));
+  }
+
+  /* restore original IO mode */
+  ioUtilGH->ioproc = old_ioproc;
+  ioUtilGH->nioprocs = old_nioprocs;
+  ioUtilGH->ioproc_every = old_ioproc_every;
+
+  /* stop the CP_TOTAL_TIMER timer and print timing information */
+  if (myGH->print_timing_info)
+  {
+    const char *timer_descriptions[3] = {"Time to dump parameters: ",
+                                         "Time to dump datasets:   ",
+                                         "Total time to checkpoint:"};
+
+
+    CCTK_TimerStopI (myGH->timers[CP_TOTAL_TIMER]);
+    IOUtil_PrintTimings ("Timing information for checkpointing "
+                         "with IOStreamedHDF5:",
+                         3, myGH->timers, timer_descriptions);
+  }
+
+  return (retval);
+}
diff --git a/src/RecoverGH.c b/src/RecoverGH.c
new file mode 100644
index 0000000..9bdcf18
--- /dev/null
+++ b/src/RecoverGH.c
@@ -0,0 +1,403 @@
+ /*@@
+   @file      RecoverGH.c
+   @date      Tue Oct 10 2000
+   @author    Thomas Radke
+   @desc 
+              Contains the routines to recover from a streamed HDF5 checkpoint.
+
+              Currently can recover from:
+                (1) One file containing recombined data
+                (2) Multiple unrecombined files, where the current
+                    number of processors and IO processors
+                    match those used to write the files.
+   @enddesc 
+   @version   $Id$
+ @@*/
+
+
+#include <stdlib.h>
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "CactusPUGH/PUGH/src/include/pugh.h"
+#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
+#include "ioStreamedHDF5GH.h"
+
+
+/* the rcs ID and its dummy function to use it */
+static char *rcsid = "$Id$";
+CCTK_FILEVERSION(BetaThorns_IOStreamedHDF5_RecoverGH_c)
+
+
+/* local function prototypes */
+static int IOStreamedHDF5_OpenFile (cGH *GH,
+                                    const char *basename,
+                                    int called_from,
+                                    fileinfo_t *fileinfo);
+
+
+ /*@@
+   @routine    IOStreamedHDF5_Recover
+   @date       Tue Oct 10 2000
+   @author     Thomas Radke
+   @desc 
+               Recovers a GH from a streamed HDF5 checkpoint file.
+               This routine is registered with IOUtil
+               as IOStreamedHDF5's recovery routine.
+   @enddesc 
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        basename
+   @vdesc      the basename containing the 'host:port' of the sending process
+   @vtype      const char *
+   @vio        in
+   @endvar
+   @var        called_from
+   @vdesc      flag indicating where this routine was called from
+               (either RECOVER or filereader)
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @calls      IOStreamedHDF5_OpenFile
+               IOHDF5Util_RecoverParameters
+               IOHDF5Util_RecoverVariables
+               IOHDF5Util_RecoverGHextensions
+               CCTK_TimerStartI
+               CCTK_TimerStopI
+               IOUtil_PrintTimings
+
+   @returntype int
+   @returndesc
+               -1 if there is no valid HDF5 file,
+               or the returncode of
+                 @seeroutine IOHDF5Util_RecoverParameters or
+                 @seeroutine IOHDF5Util_RecoverVariables or
+                 @seeroutine IOHDF5Util_RecoverGHextensions
+   @endreturndesc
+@@*/
+int IOStreamedHDF5_Recover (cGH *GH,
+                            const char *basename,
+                            int called_from)
+{
+  DECLARE_CCTK_PARAMETERS
+  int result;
+  ioStreamedHDF5GH *myGH;
+  static fileinfo_t fileinfo;  /* this is static because info is passed from
+                                  CP_RECOVERY_PARAMETERS to CP_RECOVERY_DATA */
+
+  /* start the recovery timer if we were called at CCTK_RECOVER */
+  myGH = (ioStreamedHDF5GH *) CCTK_GHExtension (GH, "IOStreamedHDF5");
+  if (myGH && myGH->print_timing_info)
+  {
+    CCTK_TimerStartI (myGH->timers[RECOVERY_TIMER]);
+  }
+
+  /* open the file if it wasn't already opened at CCTK_RECOVER_PARAMETERS */
+  /* FIXME Gab ... asymmetric levfac */
+  if (called_from == CP_RECOVER_PARAMETERS ||
+      called_from == FILEREADER_DATA ||
+      (GH && (GH->cctk_levfac[0] > 1 || GH->cctk_convlevel > 0)))
+  {
+    if (IOStreamedHDF5_OpenFile (GH, basename, called_from, &fileinfo) < 0)
+    {
+      return (-1);
+    }
+  }
+  else
+  {
+    /* This is the case for CP_RECOVER_DATA.
+       CCTK_RECOVER_PARAMETERS must have been called before 
+       and set up the file info structure. */
+    if (! fileinfo.is_HDF5_file)
+    {
+      return (-1);
+    }
+  }
+
+  /* if called at CCTK_RECOVER_PARAMETERS
+     just do this and return (keeping the file open) */
+  if (called_from == CP_RECOVER_PARAMETERS)
+  {
+    return (IOHDF5Util_RecoverParameters (&fileinfo));
+  }
+
+  /* Recover variables */
+  if (verbose)
+  {
+    CCTK_VInfo (CCTK_THORNSTRING, "Recovering %schunked data with ioproc %d, "
+                "ioproc_every %d", fileinfo.unchunked ? "un" : "",
+                fileinfo.ioproc, fileinfo.ioproc_every);
+  }
+  result = IOHDF5Util_RecoverVariables (GH, &fileinfo);
+
+  /* Recover GH extensions */
+  if (result == 0 && called_from == CP_RECOVER_DATA)
+  {
+    if (verbose)
+    {
+      CCTK_INFO ("Recovering GH extensions");
+    }
+    result = IOHDF5Util_RecoverGHextensions (GH, &fileinfo);
+  }
+
+  /* Close the file */
+  if (CCTK_MyProc (GH) == fileinfo.ioproc)
+  {
+    if (verbose)
+    {
+      if (called_from == CP_RECOVER_DATA)
+      {
+        CCTK_VInfo (CCTK_THORNSTRING, "Closing checkpoint file '%s' after "
+                                      "successful recovery", fileinfo.filename);
+      }
+      else
+      {
+        CCTK_VInfo (CCTK_THORNSTRING, "Closing data file '%s'",
+                    fileinfo.filename);
+      }
+    }
+    IOHDF5_ERROR (H5Fclose (fileinfo.file));
+  }
+ 
+  /* free the allocated filename */
+  if (fileinfo.filename)
+  {
+    free (fileinfo.filename);
+  }
+
+  /* stop recovery timer and print timing info */
+  if (called_from == CP_RECOVER_DATA && myGH->print_timing_info)
+  {
+    const char *timer_description = "Time to recover:";
+
+
+    CCTK_TimerStopI (myGH->timers[RECOVERY_TIMER]);
+    IOUtil_PrintTimings ("Timing information for recovery in IOStreamedHDF5:",
+                         1, &myGH->timers[RECOVERY_TIMER], &timer_description);
+  }
+
+  return (result);
+}
+
+
+ /*@@
+   @routine    IOStreamedHDF5_RecoverParameters
+   @date       Tue Oct 10 2000
+   @author     Thomas Radke
+   @desc
+               This routine is scheduled at CCTK_RECOVER_PARAMETERS.
+               It recovers the parameters from an HDF5 checkpoint file.
+
+               Note that it cannot be registered with IOUtil to be scheduled
+               from there (as done with the IOStreamedHDF5_Recover routine)
+               because the registration mechanism isn't activated yet
+               at CCTK_RECOVER_PARAMETERS.
+               Instead we call the generic parameter recovery routine
+               from IOUtil here, and just pass the necessary callback function
+               and its arguments.
+
+               Note also that this routine doesn't get passed any parameters,
+               not even a GH, because this doesn't exist yet at the time it is
+               being called.
+   @enddesc
+
+   @calls      IOUtil_RecoverParameters
+
+   @returntype int
+   @returndesc
+               -1 if an invalid reocvery mode was set
+               or the returncode of
+                 @seeroutine IOUtil_RecoverParameters
+   @endreturndesc
+@@*/
+int IOStreamedHDF5_RecoverParameters (void)
+{
+  DECLARE_CCTK_PARAMETERS
+  int retval;
+
+
+  if (CCTK_Equals (recover, "auto"))
+  {
+    CCTK_WARN (2, "'IO::recover = \"auto\"' selected which doesn't make sense "
+                  "for streamed HDF5 checkpoint files");
+    retval = -1;
+  }
+  else
+  {
+    retval = IOUtil_RecoverParameters (IOStreamedHDF5_Recover, NULL, NULL);
+  }
+
+  return (retval);
+}
+
+
+/**************************************************************************/
+/*                            local routines                              */
+/**************************************************************************/
+ /*@@
+   @routine    IOStreamedHDF5_OpenFile
+   @date       Tue Oct 10 2000
+   @author     Thomas Radke
+   @desc
+               Open a streamed HDF5 file given by its filename 
+               (which has format 'host:port') and checks whether
+               we can recover from that file.
+               The file information is broadcasted by the IO processor(s)
+               to all other processors so that everyone knows how to proceed.
+   @enddesc
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        basename
+   @vdesc      basename of the HDF5 file to recover from
+               For streamed files this should be of format 'host:port'.
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        called_from
+   @vdesc      flag indicating where this routine was called from
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        fileinfo
+   @vdesc      pointer to structure describing the file
+   @vtype      fileinfo_t *
+   @vio        out
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for success, -1 if file could not be opened
+   @endreturndesc
+@@*/
+static int IOStreamedHDF5_OpenFile (cGH *GH,
+                                    const char *basename,
+                                    int called_from,
+                                    fileinfo_t *fileinfo)
+{
+  DECLARE_CCTK_PARAMETERS
+  hid_t fapl;
+  hid_t group;
+  int nprocs, myproc;
+#ifdef CCTK_MPI
+  MPI_Comm comm;
+  CCTK_INT4 info[3];
+#endif
+
+
+#ifdef CCTK_MPI
+  /* Get the communicator for broadcasting the info structure */
+  /* NOTE: When recovering parameters thorn PUGH is not yet initialized
+           so that we have to use MPI_COMM_WORLD in this case */
+  comm = CCTK_GHExtensionHandle ("PUGH") < 0 ?
+           MPI_COMM_WORLD : PUGH_pGH (GH)->PUGH_COMM_WORLD;
+#endif
+
+  /* identify myself */
+  nprocs = CCTK_nProcs (GH);
+  myproc = CCTK_MyProc (GH);
+
+  /* use the basename (which should be of format 'host:port') as the filename */
+  fileinfo->filename = strdup (basename);
+
+  if (myproc == 0)
+  {
+    if (verbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING, "Opening streamed HDF5 file from '%s'",
+                  fileinfo->filename);
+    }
+
+    /* set up file access property list and select Stream VFD */
+    IOHDF5_ERROR (fapl = H5Pcreate (H5P_FILE_ACCESS));
+    IOHDF5_ERROR (H5Pset_fapl_stream (fapl, NULL));
+
+    fileinfo->file = H5Fopen (fileinfo->filename, H5F_ACC_RDONLY, fapl);
+    IOHDF5_ERROR (H5Pclose (fapl));
+    if (fileinfo->file < 0)
+    {
+      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Cannot open streamed file HDF5 file from '%s'",
+                  fileinfo->filename);
+    }
+    else if ((group = H5Gopen (fileinfo->file, GLOBAL_ATTRIBUTES_GROUP)) < 0)
+    {
+      CCTK_WARN (1, "Can't find global attributes group. "
+                    "Is this really a Cactus HDF5 datafile ?");
+    }
+    else
+    {
+      /* Determine how the data was written */
+      READ_ATTRIBUTE (group, "nprocs", H5T_NATIVE_INT, &fileinfo->nprocs);
+      READ_ATTRIBUTE (group, "unchunked", H5T_NATIVE_INT, &fileinfo->unchunked);
+      READ_ATTRIBUTE (group, "ioproc_every", H5T_NATIVE_INT,
+                      &fileinfo->ioproc_every);
+
+      IOHDF5_ERROR (H5Gclose (group));
+
+      /* If we recover from chunked file(s) the number of
+       * writing processors must match the number of reading 
+       * processors, and the total number of processors must match.
+       */
+ 
+      if ((fileinfo->ioproc_every == nprocs && nprocs > 1) ||
+          fileinfo->unchunked)
+      {
+        if (verbose)
+        {
+          CCTK_VInfo (CCTK_THORNSTRING, "Recovering from one %schunked file",
+                                        fileinfo->unchunked ? "un":"");
+        }
+        fileinfo->ioproc_every = nprocs;
+        fileinfo->is_HDF5_file = 1;
+      }
+      else
+      {
+        if (fileinfo->nprocs != nprocs)
+        {
+          CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                      "Must restart on %d processors with chunked files "
+                      "or recombine them", fileinfo->nprocs);
+        }
+        else
+        {
+          if (verbose)
+          {
+            CCTK_VInfo (CCTK_THORNSTRING, "Recovering from %d chunked files",
+                        nprocs / fileinfo->ioproc_every +
+                        (nprocs % fileinfo->ioproc_every ? 1 : 0));
+          }
+          fileinfo->is_HDF5_file = 1;
+        }
+      }
+    }
+  }
+
+  if (myproc == 0 && ! fileinfo->is_HDF5_file)
+  {
+    CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "No valid HDF5 file '%s' found", fileinfo->filename);
+  }
+
+#ifdef CCTK_MPI
+  /* Broadcast the file information to all processors
+     Need to convert everything into CCTK_INTs which can be communicated. */
+  info[0] = fileinfo->is_HDF5_file;
+  info[1] = fileinfo->unchunked;
+  info[2] = fileinfo->ioproc_every;
+  CACTUS_MPI_ERROR (MPI_Bcast (info, 3, PUGH_MPI_INT4, 0, comm));
+  fileinfo->is_HDF5_file = info[0];
+  fileinfo->unchunked = info[1];
+  fileinfo->ioproc_every = info[2];
+#endif
+
+  /* Return 0 for success otherwise negative */
+  return (fileinfo->is_HDF5_file ? 0 : -1);
+}
diff --git a/src/ioStreamedHDF5GH.h b/src/ioStreamedHDF5GH.h
new file mode 100644
index 0000000..6994ec2
--- /dev/null
+++ b/src/ioStreamedHDF5GH.h
@@ -0,0 +1,62 @@
+ /*@@
+   @header    ioStreamedHDF5GH.h
+   @date      Jun 20 2000
+   @author    Thomas Radke
+   @desc 
+              The GH extensions structure for IOStreamedHDF5.
+   @version $Id$
+ @@*/
+
+#ifndef _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_
+#define _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_
+
+#include "BetaThorns/IOHDF5Util/src/ioHDF5UtilGH.h"
+
+
+/* IOStreamedHDF5 GH extension structure */
+typedef struct
+{
+  /* how often to output */
+  int  out_every;
+
+  /* flags indicating output for var [i] */
+  char        *do_output;
+  ioHDF5Geo_t *geo_output;
+
+  /* the last iteration output */
+  int   *out_last;
+
+  /* sockets to output data and checkpoint files to */
+  int data_socket;
+  int checkpoint_socket;
+
+  /* file access property list used to write checkpoint files */
+  hid_t checkpoint_fapl;
+
+  /* flag indicating whether we want timing info on checkpointing/recovery
+     and corresponding array of Cactus timers */
+  int print_timing_info;
+  int timers[IOHDF5_NUM_TIMERS];
+
+} ioStreamedHDF5GH;
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* prototypes of functions to be registered as IOStreamedHDF5's IO method */
+int IOStreamedHDF5_OutputGH (cGH *GH);
+int IOStreamedHDF5_TriggerOutput (cGH *GH, int);
+int IOStreamedHDF5_TimeFor (cGH *GH, int);
+int IOStreamedHDF5_OutputVarAs (cGH *GH, const char *var, const char *alias);
+int IOStreamedHDF5_Recover (cGH *GH, const char *basename, int called_from);
+
+/* other function prototypes */
+void IOStreamedHDF5_Write (cGH *GH, int index, const char *alias);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif  /* _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_ */