From 9ea1696c72e3c1761ffb63875bbc63c924261ade Mon Sep 17 00:00:00 2001 From: tradke Date: Thu, 12 Oct 2000 11:55:36 +0000 Subject: Added checkpoint/recovery functionality. Moved a lot of code into IOHDF5Util and inherit from this thorn. Forgot to 'cvs add' these files at my last commit. git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOStreamedHDF5/trunk@45 0888f3d4-9f52-45d2-93bc-d00801ff5e46 --- src/DumpGH.c | 305 +++++++++++++++++++++++++++++++++++++ src/RecoverGH.c | 403 +++++++++++++++++++++++++++++++++++++++++++++++++ src/ioStreamedHDF5GH.h | 62 ++++++++ 3 files changed, 770 insertions(+) create mode 100644 src/DumpGH.c create mode 100644 src/RecoverGH.c create mode 100644 src/ioStreamedHDF5GH.h diff --git a/src/DumpGH.c b/src/DumpGH.c new file mode 100644 index 0000000..c31c1e1 --- /dev/null +++ b/src/DumpGH.c @@ -0,0 +1,305 @@ + /*@@ + @file DumpGH.c + @date Fri Oct 6 2000 + @author Thomas Radke + @desc + Checkpoint routines scheduled at CCTK_CPINITIAL, CCTK_CHECKPOINT, + and CCTK_TERMINATE. + They check the IO checkpointing parameters and - if it's time + to do so - call the routine which finally creates a checkpoint. + @enddesc + @version $Id$ + @@*/ + + +#include "cctk.h" +#include "cctk_Parameters.h" +#include "CactusBase/IOUtil/src/ioGH.h" +#include "CactusPUGH/PUGH/src/include/pugh.h" +#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h" +#include "BetaThorns/IOHDF5Util/src/ioHDF5UtilGH.h" +#include "ioStreamedHDF5GH.h" + +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +/* the rcs ID and its dummy function to use it */ +static char *rcsid = "$Id$"; +CCTK_FILEVERSION(BetaThorns_IOStreamedHDF5_DumpGH_c) + + +/* local function prototypes */ +static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from); + + + /*@@ + @routine IOStreamedHDF5_InitialDataCheckpoint + @date Fri Oct 6 2000 + @author Thomas Radke + @desc + This routine is registered at CCTK_CPINITIAL. + It checks if initial data should be checkpointed. + @enddesc + + @calls IOStreamedHDF5_Checkpoint + + @var GH + @vdesc Pointer to CCTK grid hierarchy to be checkpointed + @vtype cGH * + @vio in + @endvar +@@*/ +void IOStreamedHDF5_InitialDataCheckpoint (cGH *GH) +{ + DECLARE_CCTK_PARAMETERS + + + if (checkpoint && checkpoint_ID) + { + IOStreamedHDF5_Checkpoint (GH, CP_INITIAL_DATA); + } +} + + + /*@@ + @routine IOStreamedHDF5_EvolutionCheckpoint + @date Fri Oct 6 2000 + @author Thomas Radke + @desc + This routine is registered at CCTK_CHECKPOINT. + It periodically checks if it's time to checkpoint evolution data. + @enddesc + + @calls IOStreamedHDF5_Checkpoint + + @var GH + @vdesc Pointer to CCTK grid hierarchy to be checkpointed + @vtype cGH * + @vio in + @endvar +@@*/ +void IOStreamedHDF5_EvolutionCheckpoint (cGH *GH) +{ + DECLARE_CCTK_PARAMETERS + + + if (checkpoint && + ((checkpoint_every > 0 && GH->cctk_iteration % checkpoint_every == 0) || + checkpoint_next)) + { + if (verbose) + { + CCTK_INFO("------------------------------------------------------------"); + CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint file at " + "iteration %d", GH->cctk_iteration); + } + IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA); + + /* reset the 'checkpoint_next' parameter */ + if (checkpoint_next) + { + CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no"); + } + } +} + + + /*@@ + @routine IOStreamedHDF5_TerminationCheckpoint + @date Fri Oct 6 2000 + @author Thomas Radke + @desc + This routine is registered at CCTK_TERMINATE. + It checks if the last iteration should be checkpointed. + @enddesc + + @calls IOStreamedHDF5_Checkpoint + + @var GH + @vdesc Pointer to CCTK grid hierarchy to be checkpointed + @vtype cGH * + @vio in + @endvar +@@*/ +void IOStreamedHDF5_TerminationCheckpoint (cGH *GH) +{ + DECLARE_CCTK_PARAMETERS + + + if (checkpoint && checkpoint_on_terminate) + { + IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA); + } +} + + +/*****************************************************************************/ +/* local routines */ +/*****************************************************************************/ + /*@@ + @routine IOStreamedHDF5_Checkpoint + @date Fri Oct 6 2000 + @author Thomas Radke + @desc + The heart of checkpointing. + Called by the different wrappers, this routine creates + a new checkpoint file and then dumps away using the + dump routines from IOHDF5Util. + @enddesc + + @calls IOHDF5Util_DumpGH + + @var GH + @vdesc Pointer to CCTK grid hierarchy to be checkpointed + @vtype cGH * + @vio in + @endvar + @var called_from + @vdesc flag indicating where this routine was called from + (either CHECKPOINT_ID, CHECKPOINT, or filereader) + @vtype int + @vio in + @endvar + @returntype int + @returndesc + -1 if checkpoint file could not be created + or returncode of @seeroutine IOUtilHDF5_DumpGH + @endreturndesc +@@*/ +static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from) +{ + DECLARE_CCTK_PARAMETERS + hid_t file; + int old_ioproc; + int old_nioprocs; + int old_ioproc_every; + fd_set readset; + struct timeval timeout; + ioGH *ioUtilGH; + ioStreamedHDF5GH *myGH; + CCTK_INT4 retval; + + + retval = 0; + file = -1; + + /* Get the GH extensions for IOUtil and IOStreamedHDF5 */ + ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); + myGH = (ioStreamedHDF5GH *) CCTK_GHExtension (GH, "IOStreamedHDF5"); + + /* check if IOStreamedHDF5 was registered as IO method */ + if (myGH == NULL) + { + CCTK_WARN (2, "No IOStreamedHDF5 IO method registered"); + return (-1); + } + + /* start the CP_TOTAL_TIMER timer */ + if (myGH->print_timing_info) + { + CCTK_TimerStartI (myGH->timers[CP_TOTAL_TIMER]); + } + + /* for now we can only have IO mode "oneproc" */ + if (ioUtilGH->nioprocs != 1) + { + CCTK_WARN (2, "Output into multiple chunked files not yet implemented. " + "Switching to IO mode \"oneproc\"."); + } + old_ioproc = ioUtilGH->ioproc; + ioUtilGH->ioproc = 0; + old_nioprocs = ioUtilGH->nioprocs; + ioUtilGH->nioprocs = 1; + old_ioproc_every = ioUtilGH->ioproc_every; + ioUtilGH->ioproc_every = CCTK_nProcs (GH); + + /* Now open the file */ + if (CCTK_MyProc (GH) == ioUtilGH->ioproc && myGH->checkpoint_socket >= 0) + { + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Creating checkpoint file"); + } + + file = H5Fcreate ("dummy_name", H5F_ACC_TRUNC, H5P_DEFAULT, + myGH->checkpoint_fapl); + if (file < 0) + { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Can't create checkpoint file on port %d. " + "Checkpointing is skipped.", checkpoint_port); + retval = -1; + } + } + else + { + retval = -1; + } + +#ifdef CCTK_MPI + /* broadcast the result of the H5Fcreate() call */ + CACTUS_MPI_ERROR (MPI_Bcast (&retval, 1, PUGH_MPI_INT4, 0, + PUGH_pGH (GH)->PUGH_COMM_WORLD)); +#endif + + if (retval == 0) + { + retval = IOHDF5Util_DumpGH (GH, + myGH->print_timing_info ? myGH->timers : NULL, + file); + } + + if (file >= 0) + { + if (checkpoint_accept_timeout > 0) + { + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Waiting %d seconds to send " + "checkpoint file to port %d", + checkpoint_accept_timeout, + checkpoint_port); + } + + FD_ZERO (&readset); + FD_SET (myGH->checkpoint_socket, &readset); + timeout.tv_sec = checkpoint_accept_timeout; + timeout.tv_usec = 0; + select (myGH->checkpoint_socket + 1, &readset, NULL, NULL, &timeout); + } + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Closing checkpoint file on port %d", + checkpoint_port); + } + IOHDF5_ERROR (H5Fclose (file)); + } + + /* restore original IO mode */ + ioUtilGH->ioproc = old_ioproc; + ioUtilGH->nioprocs = old_nioprocs; + ioUtilGH->ioproc_every = old_ioproc_every; + + /* stop the CP_TOTAL_TIMER timer and print timing information */ + if (myGH->print_timing_info) + { + const char *timer_descriptions[3] = {"Time to dump parameters: ", + "Time to dump datasets: ", + "Total time to checkpoint:"}; + + + CCTK_TimerStopI (myGH->timers[CP_TOTAL_TIMER]); + IOUtil_PrintTimings ("Timing information for checkpointing " + "with IOStreamedHDF5:", + 3, myGH->timers, timer_descriptions); + } + + return (retval); +} diff --git a/src/RecoverGH.c b/src/RecoverGH.c new file mode 100644 index 0000000..9bdcf18 --- /dev/null +++ b/src/RecoverGH.c @@ -0,0 +1,403 @@ + /*@@ + @file RecoverGH.c + @date Tue Oct 10 2000 + @author Thomas Radke + @desc + Contains the routines to recover from a streamed HDF5 checkpoint. + + Currently can recover from: + (1) One file containing recombined data + (2) Multiple unrecombined files, where the current + number of processors and IO processors + match those used to write the files. + @enddesc + @version $Id$ + @@*/ + + +#include + +#include "cctk.h" +#include "cctk_Parameters.h" +#include "CactusPUGH/PUGH/src/include/pugh.h" +#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h" +#include "ioStreamedHDF5GH.h" + + +/* the rcs ID and its dummy function to use it */ +static char *rcsid = "$Id$"; +CCTK_FILEVERSION(BetaThorns_IOStreamedHDF5_RecoverGH_c) + + +/* local function prototypes */ +static int IOStreamedHDF5_OpenFile (cGH *GH, + const char *basename, + int called_from, + fileinfo_t *fileinfo); + + + /*@@ + @routine IOStreamedHDF5_Recover + @date Tue Oct 10 2000 + @author Thomas Radke + @desc + Recovers a GH from a streamed HDF5 checkpoint file. + This routine is registered with IOUtil + as IOStreamedHDF5's recovery routine. + @enddesc + @var GH + @vdesc Pointer to CCTK grid hierarchy + @vtype cGH * + @vio in + @endvar + @var basename + @vdesc the basename containing the 'host:port' of the sending process + @vtype const char * + @vio in + @endvar + @var called_from + @vdesc flag indicating where this routine was called from + (either RECOVER or filereader) + @vtype int + @vio in + @endvar + + @calls IOStreamedHDF5_OpenFile + IOHDF5Util_RecoverParameters + IOHDF5Util_RecoverVariables + IOHDF5Util_RecoverGHextensions + CCTK_TimerStartI + CCTK_TimerStopI + IOUtil_PrintTimings + + @returntype int + @returndesc + -1 if there is no valid HDF5 file, + or the returncode of + @seeroutine IOHDF5Util_RecoverParameters or + @seeroutine IOHDF5Util_RecoverVariables or + @seeroutine IOHDF5Util_RecoverGHextensions + @endreturndesc +@@*/ +int IOStreamedHDF5_Recover (cGH *GH, + const char *basename, + int called_from) +{ + DECLARE_CCTK_PARAMETERS + int result; + ioStreamedHDF5GH *myGH; + static fileinfo_t fileinfo; /* this is static because info is passed from + CP_RECOVERY_PARAMETERS to CP_RECOVERY_DATA */ + + /* start the recovery timer if we were called at CCTK_RECOVER */ + myGH = (ioStreamedHDF5GH *) CCTK_GHExtension (GH, "IOStreamedHDF5"); + if (myGH && myGH->print_timing_info) + { + CCTK_TimerStartI (myGH->timers[RECOVERY_TIMER]); + } + + /* open the file if it wasn't already opened at CCTK_RECOVER_PARAMETERS */ + /* FIXME Gab ... asymmetric levfac */ + if (called_from == CP_RECOVER_PARAMETERS || + called_from == FILEREADER_DATA || + (GH && (GH->cctk_levfac[0] > 1 || GH->cctk_convlevel > 0))) + { + if (IOStreamedHDF5_OpenFile (GH, basename, called_from, &fileinfo) < 0) + { + return (-1); + } + } + else + { + /* This is the case for CP_RECOVER_DATA. + CCTK_RECOVER_PARAMETERS must have been called before + and set up the file info structure. */ + if (! fileinfo.is_HDF5_file) + { + return (-1); + } + } + + /* if called at CCTK_RECOVER_PARAMETERS + just do this and return (keeping the file open) */ + if (called_from == CP_RECOVER_PARAMETERS) + { + return (IOHDF5Util_RecoverParameters (&fileinfo)); + } + + /* Recover variables */ + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Recovering %schunked data with ioproc %d, " + "ioproc_every %d", fileinfo.unchunked ? "un" : "", + fileinfo.ioproc, fileinfo.ioproc_every); + } + result = IOHDF5Util_RecoverVariables (GH, &fileinfo); + + /* Recover GH extensions */ + if (result == 0 && called_from == CP_RECOVER_DATA) + { + if (verbose) + { + CCTK_INFO ("Recovering GH extensions"); + } + result = IOHDF5Util_RecoverGHextensions (GH, &fileinfo); + } + + /* Close the file */ + if (CCTK_MyProc (GH) == fileinfo.ioproc) + { + if (verbose) + { + if (called_from == CP_RECOVER_DATA) + { + CCTK_VInfo (CCTK_THORNSTRING, "Closing checkpoint file '%s' after " + "successful recovery", fileinfo.filename); + } + else + { + CCTK_VInfo (CCTK_THORNSTRING, "Closing data file '%s'", + fileinfo.filename); + } + } + IOHDF5_ERROR (H5Fclose (fileinfo.file)); + } + + /* free the allocated filename */ + if (fileinfo.filename) + { + free (fileinfo.filename); + } + + /* stop recovery timer and print timing info */ + if (called_from == CP_RECOVER_DATA && myGH->print_timing_info) + { + const char *timer_description = "Time to recover:"; + + + CCTK_TimerStopI (myGH->timers[RECOVERY_TIMER]); + IOUtil_PrintTimings ("Timing information for recovery in IOStreamedHDF5:", + 1, &myGH->timers[RECOVERY_TIMER], &timer_description); + } + + return (result); +} + + + /*@@ + @routine IOStreamedHDF5_RecoverParameters + @date Tue Oct 10 2000 + @author Thomas Radke + @desc + This routine is scheduled at CCTK_RECOVER_PARAMETERS. + It recovers the parameters from an HDF5 checkpoint file. + + Note that it cannot be registered with IOUtil to be scheduled + from there (as done with the IOStreamedHDF5_Recover routine) + because the registration mechanism isn't activated yet + at CCTK_RECOVER_PARAMETERS. + Instead we call the generic parameter recovery routine + from IOUtil here, and just pass the necessary callback function + and its arguments. + + Note also that this routine doesn't get passed any parameters, + not even a GH, because this doesn't exist yet at the time it is + being called. + @enddesc + + @calls IOUtil_RecoverParameters + + @returntype int + @returndesc + -1 if an invalid reocvery mode was set + or the returncode of + @seeroutine IOUtil_RecoverParameters + @endreturndesc +@@*/ +int IOStreamedHDF5_RecoverParameters (void) +{ + DECLARE_CCTK_PARAMETERS + int retval; + + + if (CCTK_Equals (recover, "auto")) + { + CCTK_WARN (2, "'IO::recover = \"auto\"' selected which doesn't make sense " + "for streamed HDF5 checkpoint files"); + retval = -1; + } + else + { + retval = IOUtil_RecoverParameters (IOStreamedHDF5_Recover, NULL, NULL); + } + + return (retval); +} + + +/**************************************************************************/ +/* local routines */ +/**************************************************************************/ + /*@@ + @routine IOStreamedHDF5_OpenFile + @date Tue Oct 10 2000 + @author Thomas Radke + @desc + Open a streamed HDF5 file given by its filename + (which has format 'host:port') and checks whether + we can recover from that file. + The file information is broadcasted by the IO processor(s) + to all other processors so that everyone knows how to proceed. + @enddesc + + @var GH + @vdesc Pointer to CCTK grid hierarchy + @vtype cGH * + @vio in + @endvar + @var basename + @vdesc basename of the HDF5 file to recover from + For streamed files this should be of format 'host:port'. + @vtype int + @vio in + @endvar + @var called_from + @vdesc flag indicating where this routine was called from + @vtype int + @vio in + @endvar + @var fileinfo + @vdesc pointer to structure describing the file + @vtype fileinfo_t * + @vio out + @endvar + + @returntype int + @returndesc + 0 for success, -1 if file could not be opened + @endreturndesc +@@*/ +static int IOStreamedHDF5_OpenFile (cGH *GH, + const char *basename, + int called_from, + fileinfo_t *fileinfo) +{ + DECLARE_CCTK_PARAMETERS + hid_t fapl; + hid_t group; + int nprocs, myproc; +#ifdef CCTK_MPI + MPI_Comm comm; + CCTK_INT4 info[3]; +#endif + + +#ifdef CCTK_MPI + /* Get the communicator for broadcasting the info structure */ + /* NOTE: When recovering parameters thorn PUGH is not yet initialized + so that we have to use MPI_COMM_WORLD in this case */ + comm = CCTK_GHExtensionHandle ("PUGH") < 0 ? + MPI_COMM_WORLD : PUGH_pGH (GH)->PUGH_COMM_WORLD; +#endif + + /* identify myself */ + nprocs = CCTK_nProcs (GH); + myproc = CCTK_MyProc (GH); + + /* use the basename (which should be of format 'host:port') as the filename */ + fileinfo->filename = strdup (basename); + + if (myproc == 0) + { + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Opening streamed HDF5 file from '%s'", + fileinfo->filename); + } + + /* set up file access property list and select Stream VFD */ + IOHDF5_ERROR (fapl = H5Pcreate (H5P_FILE_ACCESS)); + IOHDF5_ERROR (H5Pset_fapl_stream (fapl, NULL)); + + fileinfo->file = H5Fopen (fileinfo->filename, H5F_ACC_RDONLY, fapl); + IOHDF5_ERROR (H5Pclose (fapl)); + if (fileinfo->file < 0) + { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot open streamed file HDF5 file from '%s'", + fileinfo->filename); + } + else if ((group = H5Gopen (fileinfo->file, GLOBAL_ATTRIBUTES_GROUP)) < 0) + { + CCTK_WARN (1, "Can't find global attributes group. " + "Is this really a Cactus HDF5 datafile ?"); + } + else + { + /* Determine how the data was written */ + READ_ATTRIBUTE (group, "nprocs", H5T_NATIVE_INT, &fileinfo->nprocs); + READ_ATTRIBUTE (group, "unchunked", H5T_NATIVE_INT, &fileinfo->unchunked); + READ_ATTRIBUTE (group, "ioproc_every", H5T_NATIVE_INT, + &fileinfo->ioproc_every); + + IOHDF5_ERROR (H5Gclose (group)); + + /* If we recover from chunked file(s) the number of + * writing processors must match the number of reading + * processors, and the total number of processors must match. + */ + + if ((fileinfo->ioproc_every == nprocs && nprocs > 1) || + fileinfo->unchunked) + { + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Recovering from one %schunked file", + fileinfo->unchunked ? "un":""); + } + fileinfo->ioproc_every = nprocs; + fileinfo->is_HDF5_file = 1; + } + else + { + if (fileinfo->nprocs != nprocs) + { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Must restart on %d processors with chunked files " + "or recombine them", fileinfo->nprocs); + } + else + { + if (verbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "Recovering from %d chunked files", + nprocs / fileinfo->ioproc_every + + (nprocs % fileinfo->ioproc_every ? 1 : 0)); + } + fileinfo->is_HDF5_file = 1; + } + } + } + } + + if (myproc == 0 && ! fileinfo->is_HDF5_file) + { + CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, + "No valid HDF5 file '%s' found", fileinfo->filename); + } + +#ifdef CCTK_MPI + /* Broadcast the file information to all processors + Need to convert everything into CCTK_INTs which can be communicated. */ + info[0] = fileinfo->is_HDF5_file; + info[1] = fileinfo->unchunked; + info[2] = fileinfo->ioproc_every; + CACTUS_MPI_ERROR (MPI_Bcast (info, 3, PUGH_MPI_INT4, 0, comm)); + fileinfo->is_HDF5_file = info[0]; + fileinfo->unchunked = info[1]; + fileinfo->ioproc_every = info[2]; +#endif + + /* Return 0 for success otherwise negative */ + return (fileinfo->is_HDF5_file ? 0 : -1); +} diff --git a/src/ioStreamedHDF5GH.h b/src/ioStreamedHDF5GH.h new file mode 100644 index 0000000..6994ec2 --- /dev/null +++ b/src/ioStreamedHDF5GH.h @@ -0,0 +1,62 @@ + /*@@ + @header ioStreamedHDF5GH.h + @date Jun 20 2000 + @author Thomas Radke + @desc + The GH extensions structure for IOStreamedHDF5. + @version $Id$ + @@*/ + +#ifndef _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_ +#define _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_ + +#include "BetaThorns/IOHDF5Util/src/ioHDF5UtilGH.h" + + +/* IOStreamedHDF5 GH extension structure */ +typedef struct +{ + /* how often to output */ + int out_every; + + /* flags indicating output for var [i] */ + char *do_output; + ioHDF5Geo_t *geo_output; + + /* the last iteration output */ + int *out_last; + + /* sockets to output data and checkpoint files to */ + int data_socket; + int checkpoint_socket; + + /* file access property list used to write checkpoint files */ + hid_t checkpoint_fapl; + + /* flag indicating whether we want timing info on checkpointing/recovery + and corresponding array of Cactus timers */ + int print_timing_info; + int timers[IOHDF5_NUM_TIMERS]; + +} ioStreamedHDF5GH; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* prototypes of functions to be registered as IOStreamedHDF5's IO method */ +int IOStreamedHDF5_OutputGH (cGH *GH); +int IOStreamedHDF5_TriggerOutput (cGH *GH, int); +int IOStreamedHDF5_TimeFor (cGH *GH, int); +int IOStreamedHDF5_OutputVarAs (cGH *GH, const char *var, const char *alias); +int IOStreamedHDF5_Recover (cGH *GH, const char *basename, int called_from); + +/* other function prototypes */ +void IOStreamedHDF5_Write (cGH *GH, int index, const char *alias); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* _IOSTREAMEDHDF5_IOSTREAMEDHDF5GH_H_ */ -- cgit v1.2.3