path: root/src/DumpGH.c

 /*@@
   @file      DumpGH.c
   @date      Wed Jun 10 14:13:35 1998
   @author    Paul Walker
   @desc 
   DumpGH dumps an entire grid hierarchy (except for 1D arrays) to a 
   checkpoint file. This file also contains the different wrappers for
   IOHDF5_DumpGH: in case of initial data, termination or regular checkpointing
   @enddesc 
   @version $Id$
 @@*/


#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
#include "ioHDF5GH.h"

/* the rcs ID and its dummy funtion to use it */
static char *rcsid = "$Id$";
CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_DumpGH_c)


/* local function prototypes */
void IOHDF5i_DumpParameters (cGH *GH, hid_t group);


 /*@@
   @routine    IOHDF5_ConditionallyDumpGH
   @date       Fri Aug 21 14:38:25 1998
   @author     Gabrielle Allen
   @desc 
     registered as CCTK_CHECKPOINT, checks if it's time for 
     checkpointing, sets the checkpoint type and calls IOHDF5_DumpGH 
   @enddesc 
   @calledby   CCTK Scheduler
   @history 
 
   @endhistory 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
@@*/

void IOHDF5_ConditionallyDumpGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint &&
      ((checkpoint_every > 0 && GH->cctk_iteration % checkpoint_every == 0) ||
       checkpoint_next))
  {
    if (verbose)
    {
      CCTK_INFO("------------------------------------------------------------");
      CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint file at "
                  "iteration %d", GH->cctk_iteration);
    }
    IOHDF5_DumpGH (GH, CP_EVOLUTION_DATA);

    /* reset the 'checkpoint_next' parameter */
    if (checkpoint_next)
    {
      CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no");
    }
  }
}


 /*@@
   @routine    IOHDF5_TerminationDumpGH
   @date       Fri Aug 21 14:40:21 1998
   @author     Gabrielle Allen
   @desc 
               This routine is registered as CCTK_TERMINATE
               and does a checkpoint if the parameter
               'IO::checkpoint_on_terminate' was set.
   @enddesc 
   @calledby   CCTK Scheduler
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
@@*/
void IOHDF5_TerminationDumpGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint && checkpoint_on_terminate)
  {
    IOHDF5_DumpGH (GH, CP_EVOLUTION_DATA);
  }
}

 /*@@
   @routine    IOHDF5_InitialDataDumpGH
   @date       Fri Aug 21 14:46:28 1998
   @author     Gerd Lanfermann
   @desc 
     This routine dumps the initial data of the GHs, registered 
     as CCTK_CPINITIAL; sets 'called_from' flag to CP_INITIAL_DATA
     and calls IOHDF5_DumpGH.
     BoxInBox needs special treatment since for now.
   @enddesc 
   @calledby   CCTK Scheduler
   @history 
 
   @endhistory 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
@@*/

void IOHDF5_InitialDataDumpGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS

/*** FIXME ***/
#if 0
#ifdef THORN_BOXINBOX
  /* This will go once we reorganize the CINitial calling structure! */
  /* or have a function to deregister a rfr routine */
  /* BoxinBox does its own: get maxlevel (once!) and do dumpGH for all GH*/
  /* when this is called by the highest box */
  if (Contains("boxinbox","yes")) {
    int l;
    cGH *helperGH;
    static int maxlev = 0;

    if (maxlev == 0) while (GHbyLevel(0,maxlev+1,0)) maxlev++;
    if (maxlev == 0) return; 
    
    if (GH->level == maxlev) 
      for (l = 0; l <= maxlev; l++) {
        CCTK_VInfo (CCTK_THORNSTRING, "Dumping BoxinBox level %d of %d",
                    l, maxlev);
        IOHDF5_DumpGH (GHbyLevel (0, l, 0), CP_INITIAL_DATA);
      }
    return;
  }
#endif
#endif

  if (checkpoint && checkpoint_ID)
  {
    IOHDF5_DumpGH (GH, CP_INITIAL_DATA);
  }
}


 /*@@
   @routine    IOHDF5i_DumpParameters
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc 
     Gets the parameters of all active implementations as a single string
     and writes it as an attribute into a group of an already opened HDF5 file.
   @enddesc 
   @calledby   IOHDF5_Write3D, IOHDF5_DumpGH
   @history 
 
   @endhistory 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
   @var        group
   @vdesc      the group where to dump the parameters to
   @vtype      hid_t
   @vio        in
   @endvar
@@*/
void IOHDF5i_DumpParameters (cGH *GH, hid_t group)
{
  ioHDF5GH *myGH;
  char *parameters;


  /* Get the parameter string buffer */
  parameters = IOUtil_GetAllParameters (GH);

  if (parameters) {
    /* Get the handle for IOHDF5 extensions */
    myGH = (ioHDF5GH *) GH->extensions [CCTK_GHExtensionHandle ("IOHDF5")]; 

    WRITE_ATTRIBUTE (GLOBAL_PARAMETERS, parameters, group,
                     myGH->scalarDataspace, 0, myGH->IOHDF5_STRING);
    free (parameters);
  }
}


 /*@@
   @routine    IOHDF5i_DumpGHExtensions
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc 
     Dumps the important variables from the grid hierarchy, PUGH, and IO
     into a group of an already opened HDF5 file.
   @enddesc 
   @calledby   IOHDF5_Write3D, IOHDF5_DumpGH
   @history 
 
   @endhistory 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
   @var        group
   @vdesc      the group where to dump the GH extensions to
   @vtype      hid_t
   @vio        in
   @endvar
@@*/
void IOHDF5i_DumpGHExtensions (cGH *GH, hid_t group)
{
  int attrValue, main_loop_index;
  ioGH *ioUtilGH;
  ioHDF5GH *myGH;


  /* Get the handles for IOUtil and IOHDF5 extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
  myGH = (ioHDF5GH *) GH->extensions [CCTK_GHExtensionHandle ("IOHDF5")]; 

  main_loop_index = CCTK_MainLoopIndex ();

  attrValue = CCTK_nProcs (GH);
  WRITE_ATTRIBUTE ("nprocs", &attrValue, group,
                   myGH->scalarDataspace, 0, H5T_NATIVE_INT);
  WRITE_ATTRIBUTE ("ioproc_every", &ioUtilGH->ioproc_every, group,
                   myGH->scalarDataspace, 0, H5T_NATIVE_INT);
  WRITE_ATTRIBUTE ("unchunked", &ioUtilGH->unchunked, group,
                   myGH->scalarDataspace, 0, H5T_NATIVE_INT);
  WRITE_ATTRIBUTE ("cctk_time", &GH->cctk_time, group,
                   myGH->scalarDataspace, 0, IOHDF5_REAL);
  WRITE_ATTRIBUTE ("cctk_iteration", &GH->cctk_iteration, group,
                   myGH->scalarDataspace, 0, H5T_NATIVE_INT);
  WRITE_ATTRIBUTE ("main_loop_index", &main_loop_index, group,
                   myGH->scalarDataspace, 0, H5T_NATIVE_INT);

}


 /*@@
   @routine    IOHDF5_DumpGH
   @date       Fri Aug 21 15:13:13 1998
   @author     Paul Walker
   @desc 
     The heart of checkpointing. Called by the different wrappers, this
     routines get the appropriate filename by IOUtil_PrepareFilename() and 
     then dumps away using the dump routines from IO ...
   @enddesc 
   @calledby   IOHDF5_ConditionallyDumpGH, IOHDF5_TerminationDumpGH,
               IOHDF5_InitialDataDumpGH
   @history 
   @hdate Oct 4 1998 @hauthor Gabrielle Allen
   @hdesc Removed code which checked and reset unchunked, assume that this 
          is done when the variable is first set.
   @hdate Nov 4 1998 @hauthor Gabrielle Allen
   @hdesc Do a forced synchronization before checkpointing
   @hdate Apr 16 1999 @hauthor Thomas Radke
   @hdesc Removed forced sync before checkpointing (just do a normal sync)
          Introduced a ring buffer for keeping last <checkpoint_keep> files
   @endhistory 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
   @var        called_from
   @vdesc      flag indicating where this routine was called from
               (either CHECKPOINT_ID, CHECKPOINT, or filereader)
   @vtype      int
   @vio        in
   @endvar
@@*/

void IOHDF5_DumpGH (cGH *GH, int called_from)
{
  DECLARE_CCTK_PARAMETERS
  char dumpfname [1024], tmpfname [1024];
  hid_t iof;
  int index,idim;
  int timelevel, current_timelevel;
  int *old_downsample;
  int old_out_single;
  ioGH *ioUtilGH;
  ioHDF5GH *myGH;
  ioHDF5Geo_t geo;
  static char **dumpfnames = NULL;  /* dump filename ring buffer */
  static int findex = 0;            /* index into ring buffer */


  /* Get the handles for IOUtil and IOHDF5 extensions */
  if ((index = CCTK_GHExtensionHandle ("IO")) < 0)
    return;
  ioUtilGH = (ioGH *) GH->extensions [index];
  if ((index = CCTK_GHExtensionHandle ("IOHDF5")) < 0)
    return;
  myGH = (ioHDF5GH *) GH->extensions [index];

  /* allocate the ring buffer for filenames */
  if (! dumpfnames)
    dumpfnames = (char **) calloc (checkpoint_keep, sizeof (char *));

  /* disable downsampling after saving original downsampling params */
  old_downsample = (int *) malloc (CCTK_MaxDim () * sizeof (int));
  for (index = 0; index < CCTK_MaxDim (); index++) {
    old_downsample [index] = ioUtilGH->downsample [index];
    ioUtilGH->downsample [index] = 1;
  }

  /* disable output in single precision */
  old_out_single = ioUtilGH->out_single;
  ioUtilGH->out_single = 0;

  /* start the total timer */
  if (myGH->print_timing_info)
    CCTK_TimerStartI (myGH->timers[2]);

  /* sync all groups */
  for (index = 0; index < CCTK_NumGroups (); index++)
    if (CCTK_IsImplementationActive (CCTK_ImpFromVarI (
                                       CCTK_FirstVarIndexI (index))))
      CCTK_SyncGroupI (GH, index);

  /* get the base filename ... */
  IOUtil_PrepareFilename (GH, NULL, dumpfname, called_from,
                          CCTK_MyProc (GH) / ioUtilGH->ioproc_every,
                          ioUtilGH->unchunked);

  /* ... and append the extension */
  sprintf (tmpfname,  "%s.tmp.h5", dumpfname);
  sprintf (dumpfname, "%s.h5",     dumpfname);

  /* Now open the file */
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {
    if (verbose)
      CCTK_VInfo (CCTK_THORNSTRING, "Creating checkpoint file '%s'", tmpfname);

    iof = H5Fcreate (tmpfname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
    if (iof < 0) {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Can't open checkpoint file '%s'. Checkpointing is skipped",
                  tmpfname);
      return;
    }
  } else
    iof = -1;

  /* Great; Now start dumping away! */

  /* start timer for dumping parameters */
  if (myGH->print_timing_info) {
    CCTK_TimerResetI (myGH->timers[0]);
    CCTK_TimerStartI (myGH->timers[0]);
  }

  if (iof >= 0) {
    hid_t group;

    /* all GH extension variables and parameter stuff which is not
       specific to any dataset goes into dedicated groups */
    if (out3D_parameters) {
      if (verbose)
        CCTK_INFO ("Dumping Parameters ...");

      CACTUS_IOHDF5_ERROR (group = H5Gcreate (iof, GLOBAL_PARAMETERS_GROUP, 0));
      IOHDF5i_DumpParameters (GH, group);
      CACTUS_IOHDF5_ERROR (H5Gclose (group));
    }

    if (verbose)
      CCTK_INFO ("Dumping GH extensions ...");

    CACTUS_IOHDF5_ERROR (group = H5Gcreate (iof, GHEXTENSIONS_GROUP, 0));
    IOHDF5i_DumpGHExtensions (GH, group);
    CACTUS_IOHDF5_ERROR (H5Gclose (group));
  }

  /* stop parameter timer and start timer for dumping datasets */
  if (myGH->print_timing_info) {
    CCTK_TimerStopI (myGH->timers[0]);
    CCTK_TimerResetI (myGH->timers[1]);
    CCTK_TimerStartI (myGH->timers[1]);
  }

  /* Set the geometry variables manually, do not use the 
     geometry structures set by parameters; vdim,sdim are set below */
  for (idim=0; idim<IOHDF5_MAXDIM;idim++) {
    geo.direction[idim]  = idim;
    geo.slab_start[idim] = 0;
    geo.length[idim]     = -1;
    geo.downs[idim]      = 1;
  }

  if (verbose)
    CCTK_INFO ("Dumping variables ...");

  /* ... now the variables */
  for (index = 0; index < CCTK_NumVars (); index++) {
    char oldFlag;
    const char *thorn, *impl = CCTK_ImpFromVarI (index);

    /* find out the thorn implementing variable with index */
    thorn = CCTK_ImplementationThorn (impl);
    if (! thorn)
      thorn = impl;

    /* let only variables pass which belong to an active thorn and
       have storage assigned */
    if (! CCTK_IsThornActive (thorn) ||
        ! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index)))
      continue;

    if (verbose && iof >= 0)
      CCTK_VInfo (CCTK_THORNSTRING, "  %s", CCTK_VarName (index));

    /* get the current timelevel */
    current_timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
    if (current_timelevel > 0)
      current_timelevel--;
    
    /* Set dimension information based on the variable */
    geo.vdim = CCTK_GroupDimFromVarI(index);
    geo.sdim = geo.vdim;

    /* reset this flag because checkpoint files are always created anew */
    oldFlag = myGH->checkForExistingObjects [index];
    myGH->checkForExistingObjects [index] = 0;

    /* now dump all timelevels up to the current */
    for (timelevel = 0; timelevel <= current_timelevel; timelevel++)
      IOHDF5_DumpVar (GH, index, timelevel, &geo, iof);

    /* restore the original value of the flag */
    myGH->checkForExistingObjects [index] = oldFlag;

  } /* end of loop over all variables */

  /* stop timer for dumping datasets */
  if (myGH->print_timing_info)
    CCTK_TimerStopI (myGH->timers[1]);

  if (iof >= 0)
    CACTUS_IOHDF5_ERROR (H5Fclose (iof));
  
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {

    /* rename successfully created temporary file
       and overwrite the old checkpoint file */
    if (verbose)
      CCTK_VInfo (CCTK_THORNSTRING, "Closing and renaming checkpoint file "
                  "into '%s'", dumpfname);

    if (rename (tmpfname, dumpfname))
      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Could not rename temporary checkpoint file '%s' to '%s'",
                  tmpfname, dumpfname);

    /* delete the oldest checkpoint file if checkpoint_keep_all isn't set
       and put the new filename into the ring buffer */
    if (dumpfnames [findex]) {
      if (! checkpoint_keep_all)
        remove (dumpfnames [findex]);
      free (dumpfnames [findex]);
    }
    dumpfnames [findex] = strdup (dumpfname);
    findex = (findex + 1) % checkpoint_keep;
  }

  /* restore output precision flag */
  ioUtilGH->out_single = old_out_single;

  /* restore original downsampling params */
  memcpy (ioUtilGH->downsample, old_downsample, CCTK_MaxDim () * sizeof (int));
  free (old_downsample);

  /* stop total checkpoint timer and print timing info */
  if (myGH->print_timing_info) {
    const char *timer_descriptions [3] = {"Time to dump parameters: ",
                                          "Time to dump datasets:   ",
                                          "Total time to checkpoint:"};


    CCTK_TimerStopI (myGH->timers[2]);
    IOUtil_PrintTimings ("Timing information for checkpointing in IOHDF5:", 3,
                         myGH->timers, timer_descriptions);
  }
}