/*@@
   @file      DumpGH.c
   @date      Fri Oct 6 2000
   @author    Thomas Radke
   @desc 
              Checkpoint routines scheduled at CCTK_CPINITIAL, CCTK_CHECKPOINT,
              and CCTK_TERMINATE.
              They check the IO checkpointing parameters and - if it's time
              to do so - call the routine which finally creates a checkpoint.
   @enddesc 
   @version   $Id$
 @@*/


#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusPUGH/PUGH/src/include/pugh.h"
#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
#include "BetaThorns/IOHDF5Util/src/ioHDF5UtilGH.h"
#include "ioStreamedHDF5GH.h"

#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

/* the rcs ID and its dummy function to use it */
static const char *rcsid = "$Id$";
CCTK_FILEVERSION(BetaThorns_IOStreamedHDF5_DumpGH_c)


/* prototypes of routines defined in this source file */
void IOStreamedHDF5_InitialDataCheckpoint (cGH *GH);
void IOStreamedHDF5_EvolutionCheckpoint (cGH *GH);
void IOStreamedHDF5_TerminationCheckpoint (cGH *GH);
static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from);


 /*@@
   @routine    IOStreamedHDF5_InitialDataCheckpoint
   @date       Fri Oct 6 2000
   @author     Thomas Radke
   @desc 
               This routine is registered at CCTK_CPINITIAL.
               It checks if initial data should be checkpointed.
   @enddesc 

   @calls      IOStreamedHDF5_Checkpoint

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
   @vtype      cGH *
   @vio        in
   @endvar
@@*/
void IOStreamedHDF5_InitialDataCheckpoint (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint && checkpoint_ID)
  {
    IOStreamedHDF5_Checkpoint (GH, CP_INITIAL_DATA);
  }
}


 /*@@
   @routine    IOStreamedHDF5_EvolutionCheckpoint
   @date       Fri Oct 6 2000
   @author     Thomas Radke
   @desc 
               This routine is registered at CCTK_CHECKPOINT.
               It periodically checks if it's time to checkpoint evolution data.
   @enddesc 
 
   @calls      IOStreamedHDF5_Checkpoint

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
   @vtype      cGH *
   @vio        in
   @endvar
@@*/
void IOStreamedHDF5_EvolutionCheckpoint (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint &&
      ((checkpoint_every > 0 && GH->cctk_iteration % checkpoint_every == 0) ||
       checkpoint_next))
  {
    if (verbose)
    {
      CCTK_INFO("------------------------------------------------------------");
      CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint file at "
                  "iteration %d", GH->cctk_iteration);
    }
    IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA);

    /* reset the 'checkpoint_next' parameter */
    if (checkpoint_next)
    {
      CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no");
    }
  }
}


 /*@@
   @routine    IOStreamedHDF5_TerminationCheckpoint
   @date       Fri Oct 6 2000
   @author     Thomas Radke
   @desc 
               This routine is registered at CCTK_TERMINATE.
               It checks if the last iteration should be checkpointed.
   @enddesc 

   @calls      IOStreamedHDF5_Checkpoint
 
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
   @vtype      cGH *
   @vio        in
   @endvar
@@*/
void IOStreamedHDF5_TerminationCheckpoint (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint && checkpoint_on_terminate)
  {
    IOStreamedHDF5_Checkpoint (GH, CP_EVOLUTION_DATA);
  }
}


/*****************************************************************************/
/*                             local routines                                */
/*****************************************************************************/
 /*@@
   @routine    IOStreamedHDF5_Checkpoint
   @date       Fri Oct 6 2000
   @author     Thomas Radke
   @desc 
               The heart of checkpointing.
               Called by the different wrappers, this routine creates
               a new checkpoint file and then dumps away using the
               dump routines from IOHDF5Util.
   @enddesc 

   @calls      IOHDF5Util_DumpGH

   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy to be checkpointed
   @vtype      cGH *
   @vio        in
   @endvar
   @var        called_from
   @vdesc      flag indicating where this routine was called from
               (either CHECKPOINT_ID, CHECKPOINT, or filereader)
   @vtype      int
   @vio        in
   @endvar
   @returntype int
   @returndesc
               -1 if checkpoint file could not be created
               or returncode of @seeroutine IOUtilHDF5_DumpGH 
   @endreturndesc
@@*/
static int IOStreamedHDF5_Checkpoint (cGH *GH, int called_from)
{
  DECLARE_CCTK_PARAMETERS
  hid_t file;
  int old_ioproc;
  int old_nioprocs;
  int old_ioproc_every;
  fd_set readset;
  struct timeval timeout;
  ioGH *ioUtilGH;
  ioStreamedHDF5GH *myGH;
  CCTK_INT4 retval;


  /* suppress compiler warning about unused variables */
  called_from = called_from;

  retval = 0;
  file = -1;

  /* Get the GH extensions for IOUtil and IOStreamedHDF5 */
  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
  myGH = (ioStreamedHDF5GH *) CCTK_GHExtension (GH, "IOStreamedHDF5");

  /* check if IOStreamedHDF5 was registered as IO method */
  if (myGH == NULL)
  {
    CCTK_WARN (2, "No IOStreamedHDF5 IO method registered");
    return (-1);
  }

  /* start the CP_TOTAL_TIMER timer */
  if (myGH->print_timing_info)
  {
    CCTK_TimerStartI (myGH->timers[CP_TOTAL_TIMER]);
  }

  /* for now we can only have IO mode "oneproc" */
  if (ioUtilGH->nioprocs != 1)
  {
    CCTK_WARN (2, "Output into multiple chunked files not yet implemented. "
                  "Switching to IO mode \"oneproc\".");
  }
  old_ioproc = ioUtilGH->ioproc;
  ioUtilGH->ioproc = 0;
  old_nioprocs = ioUtilGH->nioprocs;
  ioUtilGH->nioprocs = 1;
  old_ioproc_every = ioUtilGH->ioproc_every;
  ioUtilGH->ioproc_every = CCTK_nProcs (GH);

  /* Now open the file */
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc && myGH->checkpoint_socket >= 0)
  {
    if (verbose)
    {
      CCTK_VInfo (CCTK_THORNSTRING, "Creating checkpoint file");
    }

    file = H5Fcreate ("dummy_name", H5F_ACC_TRUNC, H5P_DEFAULT,
                      myGH->checkpoint_fapl);
    if (file < 0)
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Can't create checkpoint file on port %d. "
                  "Checkpointing is skipped.", checkpoint_port);
      retval = -1;
    }
  }
  else
  {
    retval = -1;
  }

#ifdef CCTK_MPI
  /* broadcast the result of the H5Fcreate() call */
  CACTUS_MPI_ERROR (MPI_Bcast (&retval, 1, PUGH_MPI_INT4, 0,
                               PUGH_pGH (GH)->PUGH_COMM_WORLD));
#endif

  if (retval == 0)
  {
    retval = IOHDF5Util_DumpGH (GH,
                                myGH->print_timing_info ? myGH->timers : NULL,
                                file);
  }

  if (file >= 0)
  {
    if (checkpoint_accept_timeout > 0)
    {
      if (verbose)
      {
        CCTK_VInfo (CCTK_THORNSTRING, "Waiting %d seconds to send "
                                      "checkpoint file to port %d",
                                      checkpoint_accept_timeout,
                                      checkpoint_port);
      }

      FD_ZERO (&readset);
      FD_SET (myGH->checkpoint_socket, &readset);
      timeout.tv_sec = checkpoint_accept_timeout;
      timeout.tv_usec = 0;
      select (myGH->checkpoint_socket + 1, &readset, NULL, NULL, &timeout);
    }
    if (verbose)
    {
      CCTK_VInfo (CCTK_THORNSTRING, "Closing checkpoint file on port %d",
                  checkpoint_port);
    }
    IOHDF5_ERROR (H5Fclose (file));
  }

  /* restore original IO mode */
  ioUtilGH->ioproc = old_ioproc;
  ioUtilGH->nioprocs = old_nioprocs;
  ioUtilGH->ioproc_every = old_ioproc_every;

  /* stop the CP_TOTAL_TIMER timer and print timing information */
  if (myGH->print_timing_info)
  {
    const char *timer_descriptions[3] = {"Time to dump parameters: ",
                                         "Time to dump datasets:   ",
                                         "Total time to checkpoint:"};


    CCTK_TimerStopI (myGH->timers[CP_TOTAL_TIMER]);
    IOUtil_PrintTimings ("Timing information for checkpointing "
                         "with IOStreamedHDF5:",
                         3, myGH->timers, timer_descriptions);
  }

  return (retval);
}