/*@@
   @file      DumpGH.c
   @date      Wed Jun 10 14:13:35 1998
   @author    Paul Walker
   @desc
              Checkpoint routines scheduled at CCTK_CPINITIAL, CCTK_CHECKPOINT,
              and CCTK_TERMINATE.
              They check the IO checkpointing parameters and - if it's time
              to do so - call the routine which finally creates a checkpoint.
   @enddesc
   @version   $Id$
 @@*/


#include "cctk.h"
#include "cctk_Version.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
#include "ioFlexGH.h"

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

/* the rcs ID and its dummy funtion to use it */
static const char *rcsid = "$Id$";
CCTK_FILEVERSION(CactusPUGHIO_IOFlexIO_DumpGH_c)


/********************************************************************
 ********************    External Routines   ************************
 ********************************************************************/
void IOFlexIO_InitialDataCheckpoint (const cGH *GH);
void IOFlexIO_EvolutionCheckpoint (const cGH *GH);
void IOFlexIO_TerminationCheckpoint (const cGH *GH);


/********************************************************************
 ********************    Internal Routines   ************************
 ********************************************************************/
static int Checkpoint (const cGH *GH, int called_from);
 /*@@
   @routine    IOFlexIO_InitialDataCheckpoint
   @date       Fri Aug 21 14:46:28 1998
   @author     Gerd Lanfermann
   @desc
               This routine is registered at CCTK_CPINITIAL.
               It checks if initial data should be checkpointed.
   @enddesc

   @calls      Checkpoint

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
@@*/
void IOFlexIO_InitialDataCheckpoint (const cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint && checkpoint_ID)
  {
    if (! CCTK_Equals (verbose, "none"))
    {
      CCTK_INFO ("---------------------------------------------------------");
      CCTK_INFO ("Dumping initial data checkpoint");
      CCTK_INFO ("---------------------------------------------------------");
    }
    Checkpoint (GH, CP_INITIAL_DATA);
  }
}


 /*@@
   @routine    IOFlexIO_EvolutionCheckpoint
   @date       Fri Aug 21 14:38:25 1998
   @author     Gabrielle Allen
   @desc
               This routine is registered at CCTK_CHECKPOINT.
               It periodically checks if it's time to checkpoint evolution data.
   @enddesc

   @calls      Checkpoint

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
@@*/
void IOFlexIO_EvolutionCheckpoint (const cGH *GH)
{
  DECLARE_CCTK_PARAMETERS


  if (checkpoint &&
      ((checkpoint_every > 0 && GH->cctk_iteration % checkpoint_every == 0) ||
       checkpoint_next))
  {
    if (! CCTK_Equals (verbose, "none"))
    {
      CCTK_INFO ("---------------------------------------------------------");
      CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint at "
                  "iteration %d", GH->cctk_iteration);
      CCTK_INFO ("---------------------------------------------------------");
    }
    Checkpoint (GH, CP_EVOLUTION_DATA);

    /* reset the 'checkpoint_next' parameter */
    if (checkpoint_next)
    {
      CCTK_ParameterSet ("checkpoint_next", CCTK_THORNSTRING, "no");
    }
  }
}


 /*@@
   @routine    IOFlexIO_TerminationCheckpoint
   @date       Fri Aug 21 14:40:21 1998
   @author     Gabrielle Allen
   @desc
               This routine is registered at CCTK_TERMINATE.
               It checks if the last iteration should be checkpointed.
   @enddesc

   @calls      Checkpoint

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
@@*/
void IOFlexIO_TerminationCheckpoint (const cGH *GH)
{
  const flexioGH *myGH;
  DECLARE_CCTK_PARAMETERS


  myGH = CCTK_GHExtension (GH, "IOFlexIO");
  if (checkpoint && checkpoint_on_terminate && myGH)
  {
    if (myGH->last_checkpoint_iteration < GH->cctk_iteration)
    {
      if (! CCTK_Equals (verbose, "none"))
      {
        CCTK_INFO ("---------------------------------------------------------");
        CCTK_VInfo (CCTK_THORNSTRING, "Dumping termination checkpoint at "
                    "iteration %d", GH->cctk_iteration);
        CCTK_INFO ("---------------------------------------------------------");
      }
      Checkpoint (GH, CP_EVOLUTION_DATA);
    }
    else if (! CCTK_Equals (verbose, "none"))
    {
      CCTK_INFO ("---------------------------------------------------------");
      CCTK_VInfo (CCTK_THORNSTRING, "Termination checkpoint already dumped "
                  "as last evolution checkpoint at iteration %d",
                  myGH->last_checkpoint_iteration);
      CCTK_INFO ("---------------------------------------------------------");
    }
  }
}


 /*@@
   @routine    IOFlexIOi_DumpParameters
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc
               Gets the parameters of all active implementations and writes them
               as a single string attribute into a group GLOBAL_PARAMETERS.
   @enddesc
   @calls      IOUtil_GetAllParameters

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        all
   @vdesc      flag indicating whether to save all parameters or just the ones
               which have been set before
   @vtype      int
   @vio        in
   @endvar
   @var        file
   @vdesc      the IEEEIO file where to dump the parameters to
   @vtype      IOFile
   @vio        in
   @endvar
@@*/
void IOFlexIOi_DumpParameters (const cGH *GH, int all, IOFile file)
{
  char *parameters;


  parameters = IOUtil_GetAllParameters (GH, all);

  if (parameters)
  {
    FLEXIO_ERROR (IOwriteAttribute (file, GLOBAL_PARAMETERS, CHAR,
                                    strlen (parameters) + 1, parameters));
    free (parameters);
  }
}


 /*@@
   @routine    IOFlexIOi_DumpGHExtensions
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc
               Dumps the important variables from the grid hierarchy, PUGH,
               and IO into a group of an already opened IEEEIO file.
   @enddesc

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        file
   @vdesc      the IEEEIO file where to dump the GH extensions to
   @vtype      IOFile
   @vio        in
   @endvar
@@*/
void IOFlexIOi_DumpGHExtensions (const cGH *GH, IOFile file)
{
  CCTK_INT4 itmp;
  CCTK_REAL dtmp;
  const char *version;
  ioGH *ioUtilGH;


  /* get the handle for IOUtil extensions */
  ioUtilGH = CCTK_GHExtension (GH, "IO");

  itmp = CCTK_MainLoopIndex ();
  FLEXIO_ERROR (IOwriteAttribute (file, "main loop index",FLEXIO_INT4,1,&itmp));

  itmp = GH->cctk_iteration;
  FLEXIO_ERROR (IOwriteAttribute (file, "GH$iteration", FLEXIO_INT4, 1, &itmp));

  itmp = ioUtilGH->ioproc_every;
  FLEXIO_ERROR (IOwriteAttribute (file, "GH$ioproc_every",FLEXIO_INT4,1,&itmp));

  itmp = CCTK_nProcs (GH);
  FLEXIO_ERROR (IOwriteAttribute (file, "GH$nprocs", FLEXIO_INT4, 1, &itmp));

  dtmp = GH->cctk_time;
  FLEXIO_ERROR (IOwriteAttribute (file, "GH$time", FLEXIO_REAL, 1, &dtmp));

  version = CCTK_FullVersion ();
  FLEXIO_ERROR (IOwriteAttribute (file, "Cactus version", CHAR,
                                  strlen (version) + 1, version));
}


/********************************************************************
 ********************    Internal Routines   ************************
 ********************************************************************/
 /*@@
   @routine    Checkpoint
   @date       Fri Aug 21 15:13:13 1998
   @author     Paul Walker
   @desc
               The heart of checkpointing.
               Called by the different wrappers, this routine creates
               a new checkpoint file and then dumps away all grid variables.
   @enddesc

   @calls      IOUtil_AssembleFilename
               IOUtil_PrintTimings
               IOFlexIOi_DumpParameters
               IOFlexIOi_DumpGHExtensions

   @var        GH
   @vdesc      pointer to CCTK grid hierarchy
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        called_from
   @vdesc      flag indicating where this routine was called from
               (either CHECKPOINT_ID, CHECKPOINT, or filereader)
   @vtype      int
   @vio        in
   @endvar

   @history
   @hdate      Oct 4 1998
   @hauthor    Gabrielle Allen
   @hdesc      Removed code which checked and reset unchunked, assume that this
               is done when the variable is first set.
   @hdate      Nov 4 1998
   @hauthor    Gabrielle Allen
   @hdesc      Do a forced synchronization before checkpointing
   @hdate      Apr 16 1999
   @hauthor    Thomas Radke
   @hdesc      Removed forced sync before checkpointing (just do a normal sync)
               Introduced a ring buffer for keeping last <checkpoint_keep> files
   @endhistory

   @returntype int
   @returndesc
               the accumulated return code from @seeroutine IOFlexIO_DumpVar
               (should be 0 for success, or negative if some error occured), or
               -1 if checkpoint file could not be created
   @endreturndesc
@@*/
static int Checkpoint (const cGH *GH, int called_from)
{
  IOFile file;
  int dim, groupvarsize, myproc, first_vindex, gindex, retval;
  cGroup gdata;
  cGroupDynamicData gdynamicdata;
  ioRequest *request;
  flexioGH *myGH;
  const ioGH *ioUtilGH;
  char *filename, *tempname, *fullname;
  const char *timer_descriptions[] = {"Time to dump parameters: ",
                                      "Time to dump datasets:   ",
                                      "Total time to checkpoint:"};
  DECLARE_CCTK_PARAMETERS


  retval = 0;

  /* start the total timer */
  myGH = CCTK_GHExtension (GH, "IOFlexIO");
  if (myGH->print_timing_info)
  {
    CCTK_TimerStartI (myGH->timers[2]);
  }

  myproc = CCTK_MyProc (GH);
  ioUtilGH = CCTK_GHExtension (GH, "IO");

  /* get the filenames for both the temporary and final checkpoint file */
  filename = IOUtil_AssembleFilename (GH, NULL, "", ".ieee", called_from,
                                      myproc / ioUtilGH->ioproc_every,
                                      ioUtilGH->unchunked);
  tempname = IOUtil_AssembleFilename (GH, NULL, ".tmp", ".ieee", called_from,
                                      myproc / ioUtilGH->ioproc_every,
                                      ioUtilGH->unchunked);

  /* now open the file */
  if (myproc == ioUtilGH->ioproc)
  {
    if (CCTK_Equals (verbose, "full"))
    {
      CCTK_VInfo (CCTK_THORNSTRING, "Creating file '%s'", tempname);
    }
    file = IEEEopen (tempname, "w");
    if (! IOisValid (file))
    {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Can't open checkpoint file '%s'. Checkpointing is skipped",
                  tempname);
      return (-1);
    }
  }
  else
  {
    file = (IOFile) NULL;
  }

  /* start timer for dumping parameters */
  if (myGH->print_timing_info)
  {
    CCTK_TimerResetI (myGH->timers[0]);
    CCTK_TimerStartI (myGH->timers[0]);
  }

  /* first dump the parameters ... */
  if (file)
  {
    if (CCTK_Equals (verbose, "full"))
    {
      CCTK_INFO ("Dumping Params ...");
    }
    IOFlexIOi_DumpParameters (GH, 1, file);
    IOFlexIOi_DumpGHExtensions (GH, file);
  }

  /* stop parameter timer and start timer for dumping datasets */
  if (myGH->print_timing_info)
  {
    CCTK_TimerStopI (myGH->timers[0]);
    CCTK_TimerResetI (myGH->timers[1]);
    CCTK_TimerStartI (myGH->timers[1]);
  }

  /* ... now the variables, sorted by groups */
  if (CCTK_Equals (verbose, "full"))
  {
    CCTK_INFO ("Dumping Grid Variables ...");
  }
  for (gindex = CCTK_NumGroups () - 1; gindex >= 0; gindex--)
  {
    /* skip empty groups */
    if (CCTK_NumVarsInGroupI (gindex) <= 0)
    {
      continue;
    }

    /* only dump groups which have storage assigned */
    if (CCTK_QueryGroupStorageI (GH, gindex) <= 0)
    {
      continue;
    }

    /* get the number of allocated timelevels */
    CCTK_GroupData (gindex, &gdata);
    gdata.numtimelevels = 0;
    gdata.numtimelevels = CCTK_GroupStorageIncrease (GH, 1, &gindex,
                                                     &gdata.numtimelevels,NULL);

    /* dump all timelevels except the oldest (for multi-level groups) */
    if (gdata.numtimelevels > 1)
    {
      gdata.numtimelevels--;
    }

    /* skip zero-sized groups */
    CCTK_GroupDynamicData (GH, gindex, &gdynamicdata);
    groupvarsize = 1;
    for (dim = 0; dim < gdynamicdata.dim; dim++)
    {
      groupvarsize *= gdynamicdata.gsh[dim];
    }
    if (groupvarsize <= 0)
    {
      continue;
    }

    first_vindex = CCTK_FirstVarIndexI (gindex);

    /* get the default I/O request for this group */
    request = IOUtil_DefaultIORequest (GH, first_vindex, 1, -1.0);

    /* disable checking for old data objects,
       disable datatype conversion and downsampling,
       request hyperslab output including ghostzones */
    request->check_exist = 0;
    request->hdatatype = gdata.vartype;
    for (request->hdim = 0; request->hdim < request->vdim; request->hdim++)
    {
      request->downsample[request->hdim] = 1;
    }
    request->with_ghostzones = 1;

    /* loop over all variables in this group */
    for (request->vindex = first_vindex;
         request->vindex < first_vindex + gdata.numvars;
         request->vindex++)
    {
      /* loop over all allocated timelevels of this variable */
      for (request->timelevel = 0;
           request->timelevel < gdata.numtimelevels;
           request->timelevel++)
      {
        if (CCTK_Equals (verbose, "full"))
        {
          fullname = CCTK_FullName (request->vindex);
          CCTK_VInfo (CCTK_THORNSTRING, "  %s (timelevel %d)",
                      fullname, request->timelevel);
          free (fullname);
        }

        retval += IOFlexIO_DumpVar (GH, request, file);
      }
    } /* end of loop over all variables */
  } /* end of loop over all groups */

  /* stop timer for dumping datasets */
  if (myGH->print_timing_info)
  {
    CCTK_TimerStopI (myGH->timers[1]);
  }

  /* close the temporary checkpoint file */
  FLEXIO_ERROR (IOclose (file));

  /* free I/O request */
  IOUtil_FreeIORequest (&request);

  if (myproc == ioUtilGH->ioproc)
  {
    if (CCTK_Equals (verbose, "full"))
    {
      CCTK_VInfo (CCTK_THORNSTRING, "Closing and renaming checkpoint file "
                  "into '%s'", filename);
    }

#ifdef _WIN32
    /* Windows' rename(2) routine isn't POSIX compatible in that it would
       unlink an existing file */
    unlink (filename);
#endif
    if (rename (tempname, filename))
    {
      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Could not rename temporary checkpoint file %s to %s",
                  tempname, filename);
    }
  }

  /* delete the oldest dumpfile if checkpoint_keep_all isn't set
     and put the new filename into the ring buffer */
  if (called_from == CP_EVOLUTION_DATA && checkpoint_keep > 0)
  {
    if (myGH->cp_filenames[myGH->cp_fileindex])
    {
      remove (myGH->cp_filenames[myGH->cp_fileindex]);
      free (myGH->cp_filenames[myGH->cp_fileindex]);
      myGH->cp_filenames[myGH->cp_fileindex] = NULL;
    }

    myGH->cp_filenames[myGH->cp_fileindex] = strdup (filename);
    myGH->cp_fileindex = (myGH->cp_fileindex+1) % checkpoint_keep;
    if (myGH->checkpoint_keep != checkpoint_keep)
    {
      char **cp_filenames = calloc (checkpoint_keep, sizeof (char *));
      int min = myGH->checkpoint_keep < checkpoint_keep ?
                myGH->checkpoint_keep : checkpoint_keep;
      while (min-- > 0)
      {
        cp_filenames[min] = myGH->cp_filenames[min];
      }
      free (myGH->cp_filenames);
      myGH->cp_filenames = cp_filenames;
      myGH->checkpoint_keep = checkpoint_keep;
    }
  }

  /* stop total checkpoint timer and print timing info */
  if (myGH->print_timing_info)
  {
    CCTK_TimerStopI (myGH->timers[2]);
    IOUtil_PrintTimings ("Timing information for checkpointing in IOFlexIO:", 3,
                         myGH->timers, timer_descriptions);
  }

  /* save the iteration number of this checkpoint */
  myGH->last_checkpoint_iteration = GH->cctk_iteration;

  /* free allocated resources */
  free (tempname);
  free (filename);

  return (retval);
}