/*@@
   @file      DumpGH.c
   @date      Wed Jun 10 14:13:35 1998
   @author    Paul Walker
   @desc 
   DumpGH dumps an entire grid hierarchy (except for 1D arrays) to a 
   checkpoint file. This file also contains the different wrappers for
   IOFlexIODumpGH: in case of initial data, termination or regular checkpointing
   @enddesc 
   @version $Id$
 @@*/


#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "ioFlexGH.h"

/* the rcs ID and its dummy funtion to use it */
static char *rcsid = "$Id$";
CCTK_FILEVERSION (CactusPUGHIO_IOFlexIO_DumpGH_c);


 /*@@
   @routine    IOFlexIO_ConditionallyDumpGH
   @date       Fri Aug 21 14:38:25 1998
   @author    Gabrielle Allen
   @desc 
     registered as CCTK_CHECKPOINT, checks if it's time for 
     checkpointing, sets the checkpoint type and calls DumpGH 
   @enddesc 
   @calls    IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_ConditionallyDumpGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS

  if (checkpoint_every > 0 &&
      GH->cctk_iteration % checkpoint_every == 0) {
    if (verbose) {
      CCTK_INFO("------------------------------------------------------------");
      CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint file at "
                  "iteration %d", GH->cctk_iteration);
    }
    IOFlexIO_DumpGH (GH, CP_EVOLUTION_DATA);
  }
  
}


 /*@@
   @routine    IOFlexIO_TerminationDumpGH
   @date       Fri Aug 21 14:40:21 1998
   @author     Gabrielle Allen
   @desc 
     This routine is registered as CCTK_TERMINATE, it checks if a
     termination signal is raised (that is: global var cactus_terminate
     set to TERMINATION_RAISED_BRDCAST by ./steppers/TerminationStepper.c)
     and checkpoints all available grid hierarchies.
   @enddesc 
   @calls     IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_TerminationDumpGH (cGH *GH)
{
/*** FIXME: no cactus_terminate flag anymore ? ***/
#if	0
  if (cactus_terminate == TERMINATION_RAISED_BRDCAST) {
    IOFlexIO_DumpGH (GH, CP_EVOLUTION_DATA); 
  }
#else
  CCTK_WARN (1, "TerminationDumpGH() not yet implemented !");
#endif
}

 /*@@
   @routine    IOFlexIO_InitialDataDumpGH
   @date       Fri Aug 21 14:46:28 1998
   @author     Gerd Lanfermann
   @desc 
     This routine dumps the initial data of the GHs, registered 
     as CCTK_CPINITIAL; sets cp_type to CP_INITIAL_DATA and calls
     DumpGH; BoxInBox needs special treatment since for now.
   @enddesc 
   @calls     IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_InitialDataDumpGH (cGH *GH)
{
/*** FIXME ***/
#if	0
  cGH *helperGH;
  static int maxlev = 0;
  int l;

#ifdef THORN_BOXINBOX
  /* This will go once we reorganize the CINitial calling structure! */
  /* or have a function to deregister a rfr routine */
  /* BoxinBox does its own: get maxlevel (once!) and do dumpGH for all GH*/
  /* when this is called by the highest box */
  if (Contains("boxinbox","yes")) {
    if (maxlev==0) while (GHbyLevel(0,maxlev+1,0)) maxlev++;
    if (maxlev==0) return; 
    
    if (GH->level==maxlev) 
      for (l=0;l<=maxlev;l++) {
	printf("Dumping BoxinBox level %d of %d \n",l,maxlev);
	IOFlexIO_DumpGH(GHbyLevel(0,l,0), CP_INITIAL_DATA);
      }
    return;
  }
#endif
#endif
  IOFlexIO_DumpGH (GH, CP_INITIAL_DATA);
}


 /*@@
   @routine    IOFlexIOi_DumpParameters
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc
     Gets the parameters of all active implementations as a single string
     and writes it as an attribute into a group of an already opened HDF5 file.
   @enddesc
   @calledby   IOFlexIO_Write3D, IOFlexIO_DumpGH
   @history

   @endhistory
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
   @var        iof
   @vdesc      the IEEEIO file where to dump the parameters to
   @vtype      IOFile
   @vio        in
   @endvar
@@*/
void IOFlexIOi_DumpParameters (cGH *GH, IOFile iof)
{
  char *parameters;


  parameters = IOUtil_GetAllParameters (GH);

  if (parameters) {
    CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, GLOBAL_PARAMETERS, FLEXIO_CHAR,
                         strlen (parameters) + 1, parameters));
    free (parameters);
  }
}


 /*@@
   @routine    IOFlexIOi_DumpGHExtensions
   @date       Thu Jan 20 2000
   @author     Thomas Radke
   @desc
     Dumps the important variables from the grid hierarchy, PUGH, and IO
     into a group of an already opened IEEEIO file.
   @enddesc
   @calledby   IOFlexIO_Write3D, IOFlexIO_DumpGH
   @history

   @endhistory
   @var        GH
   @vdesc      Pointer to CCTK grid hierarchy
   @vtype      cGH
   @vio        in
   @endvar
   @var        iof
   @vdesc      the IEEEIO file where to dump the GH extensions to
   @vtype      IOFile
   @vio        in
   @endvar
@@*/
void IOFlexIOi_DumpGHExtensions (cGH *GH, IOFile iof)
{
  CCTK_INT4 i_temp;
  CCTK_REAL d_temp;
  ioGH *ioUtilGH;
  extern int PUGH_MainLoopIndex (void);

  /* Get the handle for IOUtil extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];

  i_temp = PUGH_MainLoopIndex ();
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "main loop index", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = GH->cctk_iteration;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$iteration", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = ioUtilGH->ioproc_every;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$ioproc_every", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = CCTK_nProcs (GH);
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$nprocs", FLEXIO_INT4,
                       1, &i_temp));

  d_temp = GH->cctk_time;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$time", FLEXIO_REAL,
                       1, &d_temp));
}


 /*@@
   @routine    IOFlexIO_DumpGH
   @date       Fri Aug 21 15:13:13 1998
   @author     Paul Walker
   @desc 
     The heart of checkpointing. Called by the different wrappers, this
     routines get the appropriate filename by IOUtil_PrepareFilename() and 
     then dumps away using the dump routines from IO ...
   @enddesc 
   @calledby   IOFlexIO_ConditionallyDumpGH, IOFlexIO_TerminationDumpGH,
               IOFlexIO_InitialDataDumpGH
   @history 
   @hdate Oct 4 1998 @hauthor Gabrielle Allen
   @hdesc Removed code which checked and reset unchunked, assume that this 
          is done when the variable is first set.
   @hdate Nov 4 1998 @hauthor Gabrielle Allen
   @hdesc Do a forced synchronization before checkpointing
   @hdate Apr 16 1999 @hauthor Thomas Radke
   @hdesc Removed forced sync before checkpointing (just do a normal sync)
          Introduced a ring buffer for keeping last <checkpoint_keep> files
   @endhistory 

@@*/

void IOFlexIO_DumpGH (cGH *GH, int called_from)
{
  DECLARE_CCTK_PARAMETERS
  char dumpfname [1024], tmpfname [1024];
  IOFile iof;
  int index;
  int timelevel, current_timelevel;
  int *old_downsample;
  int old_out_single;
  ioGH *ioUtilGH;
  flexioGH *myGH;
  static char **dumpfnames = NULL;  /* dump filename ring buffer */
  static int findex = 0;            /* index into ring buffer */


  /* Get the handles for IOUtil and IOFlexIO extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
  myGH = (flexioGH *) GH->extensions [CCTK_GHExtensionHandle ("IOFlexIO")];

  /* allocate the ring buffer for filenames */
  if (! dumpfnames)
    dumpfnames = (char **) calloc (checkpoint_keep, sizeof (char *));

  /* disable downsampling after saving original downsampling params */
  old_downsample = (int *) malloc (CCTK_MaxDim () * sizeof (int));
  for (index = 0; index < CCTK_MaxDim (); index++) {
    old_downsample [index] = ioUtilGH->downsample [index];
    ioUtilGH->downsample [index] = 1;
  }

  /* disable output in single precision */
  old_out_single = ioUtilGH->out_single;
  ioUtilGH->out_single = 0;

  /* start the total timer */
  if (print_timing_info)
    CCTK_TimerStartI (myGH->checkpointTotalTimer);

  /* sync all active groups */
  for (index = 0; index < CCTK_NumGroups (); index++)
    if (CCTK_IsImplementationActive (CCTK_ImpFromVarI (
                                       CCTK_FirstVarIndexI (index))))
      CCTK_SyncGroupI (GH, index);

  /* get the base filename ... */
  IOUtil_PrepareFilename (GH, NULL, dumpfname, called_from,
                          CCTK_MyProc (GH) / ioUtilGH->ioproc_every,
                          ioUtilGH->unchunked);

  /* ... and append the extension */
  sprintf (tmpfname,  "%s.chkpt_tmp.ieee", dumpfname);
  sprintf (dumpfname, "%s.chkpt.ieee",     dumpfname);

  /* Now open the file */
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {
    if (verbose)
      CCTK_VInfo (CCTK_THORNSTRING, "Creating file '%s'", tmpfname);
    iof = IEEEopen (tmpfname, "w");
    if (! IOisValid (iof)) {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Can't open checkpoint file '%s'. Checkpointing is skipped",
                  tmpfname);
      return;
    }
  } else
    iof = (IOFile) NULL;

  /* Great; Now start dumping away! */

  /* start timer for dumping parameters */
  if (print_timing_info) {
    CCTK_TimerResetI (myGH->dumpParamsTimer);
    CCTK_TimerStartI (myGH->dumpParamsTimer);
  }

  /* first the parameters ... */
  if (iof) {
    if (out3D_parameters) {
      if (verbose)
        CCTK_INFO ("Dumping Params ...");
      IOFlexIOi_DumpParameters (GH, iof);
    }
    IOFlexIOi_DumpGHExtensions (GH, iof);
  }

  /* stop parameter timer and start timer for dumping datasets */
  if (print_timing_info) {
    CCTK_TimerStopI (myGH->dumpParamsTimer);
    CCTK_TimerResetI (myGH->dumpVarsTimer);
    CCTK_TimerStartI (myGH->dumpVarsTimer);
  }

  /* ... now the variables */
  for (index = 0; index < CCTK_NumVars (); index++) {
    const char *thorn, *impl = CCTK_ImpFromVarI (index);

    /* find out the thorn implementing variable with index */
    thorn = CCTK_ImplementationThorn (impl);
    if (! thorn)
      thorn = impl;

    /* let only variables pass which belong to an active thorn and
       have storage assigned */
    if (! CCTK_IsThornActive (thorn) ||
        ! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index)))
      continue;

    if (verbose)
      CCTK_VInfo (CCTK_THORNSTRING, "  %s", CCTK_VarName (index));

    /* get the current timelevel */
    current_timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
    if (current_timelevel > 0)
      current_timelevel--;

    /* now dump all timelevels up to the current */
    for (timelevel = 0; timelevel <= current_timelevel; timelevel++)
      IOFlexIO_DumpVar (GH, index, timelevel, iof);
  } /* end of loop over all variables */

  /* stop timer for dumping datasets */
  if (print_timing_info)
    CCTK_TimerStopI (myGH->dumpVarsTimer);

  CACTUS_IEEEIO_ERROR (IOclose (iof));
  
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {
    if (verbose)
      CCTK_VInfo (CCTK_THORNSTRING, "Closing and renaming checkpoint file "
                  "into '%s'", dumpfname);

    if (rename (tmpfname, dumpfname))
      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Could not rename temporary checkpoint file %s to %s",
                  tmpfname, dumpfname);
  }

  /* delete the oldest dumpfile if checkpoint_keep_all isn't set
     and put the new filename into the ring buffer */
  if (dumpfnames [findex]) {
    if (! checkpoint_keep_all)
      remove (dumpfnames [findex]);
    free (dumpfnames [findex]);
  }
  dumpfnames [findex] = strdup (dumpfname);
  findex = (findex + 1) % checkpoint_keep;

  /* restore output precision flag */
  ioUtilGH->out_single = old_out_single;

  /* restore original downsampling params */
  memcpy (ioUtilGH->downsample, old_downsample, CCTK_MaxDim () * sizeof (int));
  free (old_downsample);

  /* stop total checkpoint timer and print timing info */
  if (print_timing_info) {
    cTimerData *info;

    CCTK_TimerStopI (myGH->checkpointTotalTimer);

    CCTK_INFO ("timing information for checkpointing:");

/*** FIXME: select timer to get info from at runtime ***/
#ifdef HAVE_TIME_GETTIMEOFDAY
    info = CCTK_TimerCreateData();
    if (info) {

      if (myGH->dumpParamsTimer>= 0) {
        CCTK_TimerI (myGH->dumpParamsTimer, info);
        CCTK_VInfo (CCTK_THORNSTRING, "  Time to dump parameters:  %5.1f sec",
                    info->vals[0].val.d);
      }

      if (myGH->dumpVarsTimer >= 0) {
        CCTK_TimerI (myGH->dumpVarsTimer, info);
        CCTK_VInfo (CCTK_THORNSTRING, "  Time to dump datasets:    %5.1f sec",
                    info->vals[0].val.d);
      }

      if (myGH->checkpointTotalTimer >= 0) {
        CCTK_TimerI (myGH->checkpointTotalTimer, info);
        CCTK_VInfo (CCTK_THORNSTRING, "  Total time to checkpoint: %5.1f sec",
                    info->vals[0].val.d);
      }

      CCTK_TimerDestroyData (info);
    } else
      CCTK_WARN (1, "Couldn't create timer info structure ! No timing output "
                    "available.");
#else
    CCTK_WARN (1, "No GETTIMEOFDAY timer available.");
#endif

    CCTK_INFO ("--------------------------------------------------------------"
               "-------------\n");
  }
}