/*@@
   @file      DumpGH.c
   @date      Wed Jun 10 14:13:35 1998
   @author    Paul Walker
   @desc 
   DumpGH dumps an entire grid hierarchy (except for 1D arrays) to a 
   checkpoint file. This file also contains the different wrappers for
   IOFlexIODumpGH: in case of initial data, termination or regular checkpointing
   @enddesc 
   @version $Id$
 @@*/

static char *rcsid = "$Id$";

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "ioFlexGH.h"


 /*@@
   @routine    IOFlexIO_ConditionallyDumpGH
   @date       Fri Aug 21 14:38:25 1998
   @author    Gabrielle Allen
   @desc 
     registered as CCTK_CHECKPOINT, checks if it's time for 
     checkpointing, sets the checkpoint type and calls DumpGH 
   @enddesc 
   @calls    IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_ConditionallyDumpGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS

  if (checkpoint_every > 0 &&
      GH->cctk_iteration % checkpoint_every == 0) {
    if (verbose) {
      printf ("------------------------------------------------------------\n");
      printf ("Dumping periodic checkpoint file at iteration %d\n",
              GH->cctk_iteration);
    }
    IOFlexIO_DumpGH (GH, CP_EVOLUTION_DATA);
  }
  
}


 /*@@
   @routine    IOFlexIO_TerminationDumpGH
   @date       Fri Aug 21 14:40:21 1998
   @author     Gabrielle Allen
   @desc 
     This routine is registered as CCTK_TERMINATE, it checks if a
     termination signal is raised (that is: global var cactus_terminate
     set to TERMINATION_RAISED_BRDCAST by ./steppers/TerminationStepper.c)
     and checkpoints all available grid hierarchies.
   @enddesc 
   @calls     IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_TerminationDumpGH (cGH *GH)
{
/*** FIXME: no cactus_terminate flag anymore ? ***/
#if	0
  if (cactus_terminate == TERMINATION_RAISED_BRDCAST) {
    IOFlexIO_DumpGH (GH, CP_EVOLUTION_DATA); 
  }
#else
  CCTK_WARN (1, "TerminationDumpGH() not yet implemented !");
#endif
}

 /*@@
   @routine    IOFlexIO_InitialDataDumpGH
   @date       Fri Aug 21 14:46:28 1998
   @author     Gerd Lanfermann
   @desc 
     This routine dumps the initial data of the GHs, registered 
     as CCTK_CPINITIAL; sets cp_type to CP_INITIAL_DATA and calls
     DumpGH; BoxInBox needs special treatment since for now.
   @enddesc 
   @calls     IOFlexIO_DumpGH
   @history 
 
   @endhistory 

@@*/

void IOFlexIO_InitialDataDumpGH (cGH *GH)
{
/*** FIXME ***/
#if	0
  cGH *helperGH;
  static int maxlev = 0;
  int l;

#ifdef THORN_BOXINBOX
  /* This will go once we reorganize the CINitial calling structure! */
  /* or have a function to deregister a rfr routine */
  /* BoxinBox does its own: get maxlevel (once!) and do dumpGH for all GH*/
  /* when this is called by the highest box */
  if (Contains("boxinbox","yes")) {
    if (maxlev==0) while (GHbyLevel(0,maxlev+1,0)) maxlev++;
    if (maxlev==0) return; 
    
    if (GH->level==maxlev) 
      for (l=0;l<=maxlev;l++) {
	printf("Dumping BoxinBox level %d of %d \n",l,maxlev);
	IOFlexIO_DumpGH(GHbyLevel(0,l,0), CP_INITIAL_DATA);
      }
    return;
  }
#endif
#endif
  IOFlexIO_DumpGH (GH, CP_INITIAL_DATA);
}


void IOFlexIO_DumpParams (cGH *GH, IOFile iof)
{
  int first;
  const char *param;


  first = 1;
  while ((param = CCTK_ParameterWalk (first, NULL)) != NULL) {
    char *impl, *name, *aname, *value;
  
    first = 0;

    if (Util_SplitString (&impl, &name, param, "::") != 0)
      CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Couldn't parse name of parameter '%s'", param);
    else {
      value = CCTK_ParameterValString (name, CCTK_ImplementationThorn (impl));
      if (value == NULL)
        CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                    "Couldn't get value for parameter '%s'", param);
      else {
        aname = (char *) malloc (sizeof (PARAMETER_PREFIX) +
                                 sizeof (ANAME_DELIMITER) + strlen (param));
        sprintf (aname, "%s%s%s", PARAMETER_PREFIX, ANAME_DELIMITER, param);

        CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, aname, FLEXIO_CHAR,
                             strlen (value) + 1, value));
        free (aname);
        free (value);
      }

      free (impl);
      free (name);

    }

    free ((void *) param);

  } /* end of loop walking over all parameters */
}


void IOFlexIO_DumpGHExtensions (cGH *GH, IOFile iof)
{
  CCTK_INT4 i_temp;
  CCTK_REAL d_temp;
  ioGH *ioUtilGH;

  /* Get the handle for IOUtil extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];

  i_temp = CCTK_MainLoopIndex ();
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "main loop index", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = GH->cctk_iteration;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$iteration", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = ioUtilGH->ioproc_every;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$ioproc_every", FLEXIO_INT4,
                       1, &i_temp));

  i_temp = CCTK_nProcs (GH);
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$nprocs", FLEXIO_INT4,
                       1, &i_temp));

  d_temp = GH->cctk_time;
  CACTUS_IEEEIO_ERROR (IOwriteAttribute (iof, "GH$time", FLEXIO_REAL,
                       1, &d_temp));
}


 /*@@
   @routine    IOFlexIO_DumpGH
   @date       Fri Aug 21 15:13:13 1998
   @author     Paul Walker
   @desc 
     The heart of checkpointing. Called by the different wrappers, this
     routines get the appropriate filename by IOUtil_PrepareFilename() and 
     then dumps away using the dump routines from IO ...
   @enddesc 
   @calls IO_DumpGF IO_DumpArray IO_DumpScalar
   @history 
   @hdate Oct 4 1998 @hauthor Gabrielle Allen
   @hdesc Removed code which checked and reset unchunked, assume that this 
          is done when the variable is first set.
   @hdate Nov 4 1998 @hauthor Gabrielle Allen
   @hdesc Do a forced synchronization before checkpointing
   @hdate Apr 16 1999 @hauthor Thomas Radke
   @hdesc Removed forced sync before checkpointing (just do a normal sync)
          Introduced a ring buffer for keeping last <checkpoint_keep> files
   @endhistory 

@@*/

void IOFlexIO_DumpGH (cGH *GH, int called_from)
{
  DECLARE_CCTK_PARAMETERS
  char dumpfname [1024], tmpfname [1024];
  IOFile iof;
  int index, wrotech;
  int timelevel, current_timelevel;
  int old_downsample_x, old_downsample_y, old_downsample_z;
  int old_out_single;
  ioGH *ioUtilGH;
  flexioGH *myGH;
  static char **dumpfnames = NULL;  /* dump filename ring buffer */
  static int findex = 0;            /* index into ring buffer */


  /* Get the handles for IOUtil and IOFlexIO extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
  myGH = (flexioGH *) GH->extensions [CCTK_GHExtensionHandle ("IOFlexIO")];

  /* allocate the ring buffer for filenames */
  if (! dumpfnames)
    dumpfnames = (char **) calloc (checkpoint_keep, sizeof (char *));

  /* disable downsampling after saving original downsampling params */
  old_downsample_x = ioUtilGH->downsample_x;
  old_downsample_y = ioUtilGH->downsample_y;
  old_downsample_z = ioUtilGH->downsample_z;
  ioUtilGH->downsample_x = ioUtilGH->downsample_y = ioUtilGH->downsample_z = 1;

  /* disable output in single precision */
  old_out_single = ioUtilGH->out_single;
  ioUtilGH->out_single = 0;

  /* start the total timer */
  if (print_timing_info)
    CCTK_TimerStartI (myGH->checkpointTotalTimer);

  /* sync all groups */
  for (index = 0; index < CCTK_NumGroups (); index++) {
    char *gname = CCTK_GroupName (index);

    CCTK_SyncGroup (GH, gname);
    free (gname);
  }

  /* get the base filename ... */
  IOUtil_PrepareFilename (GH, NULL, dumpfname, called_from,
                          CCTK_MyProc (GH) / ioUtilGH->ioproc_every,
                          ioUtilGH->unchunked);

  /* ... and append the extension */
  sprintf (tmpfname,  "%s.chkpt_tmp.ieee", dumpfname);
  sprintf (dumpfname, "%s.chkpt.ieee",     dumpfname);

  /* Now open the file */
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {
    if (verbose)
      printf ("Creating file %s\n", tmpfname);
    iof = IEEEopen (tmpfname, "w");
    if (! IOisValid (iof)) {
      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Can't open checkpoint file '%s'. Checkpointing is skipped",
                  tmpfname);
      return;
    }
  } else
    iof = (IOFile) NULL;

  /* Great; Now start dumping away! */

  /* start timer for dumping parameters */
  if (print_timing_info) {
    CCTK_TimerResetI (myGH->dumpParamsTimer);
    CCTK_TimerStartI (myGH->dumpParamsTimer);
  }

  if (verbose)
    printf ("Dumping Params ...\n -- ");

  /* first the parameters ... */
  if (iof) {
    if (out3D_parameters)
      IOFlexIO_DumpParams (GH, iof);
    IOFlexIO_DumpGHExtensions (GH, iof);
  }

  /* stop parameter timer and start timer for dumping datasets */
  if (print_timing_info) {
    CCTK_TimerStopI (myGH->dumpParamsTimer);
    CCTK_TimerResetI (myGH->dumpVarsTimer);
    CCTK_TimerStartI (myGH->dumpVarsTimer);
  }

  wrotech = 0;
 
  /* ... now the variables */
  for (index = 0; index < CCTK_NumVars (); index++) {

    /* let only variables pass which have storage assigned */
    if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index)))
      continue;

    if (verbose) {
      const char *varname = CCTK_VarName (index);

      printf (" %s", varname);
      wrotech += strlen (varname) + 1;
      if (wrotech > 65) {
        printf ("\n    ");
        wrotech = 0;
      }
    }

    /* get the current timelevel */
    current_timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
    if (current_timelevel > 0)
      current_timelevel--;

    /* now dump all timelevels up to the current */
    for (timelevel = 0; timelevel <= current_timelevel; timelevel++)
      IOFlexIO_DumpVar (GH, index, timelevel, iof);
  } /* end of loop over all variables */

  /* stop timer for dumping datasets */
  if (print_timing_info)
    CCTK_TimerStopI (myGH->dumpVarsTimer);

  CACTUS_IEEEIO_ERROR (IOclose (iof));
  
  if (CCTK_MyProc (GH) == ioUtilGH->ioproc) {
    if (rename (tmpfname, dumpfname))
      CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                  "Could not rename temporary checkpoint file %s to %s",
                  tmpfname, dumpfname);
  }

  /* delete the oldest dumpfile if checkpoint_keep_all isn't set
     and put the new filename into the ring buffer */
  if (dumpfnames [findex]) {
    if (! checkpoint_keep_all)
      remove (dumpfnames [findex]);
    free (dumpfnames [findex]);
  }
  dumpfnames [findex] = strdup (dumpfname);
  findex = (findex + 1) % checkpoint_keep;

  /* restore output precision flag */
  ioUtilGH->out_single = old_out_single;

  /* restore original downsampling params */
  ioUtilGH->downsample_x = old_downsample_x;
  ioUtilGH->downsample_y = old_downsample_y;
  ioUtilGH->downsample_z = old_downsample_z;

  /* stop total checkpoint timer and print timing info */
  if (print_timing_info) {
    cTimerData *info;

    CCTK_TimerStopI (myGH->checkpointTotalTimer);

    printf ("%s timing information for checkpointing:\n", CCTK_THORNSTRING);

/*** FIXME: select timer to get info from at runtime ***/
#ifdef HAVE_TIME_GETTIMEOFDAY
    info = CCTK_TimerCreateData();
    if (info) {

      if (myGH->dumpParamsTimer>= 0) {
        CCTK_TimerI (myGH->dumpParamsTimer, info);
        printf ("  Time to dump parameters:  %5.1f sec\n", info->vals[0].val.d);
      }

      if (myGH->dumpVarsTimer >= 0) {
        CCTK_TimerI (myGH->dumpVarsTimer, info);
        printf ("  Time to dump datasets:    %5.1f sec\n", info->vals[0].val.d);
      }

      if (myGH->checkpointTotalTimer >= 0) {
        CCTK_TimerI (myGH->checkpointTotalTimer, info);
        printf ("  Total time to checkpoint: %5.1f sec\n", info->vals[0].val.d);
      }

      CCTK_TimerDestroyData (info);
    } else
      CCTK_WARN (1, "Couldn't create timer info structure ! No timing output "
                    "available.");
#else
    CCTK_WARN (1, "No GETTIMEOFDAY timer available.");
#endif

    printf ("----------------------------------------------------------------"
            "-----------\n");
  }

  

}