/*@@
   @file      Output.c
   @date      01 Oct 1999
   @author    Jonghyun Lee
   @desc
              Functions to deal with IOPanda output of grid variables
   @enddesc
   @version   $Id$
 @@*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusPUGH/PUGH/src/include/pugh.h"

#include "ioPandaGH.h"
#include "Panda/c_interface.h"

/* the rcs ID and its dummy funtion to use it */
static const char *rcsid = "$Header$";
CCTK_FILEVERSION(BetaThorns_IOPanda_Output_c)


/********************************************************************
 ********************    Internal Routines   ************************
 ********************************************************************/
static int CheckOutputVar (int vindex);
static void CheckSteerableParameters (pandaGH *myGH);
static void SetOutputFlag (int vindex, const char *optstring, void *arg);
static int IOPanda_Timestep (const cGH *GH, int vindex, const char *alias);
static void IOPanda_IEEEIOStructDump (const cGH *GH, const char *fname);
static void IOPanda_AddCommonAttributes (const cGH *GH, int vindex, int timelevel, int global_size[3], char *fname);
static void IOPanda_AddChunkAttributes (const cGH *GH, int vindex, CCTK_INT4 *geom, char *fname);


/*@@
   @routine    IOPanda_OutputGH
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Loops over all variables and outputs them if necessary
   @enddesc

   @calls      IOPanda_TimeFor
               IOPanda_Timestep

   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      const cGH *
   @vio        in
   @endvar

   @returntype int
   @returndesc
               the number of variables which were output at this iteration
               (or 0 if it wasn't time to output yet)
   @endreturndesc
@@*/
int IOPanda_OutputGH (const cGH *GH)
{
  int vindex, retval;
  pandaGH *myGH;
  char *fullname;
  const char *name;
  DECLARE_CCTK_PARAMETERS


  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) CCTK_GHExtension (GH, "IOPanda");

  CheckSteerableParameters (myGH);

  if (myGH->out_every <= 0)
  {
    return (0);
  }

  /* Loop over all variables */
  for (vindex = retval = 0; vindex < CCTK_NumVars (); vindex++)
  {
    if (IOPanda_TimeFor (GH, vindex))
    {
      name = CCTK_VarName (vindex);

      if (verbose)
      {
        fullname = CCTK_FullName (vindex);
        CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_OutputGH: "
                    "(fullname, name) = (%s, %s)", fullname, name);
        free (fullname);
      }

      /* Do the output */
      if (IOPanda_Timestep (GH, vindex, name) == 0)
      {
        /* Register variable as having output this iteration */
        myGH->out_last [vindex] = GH->cctk_iteration;
        retval++;
      }
    }
  }

  return (retval);
}


/*@@
   @routine    IOPanda_OutputVarAs
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Unconditional output of a variable using the IOPanda I/O method
   @enddesc

   @calls      IOPanda_Timestep

   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        fullname
   @vdesc      complete name of variable to output
   @vtype      const char *
   @vio        in
   @endvar
   @var        alias
   @vdesc      alias name of variable to output
               (used to generate output filename)
   @vtype      const char *
   @vio        in
   @endvar

   @returntype int
   @returndesc
               return code of @seeroutine IOPanda_Timestep
   @endreturndesc
@@*/
int IOPanda_OutputVarAs (const cGH *GH, const char *fullname, const char *alias)
{
  int vindex, retval;
  DECLARE_CCTK_PARAMETERS


  vindex = CCTK_VarIndex (fullname);
  if (verbose)
  {
    CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_OutputVarAs: "
                "(fullname, alias, index) = (%s, %s, %d)",
                fullname, alias, vindex);
  }

  /* Do the output */
  retval = IOPanda_Timestep (GH, vindex, alias);

  return (retval);
}


/*@@
   @routine    IOPanda_TimeFor
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Decides if it is time to output a variable
               using the IOPanda output method
   @enddesc

   @calls      CheckSteerableParameters

   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        vindex
   @vdesc      index of variable
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               1 if output should take place at this iteration, or<BR>
               0 if not
   @endreturndesc
@@*/
int IOPanda_TimeFor (const cGH *GH, int vindex)
{
  pandaGH *myGH;
  char *fullname;


  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) CCTK_GHExtension (GH, "IOPanda");

  CheckSteerableParameters (myGH);

  /* Check if any output was requested */
  if (myGH->out_every <= 0)
  {
    return (0);
  }

  /* Check this variable should be output */
  if (! (myGH->do_out[vindex] && GH->cctk_iteration % myGH->out_every == 0))
  {
    return (0);
  }

  /* Check variable not already output this iteration */
  if (myGH->out_last [vindex] == GH->cctk_iteration)
  {
    fullname = CCTK_FullName (vindex);
    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Already done IOPanda output for variable '%s' in current "
                "iteration (probably via triggers)", fullname);
    free (fullname);
    return (0);
  }

  return (1);
}


/*@@
   @routine    IOPanda_TriggerOutput
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Triggers the output a variable using the IOPanda output method
   @enddesc

   @calls      IOPanda_Timestep

   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      const cGH *
   @vio        in
   @endvar
   @var        vindex
   @vdesc      index of variable to output
   @vtype      int
   @vio        in
   @endvar

   @returntype int
   @returndesc
               return code of @seeroutine IOPanda_Timestep
   @endreturndesc
@@*/
int IOPanda_TriggerOutput (const cGH *GH, int vindex)
{
  int retval;
  pandaGH *myGH;
  const char *varname;
  DECLARE_CCTK_PARAMETERS


  varname = CCTK_VarName (vindex);

  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) CCTK_GHExtension (GH, "IOPanda");

  if (verbose)
  {
    CCTK_VInfo (CCTK_THORNSTRING, "TriggerOutput: "
                "name, index = %s, %d", varname, vindex);
  }

  /* Do the output */
  retval = IOPanda_Timestep (GH, vindex, varname);

  if (retval == 0)
  {
    /* Register variable as having output this iteration */
    myGH->out_last [vindex] = GH->cctk_iteration;
  }

  return (retval);
}


/********************************************************************
 ********************    Internal Routines   ************************
 ********************************************************************/
/*@@
   @routine    CheckSteerableParameters
   @date       Mon Oct 10 2000
   @author     Thomas Radke
   @desc
               Checks if IOPanda steerable parameters were changed
               and does appropriate re-evaluation.
   @enddesc

   @calls      CCTK_TraverseString

   @var        myGH
   @vdesc      Pointer to IOPanda GH
   @vtype      pandaGH *
   @vio        in
   @endvar
@@*/
static void CheckSteerableParameters (pandaGH *myGH)
{
  int times_set;
  static int out_vars_lastset = -1;
  DECLARE_CCTK_PARAMETERS


  /* how often to output */
  myGH->out_every = out_every;
  if (myGH->out_every < 0)
  {
    myGH->out_every = *(const CCTK_INT *)
                      CCTK_ParameterGet ("out_every",
                                         CCTK_ImplementationThorn ("IO"), NULL);
  }

  /* re-parse the 'out_vars' parameter if it was changed */
  times_set = CCTK_ParameterQueryTimesSet ("out_vars", CCTK_THORNSTRING);
  if (times_set != out_vars_lastset)
  {
    memset (myGH->do_out, 0, CCTK_NumVars ());
    CCTK_TraverseString (out_vars, SetOutputFlag, myGH->do_out,
                         CCTK_GROUP_OR_VAR);

    /* Save the last setting of 'out_vars' parameter */
    out_vars_lastset = times_set;
  }
}


/* check if this variable can be output (static conditions) */
static int CheckOutputVar (int vindex)
{
  char *errormsg, *fullname;


  errormsg = NULL;

/*** FIXME: IEEEIO doesn't provide a COMPLEX datatype
            so somehow CCTK_COMPLEX has to be mapped onto REALs or so.
            We have to check this already here because if an IEEEIO file
            is created and nothing is written to, it crashes at re-opening. ***/
  if (CCTK_VarTypeI (vindex) == CCTK_VARIABLE_COMPLEX)
  {
    errormsg = "IOPanda output for complex variable '%s' not yet supported";
  }
  else if (CCTK_GroupTypeFromVarI (vindex) == CCTK_SCALAR)
  {
    errormsg = "IOPanda output for scalar variable '%s' not supported";
  }

  if (errormsg)
  {
    fullname = CCTK_FullName (vindex);
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                errormsg, fullname);
    free (fullname);
  }

  return (errormsg != NULL);
}


/* callback for CCTK_TraverseString() to set the output flag
   for the given variable */
static void SetOutputFlag (int vindex, const char *optstring, void *arg)
{
  char *flags = (char *) arg;


  /* Check the variable type */
  flags[vindex] = CheckOutputVar (vindex) == 0;

  if (optstring)
  {
    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Optional string '%s' in variable name ignored", optstring);
  }
}


static int IOPanda_Timestep (const cGH *GH, int vindex, const char *alias)
{
  void *data;
  int tmp[1], tmp1[3], tmp2[3];
  Distribution dist1[3], dist2[3];
  CCTK_INT4 bnd[9];
  int free_flag, timelevel;
  ArrayInfo ainfo;
  ioGH *ioUtilGH;
  pGH *pughGH;
  char *fullname;
  union
  {
    char *non_const_char;
    const char *const_char;
  } cast_to_const;
  DECLARE_CCTK_PARAMETERS


  /* check if variable has storage assigned */
  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (vindex)))
  {
    fullname = CCTK_FullName (vindex);
    CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                "No IOPanda output for '%s' (no storage)", fullname);
    free (fullname);
    return (-1);
  }

  /* get the handles for PUGH and IOUtil GH extensions */
  pughGH = PUGH_pGH (GH);
  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");

  /* prevent compiler warning "cast discards `const' from pointer target type"*/
  cast_to_const.const_char = alias;
  ainfo.name_ = cast_to_const.non_const_char;

  ainfo.rank_ = 3;
  tmp1[0] = GH->cctk_gsh[2];
  tmp1[1] = GH->cctk_gsh[1];
  tmp1[2] = GH->cctk_gsh[0];
  ainfo.size_ = tmp1;

  switch (CCTK_VarTypeI (vindex))
  {
    case CCTK_VARIABLE_CHAR:
      ainfo.esize_ = CHAR;
      break;

    case CCTK_VARIABLE_INT:
#ifdef  CCTK_INTEGER_PRECISION_8
      ainfo.esize_ = INT64;
#elif   CCTK_INTEGER_PRECISION_4
      ainfo.esize_ = INT32;
#elif   CCTK_INTEGER_PRECISION_2
      ainfo.esize_ = INT16;
#endif
      break;

    case CCTK_VARIABLE_REAL:
      if (ioUtilGH->out_single)
      {
        ainfo.esize_ = FLOAT32;
      }
      else
      {
#ifdef  CCTK_REAL_PRECISION_8
        ainfo.esize_ = FLOAT64;
#elif   CCTK_REAL_PRECISION_4
        ainfo.esize_ = FLOAT32;
#endif
      }
  }

  /* FIXME: rank */
  ainfo.mem_rank_ = 3;
  tmp2[0] = pughGH->Connectivity [2]->nprocs [2];
  tmp2[1] = pughGH->Connectivity [2]->nprocs [1];
  tmp2[2] = pughGH->Connectivity [2]->nprocs [0];
  ainfo.mem_layout_ = tmp2;
  dist1[0] = dist1[1] = dist1[2] = BLOCK;
  ainfo.mem_dist_ = dist1;

  ainfo.disk_rank_ = 1;
  dist2[0] = BLOCK; dist2[1] = dist2[2] = NONE;
  tmp[0]= ((CCTK_nProcs(GH) - 1) / ioUtilGH->ioproc_every + 1);

  ainfo.disk_layout_ = tmp;
  ainfo.disk_dist_ = dist2;

  timelevel = 0;

  IOPanda_getDumpData(GH, vindex, timelevel, &data, &free_flag, bnd, ainfo.esize_);
  ainfo.data_ = (char *)data;

  /*** FIXME: asymmetric ghostzones */
  ainfo.stencil_width_ = GH->cctk_nghostzones [0];

  PandaTimestep(&ainfo);
  if (free_flag)
  {
    free (data);
  }

  IOPanda_AddCommonAttributes(GH, vindex, timelevel, ainfo.size_, ainfo.name_);

  if (! ioUtilGH->unchunked)
  {
    /* Write chunk attributes */
    IOPanda_AddChunkAttributes (GH, vindex, bnd, ainfo.name_);
  }

  if (PandaIsNewFile(ainfo.name_))
  {
    IOPanda_IEEEIOStructDump(GH, ainfo.name_);
  }

  return (0);
}


static void IOPanda_AddCommonAttributes (const cGH *GH,
                                         int vindex,
                                         int timelevel,
                                         int global_size[3],
                                         char *fname)
{
  DECLARE_CCTK_PARAMETERS
  CCTK_REAL d3_to_IO [6]; /* buffer for writing doubles to IEEEIO */
  CCTK_INT4 i_to_IO;      /* buffer for writing an int to IEEEIO */
  CCTK_INT4 i3_to_IO[3];  /* buffer for writing ints to IEEEIO */
  CCTK_REAL dummy;
  char *name, *gname;
  ioGH *ioUtilGH;


  /* Get the handle for IO extensions */
  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");

  name = CCTK_FullName (vindex);
  Panda_WriteAttribute (fname, "name", BYTE, strlen (name) + 1, name);
  free (name);

  gname = CCTK_GroupNameFromVarI (vindex);
  Panda_WriteAttribute (fname, "groupname", BYTE, strlen (gname) + 1, gname);
  free (gname);

  i_to_IO = CCTK_GroupTypeFromVarI (vindex);
  Panda_WriteAttribute (fname, "grouptype", INT32, 1, &i_to_IO);

  i_to_IO = CCTK_NumTimeLevelsFromVarI (vindex);
  Panda_WriteAttribute (fname, "ntimelevels", INT32, 1, &i_to_IO);

  i_to_IO = timelevel;
  Panda_WriteAttribute (fname, "timelevel", INT32, 1, &i_to_IO);

  Panda_WriteAttribute (fname, "time", FLOAT64, 1,&GH->cctk_time);

  d3_to_IO [0] = CCTK_CoordRange (GH, &d3_to_IO [0], &dummy, -1, "x", "cart3d");
  d3_to_IO [1] = CCTK_CoordRange (GH, &d3_to_IO [1], &dummy, -1, "y", "cart3d");
  d3_to_IO [2] = CCTK_CoordRange (GH, &d3_to_IO [2], &dummy, -1, "z", "cart3d");
  Panda_WriteAttribute (fname, "origin", FLOAT64,3,d3_to_IO);
  CCTK_CoordRange (GH, &d3_to_IO [0], &d3_to_IO [3], -1, "x", "cart3d");
  CCTK_CoordRange (GH, &d3_to_IO [1], &d3_to_IO [4], -1, "y", "cart3d");
  CCTK_CoordRange (GH, &d3_to_IO [2], &d3_to_IO [5], -1, "z", "cart3d");
  Panda_WriteAttribute (fname, "min_ext", FLOAT64, 3, d3_to_IO);
  Panda_WriteAttribute (fname, "max_ext", FLOAT64, 3, d3_to_IO+3);

  /* FIXME: dimension */
  d3_to_IO [0] = GH->cctk_delta_space [0] * ioUtilGH->downsample[0];
  d3_to_IO [1] = GH->cctk_delta_space [1] * ioUtilGH->downsample[1];
  d3_to_IO [2] = GH->cctk_delta_space [2] * ioUtilGH->downsample[2];
  Panda_WriteAttribute (fname, "delta", FLOAT64, 3, d3_to_IO);

  if (ioUtilGH->downsample[0] > 1 ||
      ioUtilGH->downsample[1] > 1 ||
      ioUtilGH->downsample[2] > 1)
  {
    d3_to_IO [0] = GH->cctk_delta_space [0];
    d3_to_IO [1] = GH->cctk_delta_space [1];
    d3_to_IO [2] = GH->cctk_delta_space [2];
    Panda_WriteAttribute (fname, "evolution_delta", FLOAT64, 3, d3_to_IO);
  }

  i3_to_IO[0] = global_size[0];
  i3_to_IO[1] = global_size[1];
  i3_to_IO[2] = global_size[2];
  Panda_WriteAttribute (fname, "global_size", INT32, 3, i3_to_IO);

  i_to_IO = CCTK_nProcs (GH);
  Panda_WriteAttribute (fname, "nprocs", INT32, 1, &i_to_IO);

  i_to_IO = ioUtilGH->ioproc_every;
  Panda_WriteAttribute (fname, "ioproc_every", INT32, 1, &i_to_IO);

  i_to_IO = ioUtilGH->unchunked;
  Panda_WriteAttribute (fname, "unchunked", INT32, 1, &i_to_IO);

  i_to_IO = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "iteration", INT32, 1, &i_to_IO);
}


static void IOPanda_AddChunkAttributes (const cGH *GH,
                                        int vindex,
                                        CCTK_INT4 *geom,
                                        char *fname)
{
  int dim;
  char *name;
  CCTK_INT4 i_to_IO;


  /* there is nothing to do for a serial run */
  if (CCTK_nProcs (GH) == 1)
  {
    return;
  }

  /* get the dimension of the variable */
  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));

  Panda_WriteAttribute (fname, "chunk_origin", INT32, dim, &geom[0]);
  Panda_WriteAttribute (fname, "subchunk_lb", INT32, dim, &geom[0]);
  Panda_WriteAttribute (fname, "global_size", INT32, dim, &geom[2*dim]);

  i_to_IO = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "chunk_dataset", INT32, 1, &i_to_IO);

  name = CCTK_FullName (vindex);
  Panda_WriteAttribute (fname, "name", CHAR, strlen (name)+1, name);
  free (name);
}


static void IOPanda_IEEEIOStructDump (const cGH *GH,
                                      const char *fname)
{
  DECLARE_CCTK_PARAMETERS
  int len;
  CCTK_INT4 i_temp;
  CCTK_REAL d_temp;
  ioGH *ioUtilGH;
  char buffer[128];


  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");

  i_temp = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "GH$iteration", INT32, 1, &i_temp);

  i_temp = ioUtilGH->ioproc_every;
  Panda_WriteAttribute (fname, "GH$ioproc_every", INT32, 1, &i_temp);

  i_temp = CCTK_nProcs (GH);
  Panda_WriteAttribute (fname, "GH$nprocs", INT32, 1, &i_temp);

  d_temp = GH->cctk_time;
  Panda_WriteAttribute (fname, "GH$time", FLOAT64, 1, &d_temp);

  /* add the parameter filename and the creation date
     as file identification attributes */
  if (CCTK_Equals (out_fileinfo, "parameter filename") ||
      CCTK_Equals (out_fileinfo, "all"))
  {
    buffer[0] = 0;
    CCTK_ParameterFilename (sizeof (buffer), buffer);
    Panda_WriteAttribute (fname, "parameter file", CHAR,
                          strlen (buffer) + 1, buffer);
  }
  if (CCTK_Equals (out_fileinfo, "creation date") ||
      CCTK_Equals (out_fileinfo, "all"))
  {
    buffer[0] = 0;
    Util_CurrentDate (sizeof (buffer), buffer);
    len = strlen (buffer) + 1;
    buffer[len-1] = ' ';
    Util_CurrentTime (sizeof (buffer) - len, buffer + len);
    Panda_WriteAttribute (fname, "creation date", CHAR,
                          strlen (buffer) + 1, buffer);
  }
}