path: root/src/Output3D.c

 /*@@
   @file      Output3D.c
   @date      01 Oct 1999
   @author    Jonghyun Lee
   @desc      Functions to deal 3D output of GFs
   @enddesc 
   @history
   @hendhistory
 @@*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusPUGH/PUGH/src/include/pugh.h"
#include "external/FlexIO/src/IOProtos.h"
#include "ioPandaGH.h"
#include "Panda/c_interface.h"


/* function prototypes */
int IOPanda_Output3DVarAs (cGH *GH, const char *var, const char *alias);
int IOPanda_TimeFor3D (cGH *GH, int index);
void IOPanda_getDumpData (cGH *GH, int index, int timelevel, void **outme,
                         int *free_outme, CCTK_INT4 *geom, int element_size);
void PandaTimestep(ArrayInfo *);
int PandaIsNewFile(char *);
void Panda_WriteAttribute(char *, char *, int, int, void *);

static int CheckOutputVar (int index);
static void CheckSteerableParameters (pandaGH *myGH);
static void SetOutputFlag (int index, const char *optstring, void *arg);
static void IOPanda_Timestep (cGH *GH, int index, const char *alias);
static void IOPanda_IEEEIOStructDump (cGH *GH, char *fname);
static void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel, int global_size[3], char *fname);
static void IOPanda_AddChunkAttributes (cGH *GH, int index, CCTK_INT4 *geom, char *fname);

/*@@
   @routine    IOPanda_Output3DGH
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Loops over all variables and outputs them if necessary
   @enddesc
   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH *
   @vio        in
   @endvar
@@*/
int IOPanda_Output3DGH (cGH *GH)
{
  DECLARE_CCTK_PARAMETERS
  int index;
  char *fullname;
  const char *name;
  pandaGH *myGH;


  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];

  CheckSteerableParameters (myGH);

  if (myGH->out3D_every <= 0)
  {
    return (0);
  }

  /* Loop over all variables */
  for (index = 0; index < CCTK_NumVars (); index++)
  {
    if (IOPanda_TimeFor3D (GH, index))
    {
      name     = CCTK_VarName (index);

      if (verbose)
      {
        fullname = CCTK_FullName (index);
        CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_Output3DGH: "
                    "(fullname, name) = (%s, %s)", fullname, name);
        free (fullname);
      }

      /* Do the 3D output */
      IOPanda_Timestep (GH, index, name);

      /* Register variable as having 3D output this iteration */
      myGH->out3D_last [index] = GH->cctk_iteration;
    }
  }

  return (0);
}


/*@@
   @routine    IOPanda_Output3DVarAs
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Unconditional output of a variable
               using the IOPanda 3D output method
   @enddesc
   @calledby   IOPanda_Output3DGH
   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH
   @vio        in
   @endvar
   @var        fullname
   @vdesc      complete name of variable to output
   @vtype      const char *
   @vio        in
   @endvar
   @var        alias
   @vdesc      alias name of variable to output
               (used to generate output filename)
   @vtype      const char *
   @vio        in
   @endvar
@@*/
int IOPanda_Output3DVarAs (cGH *GH,
                           const char *fullname,
                           const char *alias)
{
  DECLARE_CCTK_PARAMETERS
  int index;


  index = CCTK_VarIndex(fullname);

  if (verbose)
  {
    CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_Output3DVarAs: "
                "(fullname, alias, index) = (%s, %s, %d)",
                fullname, alias, index);
  }

  /* Do the 3D output */
  IOPanda_Timestep (GH, index, alias);

  return (0);
}


/*@@
   @routine    IOPanda_TimeFor3D
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Decides if it is time to output a variable
               using the IOPanda 3D output method
   @enddesc
   @calledby   IOPanda_Output3DGH
   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH
   @vio        in
   @endvar
   @var        index
   @vdesc      index of variable
   @vtype      int
   @vio        in
   @endvar
@@*/
int IOPanda_TimeFor3D (cGH *GH,
                       int index)
{
  pandaGH *myGH;


  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];

  CheckSteerableParameters (myGH);

  /* Check if any output was requested */
  if (myGH->out3D_every <= 0)
  {
    return (0);
  }

  /* Check this variable should be output */
  if (! (myGH->do_out3D [index] && GH->cctk_iteration % myGH->out3D_every == 0))
  {
    return (0);
  }

  /* Check variable not already output this iteration */
  if (myGH->out3D_last [index] == GH->cctk_iteration)
  {
    char *fullname = CCTK_FullName (index);


    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Already done IOPanda 3D output for variable '%s' in current "
                "iteration (probably via triggers)", fullname);
    free (fullname);
    return (0);
  }

  return (1);
}


/*@@
   @routine    IOPanda_TriggerOutput3D
   @date       Sat March 6 1999
   @author     Gabrielle Allen
   @desc
               Triggers the output a variable using the IOPanda 3D output method
   @enddesc
   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH
   @vio        in
   @endvar
   @var        index
   @vdesc      index of variable to output
   @vtype      int
   @vio        in
   @endvar
@@*/
int IOPanda_TriggerOutput3D (cGH *GH,
                             int index)
{
  DECLARE_CCTK_PARAMETERS
  pandaGH *myGH;
  const char *varname;


  varname = CCTK_VarName (index);

  /* Get the GH extension for IOPanda */
  myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];

  if (verbose)
  {
    CCTK_VInfo (CCTK_THORNSTRING, "TriggerOutput3D: "
                "name, index = %s, %d", varname, index);
  }

  /* Do the 3D output */
  IOPanda_Timestep (GH, index, varname);

  /* Register variable as having 3D output this iteration */
  myGH->out3D_last [index] = GH->cctk_iteration;

  return (0);
}


/**************************** local functions ******************************/
/* check if steerable parameters have changed */
static void CheckSteerableParameters (pandaGH *myGH)
{
  DECLARE_CCTK_PARAMETERS
  int times_set;
  static int out3D_vars_lastset = -1;


  /* How often to output */
  myGH->out3D_every = out_every > 0 ? out_every : -1;
  if (out3D_every > 0)
  {
    myGH->out3D_every = out3D_every;
  }

  /* re-parse the 'out3D_vars' parameter if it was changed */
  times_set = CCTK_ParameterQueryTimesSet ("out3D_vars", CCTK_THORNSTRING);
  if (times_set != out3D_vars_lastset)
  {
    memset (myGH->do_out3D, 0, CCTK_NumVars ());
    CCTK_TraverseString (out3D_vars, SetOutputFlag, myGH->do_out3D,
                         CCTK_GROUP_OR_VAR);

    /* Save the last setting of 'out3D_vars' parameter */
    out3D_vars_lastset = times_set;
  }

}


/* check if this variable can be output (static conditions) */
static int CheckOutputVar (int index)
{
  char *errormsg;
  char *fullname;


  errormsg = NULL;

/*** FIXME: IEEEIO doesn't provide a COMPLEX datatype
            so somehow CCTK_COMPLEX has to be mapped onto REALs or so.
            We have to check this already here because if an IEEEIO file
            is created and nothing is written to, it crashes at re-opening. ***/
  if (CCTK_VarTypeI (index) == CCTK_VARIABLE_COMPLEX)
  {
    errormsg = "IOPanda 3D output for complex variable '%s' not yet supported";
  }
  else if (CCTK_GroupTypeFromVarI (index) == CCTK_SCALAR)
  {
    errormsg = "IOPanda 3D output for scalar variable '%s' not supported";
  }

  if (errormsg)
  {
    fullname = CCTK_FullName (index);
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                errormsg, fullname);
    free (fullname);
  }

  return (errormsg != NULL);
}


/* callback for CCTK_TraverseString() to set the output flag
   for the given variable */
static void SetOutputFlag (int index,
                           const char *optstring,
                           void *arg)
{
  char *flags = (char *) arg;


  /* Check the variable type */
  if (CheckOutputVar (index) == 0)
  {
    flags[index] = 1;
  }

  if (optstring)
  {
    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Optional string '%s' in variable name ignored", optstring);
  }
}


static void IOPanda_Timestep (cGH *GH,
                              int index,
                              const char *alias)
{
  DECLARE_CCTK_PARAMETERS
  void *data;
  int tmp[1], tmp1[3], tmp2[3];
  Distribution dist1[3], dist2[3];
  CCTK_INT4 bnd[9];
  int free_flag, timelevel;
  ArrayInfo ainfo;
  ioGH *ioUtilGH;
  pGH *pughGH;


  /* check if variable has storage assigned */
  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index)))
  {
    char *fullname = CCTK_FullName (index);


    CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                "No IOPanda 3D output for '%s' (no storage)", fullname);
    free (fullname);
    return;
  }

  /* get the handles for PUGH and IOUtil GH extensions */
  pughGH = PUGH_pGH (GH);
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
  
  ainfo.name_ = (char *)alias;
  
  ainfo.rank_ = 3;
  tmp1[0] = GH->cctk_gsh[2]; 
  tmp1[1] = GH->cctk_gsh[1]; 
  tmp1[2] = GH->cctk_gsh[0]; 
  ainfo.size_ = tmp1;
  
  switch (CCTK_VarTypeI (index))
  {
    case CCTK_VARIABLE_CHAR:
      ainfo.esize_ = CHAR;
      break;

    case CCTK_VARIABLE_INT:
#ifdef  CCTK_INTEGER_PRECISION_8
      ainfo.esize_ = INT64;
#elif   CCTK_INTEGER_PRECISION_4
      ainfo.esize_ = INT32;
#elif   CCTK_INTEGER_PRECISION_2
      ainfo.esize_ = INT16;
#endif
      break;

    case CCTK_VARIABLE_REAL:
      if (ioUtilGH->out_single)
      {
        ainfo.esize_ = FLOAT32;
      }
      else
      {
#ifdef  CCTK_REAL_PRECISION_8
        ainfo.esize_ = FLOAT64;
#elif   CCTK_REAL_PRECISION_4
        ainfo.esize_ = FLOAT32;
#endif    
      }
  }
  
  /* FIXME: rank */
  ainfo.mem_rank_ = 3;
  tmp2[0] = pughGH->Connectivity [2]->nprocs [2];
  tmp2[1] = pughGH->Connectivity [2]->nprocs [1];
  tmp2[2] = pughGH->Connectivity [2]->nprocs [0];
  ainfo.mem_layout_ = tmp2;
  dist1[0] = dist1[1] = dist1[2] = BLOCK;
  ainfo.mem_dist_ = dist1;
  
  ainfo.disk_rank_ = 1;
  dist2[0] = BLOCK; dist2[1] = dist2[2] = NONE;
  tmp[0]= ((CCTK_nProcs(GH) - 1) / ioUtilGH->ioproc_every + 1);
  
  ainfo.disk_layout_ = tmp;
  ainfo.disk_dist_ = dist2;

  timelevel = 0;

  IOPanda_getDumpData(GH, index, timelevel, &data, &free_flag, bnd, ainfo.esize_);
  ainfo.data_ = (char *)data;

  /*** FIXME: asymmetric ghostzones */
  ainfo.stencil_width_ = GH->cctk_nghostzones [0];

  PandaTimestep(&ainfo);
  if (free_flag)
  {
    free (data);
  }

  IOPanda_AddCommonAttributes(GH, index, timelevel, ainfo.size_, ainfo.name_);

  if (! ioUtilGH->unchunked)
  {
    /* Write chunk attributes */
    IOPanda_AddChunkAttributes (GH, index, bnd, ainfo.name_);
  }

  if (PandaIsNewFile(ainfo.name_))
  {
    IOPanda_IEEEIOStructDump(GH, ainfo.name_);
  }
}


static void IOPanda_AddCommonAttributes (cGH *GH,
                                         int index,
                                         int timelevel,
                                         int global_size[3],
                                         char *fname)
{
  DECLARE_CCTK_PARAMETERS
  CCTK_REAL d3_to_IO [6]; /* buffer for writing doubles to IEEEIO */
  CCTK_INT4 i_to_IO;      /* buffer for writing an int to IEEEIO */
  CCTK_INT4 i3_to_IO[3];  /* buffer for writing ints to IEEEIO */
  CCTK_REAL dummy;
  char *name, *gname;
  ioGH *ioUtilGH;
  char *char_time_date = "";


#ifdef SGI
  time_t t = time(NULL);
  char_time_date = asctime (localtime (&t));
#endif

  /* Get the handle for IO extensions */
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];

  name = CCTK_FullName (index);
  Panda_WriteAttribute (fname, "name", BYTE, strlen (name) + 1, name);
  free (name);

  gname = CCTK_GroupNameFromVarI (index);
  Panda_WriteAttribute (fname, "groupname", BYTE, strlen (gname) + 1, gname);
  free (gname);

  i_to_IO = CCTK_GroupTypeFromVarI (index);
  Panda_WriteAttribute (fname, "grouptype", INT32, 1, &i_to_IO);

  i_to_IO = CCTK_NumTimeLevelsFromVarI (index);
  Panda_WriteAttribute (fname, "ntimelevels", INT32, 1, &i_to_IO);

  i_to_IO = timelevel;
  Panda_WriteAttribute (fname, "timelevel", INT32, 1, &i_to_IO);

  if (char_time_date && out3D_datestamp)
  {
    Panda_WriteAttribute (fname, "date", BYTE,
                         strlen (char_time_date) + 1, char_time_date);
  }

  Panda_WriteAttribute (fname, "time", FLOAT64, 1,&GH->cctk_time);

  d3_to_IO [0] = CCTK_CoordRange (GH, &d3_to_IO [0], &dummy, -1, "x", "cart3d");
  d3_to_IO [1] = CCTK_CoordRange (GH, &d3_to_IO [1], &dummy, -1, "y", "cart3d");
  d3_to_IO [2] = CCTK_CoordRange (GH, &d3_to_IO [2], &dummy, -1, "z", "cart3d");
  Panda_WriteAttribute (fname, "origin", FLOAT64,3,d3_to_IO);
  CCTK_CoordRange (GH, &d3_to_IO [0], &d3_to_IO [3], -1, "x", "cart3d");
  CCTK_CoordRange (GH, &d3_to_IO [1], &d3_to_IO [4], -1, "y", "cart3d");
  CCTK_CoordRange (GH, &d3_to_IO [2], &d3_to_IO [5], -1, "z", "cart3d");
  Panda_WriteAttribute (fname, "min_ext", FLOAT64, 3, d3_to_IO);
  Panda_WriteAttribute (fname, "max_ext", FLOAT64, 3, d3_to_IO+3);

  /* FIXME: dimension */
  d3_to_IO [0] = GH->cctk_delta_space [0] * ioUtilGH->downsample[0];
  d3_to_IO [1] = GH->cctk_delta_space [1] * ioUtilGH->downsample[1];
  d3_to_IO [2] = GH->cctk_delta_space [2] * ioUtilGH->downsample[2];
  Panda_WriteAttribute (fname, "delta", FLOAT64, 3,d3_to_IO);

  if (ioUtilGH->downsample[0] > 1 ||
      ioUtilGH->downsample[1] > 1 ||
      ioUtilGH->downsample[2] > 1)
  {
    d3_to_IO [0] = GH->cctk_delta_space [0];
    d3_to_IO [1] = GH->cctk_delta_space [1];
    d3_to_IO [2] = GH->cctk_delta_space [2];
    Panda_WriteAttribute (fname, "evolution_delta", FLOAT64, 3, d3_to_IO);
  }

  i3_to_IO[0] = global_size[0];
  i3_to_IO[1] = global_size[1];
  i3_to_IO[2] = global_size[2];
  Panda_WriteAttribute (fname, "global_size", INT32, 3, i3_to_IO);

  i_to_IO = CCTK_nProcs (GH);
  Panda_WriteAttribute (fname, "nprocs", INT32, 1, &i_to_IO);

  i_to_IO = ioUtilGH->ioproc_every;
  Panda_WriteAttribute (fname, "ioproc_every", INT32, 1, &i_to_IO);

  i_to_IO = ioUtilGH->unchunked;
  Panda_WriteAttribute (fname, "unchunked", INT32, 1, &i_to_IO);

  i_to_IO = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "iteration", INT32, 1, &i_to_IO);
}  


static void IOPanda_AddChunkAttributes (cGH *GH,
                                        int index,
                                        CCTK_INT4 *geom,
                                        char *fname)
{
  int dim;
  char *name;
  CCTK_INT4 i_to_IO;


  /* there is nothing to do for a serial run */
  if (CCTK_nProcs (GH) == 1)
    return;

  /* get the dimension of the variable */
  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (index));

  Panda_WriteAttribute (fname, "chunk_origin", INT32, dim, &geom[0]);
  Panda_WriteAttribute (fname, "subchunk_lb", INT32, dim, &geom[0]);
  Panda_WriteAttribute (fname, "global_size", INT32, dim, &geom[2*dim]);

  i_to_IO = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "chunk_dataset", INT32, 1, &i_to_IO);

  name = CCTK_FullName (index);
  Panda_WriteAttribute (fname, "name", CHAR, strlen (name)+1, name);
  free (name);
}


static void IOPanda_IEEEIOStructDump (cGH *GH,
                                      char *fname)
{
  CCTK_INT4 i_temp;
  CCTK_REAL d_temp;
  ioGH *ioUtilGH;

  
  ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];

  i_temp = GH->cctk_iteration;
  Panda_WriteAttribute (fname, "GH$iteration", INT32, 1, &i_temp);

  i_temp = ioUtilGH->ioproc_every;
  Panda_WriteAttribute (fname, "GH$ioproc_every", INT32, 1, &i_temp);

  i_temp = CCTK_nProcs (GH);
  Panda_WriteAttribute (fname, "GH$nprocs", INT32, 1, &i_temp);

  d_temp = GH->cctk_time;
  Panda_WriteAttribute (fname, "GH$time", FLOAT64, 1, &d_temp);
}