/*@@
   @file      Write.c
   @date      Fri May 21 1999
   @author    Thomas Radke, Gerd Lanfermann
   @desc 
              File handling routines for IOHDF5.
   @enddesc 
   @version   $Id$
 @@*/


#include <stdlib.h>

#include "cctk.h"
#include "cctk_Parameters.h"
#include "StoreNamedData.h"
#include "CactusBase/IOUtil/src/ioGH.h"
#include "CactusPUGH/PUGH/src/include/pugh.h"
#include "ioHDF5GH.h"


/* the rcs ID and its dummy funtion to use it */
static char *rcsid = "$Id$";
CCTK_FILEVERSION(AlphaThorns_IOHDF5_Write_c)


/* prototypes of routines defined in this source file */
static char *IOHDF5_GetFilename (cGH *GH,
                                 int vindex,
                                 ioHDF5Geo_t *slab,
                                 const char *name,
                                 int *is_new_file);


/*@@
   @routine    IOHDF5_Write
   @date       Tue Oct 10 2000
   @author     Thomas Radke
   @desc
               Opens the HDF5 output file, calls the dump routine,
               and closes it again.
   @enddesc

   @calls      IOHDF5Util_DumpVar

   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH *
   @vio        in
   @endvar
   @var        vindex
   @vdesc      index of variable
   @vtype      int
   @vio        in
   @endvar
   @var        alias
   @vdesc      alias name of variable to output
   @vtype      const char *
   @vio        in
   @endvar
@@*/
void IOHDF5_Write (cGH *GH,
                   int vindex,
                   const char *alias)
{
  DECLARE_CCTK_PARAMETERS
  int timelevel;
  ioGH *ioUtilGH;
  ioHDF5GH *myGH;
  hid_t file, plist;
  int is_new_file;
  char *filename;
  char *fullname;


  /* Get the handle for IO extensions */
  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
  myGH     = (ioHDF5GH *) CCTK_GHExtension (GH, "IOHDF5");

  /* check if variable has storage assigned */
  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (vindex)))
  {
    fullname = CCTK_FullName (vindex);
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "No IOHDF5 output for '%s' (no storage)", fullname);
    free (fullname);
    return;
  }

  /* get the filename for output */
  filename = IOHDF5_GetFilename (GH, vindex, myGH->out_geo[vindex],
                                 alias, &is_new_file);
  if (! filename)
  {
    fullname = CCTK_FullName (vindex);
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Unable to construct file name for '%s'", fullname);
    free (fullname);
    return;
  }

  /* open the output file on all IO processors or - in case of parallel HDF5 -
     on every processor */
#ifdef HAVE_PARALLEL
  if (CCTK_MyProc (GH) != ioUtilGH->ioproc && ! ioUtilGH->unchunked)
#else
  if (CCTK_MyProc (GH) != ioUtilGH->ioproc)
#endif
  {
    file = -1;
  }
  else
  {
    if (verbose)
    {
      CCTK_VInfo (CCTK_THORNSTRING, "%s HDF5 output file '%s'",
                  is_new_file ? "Opening" : "Appending", filename);
    }

    IOHDF5_ERROR (plist = H5Pcreate (H5P_FILE_ACCESS));
#ifdef CCTK_MPI
#ifdef HAVE_PARALLEL
    if (ioUtilGH->unchunked)
    {
      IOHDF5_ERROR (H5Pset_mpi (plist, PUGH_pGH (GH)->PUGH_COMM_WORLD,
                                MPI_INFO_NULL));
    }
#endif
#endif

    if (is_new_file)
    {
      IOHDF5_ERROR (file = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT,
                                      plist));
    }
    else
    {
      IOHDF5_ERROR (file = H5Fopen (filename, H5F_ACC_RDWR, plist));
    }

    IOHDF5_ERROR (H5Pclose (plist));
  }

  /* get the current timelevel */
  timelevel = CCTK_NumTimeLevelsFromVarI (vindex) - 1;
  if (timelevel > 0)
  {
    timelevel--;
  }

  /* output global attributes */
  if (is_new_file && file >= 0)
  {
    /* all GH extension variables and parameter stuff which is not
       specific to any dataset goes into dedicated groups */
    if (out3D_parameters)
    {
      IOHDF5Util_DumpParameters (GH, file);
    }

    IOHDF5Util_DumpGHExtensions (GH, file);
  }

  /* output the data */
  IOHDF5Util_DumpVar (GH, vindex, timelevel, myGH->out_geo[vindex], file,
                      myGH->check_exisiting_objects[vindex]);

  /* close the file */
  if (file >= 0)
  {
    if (verbose)
    {
      CCTK_INFO ("Closing HDF5 output file from this iteration");
    }
    IOHDF5_ERROR (H5Fclose (file));
  }

  /* free the filename if it was not registered for re-opening
     in IOHDF5_net3D_filename () */
  if (out3D_septimefiles)
  {
    free (filename);
  }
}


/*@@
   @routine    IOHDF5_GetFilename
   @author     Paul Walker
   @date       Feb 1997
   @desc       Builds the filename for output.
               The names of all output files are stored in the filename
               database.
               The routine first searches in this database if there is already
               registered a filename for alias. If so, this name is returned.
               Otherwise it builds the filename from alias and stores it in the
               database.
               Generalized for Hyperslab extraction (GL)
   @enddesc
   @var        GH
   @vdesc      Pointer to CCTK GH
   @vtype      cGH *
   @vio        in
   @endvar
   @var        vindex
   @vdesc      index of variable to output
   @vtype      int
   @vio        in
   @endvar
   @var        slab
   @vdesc      pointer to hyperslab structure
   @vtype      ioHDF5Geo_t *
   @vio        in
   @endvar
   @var        varname
   @vdesc      alias name of variable to output
   @vtype      const char *
   @vio        in
   @endvar
   @var        is_new_file
   @vdesc      flag indicating if the file is to be created
   @vtype      int
   @vio        out
   @endvar

   @returntype char *
   @returndesc
               the allocated string containing the filename
   @endreturndesc
@@*/
static char *IOHDF5_GetFilename (cGH *GH,
                                 int vindex,
                                 ioHDF5Geo_t *slab,
                                 const char *varname,
                                 int *is_new_file)
{
  DECLARE_CCTK_PARAMETERS
  ioGH *ioUtilGH;          /* handle for IOUtil extensions */
  ioHDF5GH *myGH;          /* handle for IOHDF5 extensions */
  ioHDF5UtilGH *h5UtilGH;  /* handle for IOHDF5Util extensions */
  pGH *pughGH;             /* handle for PUGH extensions */
  int myproc;
  /* FIXME: make these strings dynamic */
  char extra[256];         /* Extra stuff in fname based on mode */
  char extradir[256];      /* Extra stuff for an output dir */
  char *filename;          /* the return value */
  int result;
  char *outputdir;
#if 0
  int idim;
  char *extrageo;          /* Extra stuff for geometry information */
#endif
  char name[128];


  /* get GH extensions for PUGH, IOUtil, IOHDF5Util, and IOHDF5 */
  pughGH   = PUGH_pGH (GH);
  ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO");
  myGH     = (ioHDF5GH *) CCTK_GHExtension (GH, "IOHDF5");
  h5UtilGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util");

#if 0
  /*** FIXME: use new filenaming scheme ***/
  /* To identify the slab in a volume  (if sdim<vdim)
     build a string like: "012" (3d volume, spanned by xyz)
     or "12" (2d volume, spanned by yz(!) and append this to the name:
     You get something like this: phi_s12_3d */
  if (slab->sdim<slab->vdim) {
    extrageo = (char*) malloc((slab->sdim+1)*sizeof(char));
    sprintf(extrageo,"%d",slab->direction[0]);
    for (idim=1;idim<slab->sdim;idim++)
      sprintf(extrageo,"%s%d",extrageo,slab->direction[idim]);
    sprintf(name,"%s_s%s_%dd",varname,extrageo,slab->vdim);
    free(extrageo);
  }
  /* If sdim==vdim, make it simple: e.g. phi_3d */
  else if (slab->sdim==slab->vdim)
  { 
    sprintf(name,"%s_%dd",varname,slab->vdim);
  }
  else {
    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Inconsitent dimensions for >%s<: slab (%dD), variable (%dD)\n",
                varname, slab->sdim,slab->vdim);
    return(NULL);
  }
#else
  strcpy (name, varname);
#endif

  filename = (char *) GetNamedData (myGH->open_output_files, name);
  if (filename != NULL)
  {
    /* set flag to indicate that file should be opened in append mode */
    *is_new_file = 0;
    return (filename);
  }

  extra[0]   = '\0';
  extradir[0]= '\0';

  myproc = CCTK_MyProc (GH);

  if (out3D_septimefiles)
  {
    char *tmp = extra;

    sprintf (extra, "%s.time_%7.3f", extra, GH->cctk_time);

    /* And be sure to replace any spaces in the filename with an _ */
    do
      if (*tmp == ' ')
        *tmp = '_';
    while (*++tmp);
  }

  
  /*  OUTPUT ONE FILE FOR EACH N PROCESSORS
   *  ------------------------------------- 
   *  
   *  If only one output file, the single file for each variable is written 
   *  in the normal output dir (that is extradir = ""). Otherwise
   *  a directory is created for each output variable to hold the multiple
   *  files.
   */

  if (! ioUtilGH->unchunked && ioUtilGH->ioproc_every < CCTK_nProcs (GH))
  {
    /* Add the output processor number to the extra string */
    sprintf (extra, "%s.file_%d", extra, myproc / ioUtilGH->ioproc_every);
    
    /* If necessary create the output directory */
    if (myproc == 0)
    {
      outputdir = (char *) malloc (strlen (myGH->outdir) +
                                         strlen (name) + 5);

      sprintf (outputdir, "%s/%s_3d", myGH->outdir, name);
      
      result = CCTK_CreateDirectory (0755, outputdir);
      if (result < 0)
      {
        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                    "Problem creating output directory '%s'", outputdir);
      }
      if (result > 0)
      {
        CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                    "Output directory '%s' already exists", outputdir);
      }

      free (outputdir);
    }
    
#ifdef CCTK_MPI
    /* Wait for all processors to catch up */
    CCTK_Barrier (GH);
#endif
    
    /* extradir is the relative output directory */
    sprintf (extradir, "%s_3d/", name);
    
  }
  
  /* CREATE THE COMPLETE OUTPUT FILENAME 
     ----------------------------------- */
  filename = (char *) malloc (strlen (myGH->outdir) +
                              strlen (extradir) +
                              strlen (name) +
                              strlen (extra) +
                              (pughGH->identity_string ? strlen (pughGH->identity_string) : 0) +
                              8);
  sprintf (filename, "%s/%s%s%s%s.h5", 
           myGH->outdir, extradir, name, extra, (pughGH->identity_string ? pughGH->identity_string : ""));

  /* no need to store filenames if used only once */
  if (! out3D_septimefiles)
  {
    if (myproc == ioUtilGH->ioproc)
    {
      if (ioUtilGH->recovered)
      {
        IOHDF5_ERROR (H5Eset_auto (NULL, NULL));
        *is_new_file = H5Fis_hdf5 (filename) <= 0;
        IOHDF5_ERROR (H5Eset_auto (h5UtilGH->print_error_fn,
                                   h5UtilGH->print_error_fn_arg));
        if (! *is_new_file)
        {
          myGH->check_exisiting_objects[vindex] = 1;
        }
      }
      else
      {
        *is_new_file = 1;
      }
    }
    StoreNamedData (&myGH->open_output_files, name, filename);
  }
  else
  {
    *is_new_file = 1;
  }

  return (filename);
}