The hopefully final renaming of the FlexIO thorn.

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOFlexIO/trunk@2 ebee0441-1374-4afa-a3b5-247f3ba15b9a

1 files changed, 462 insertions, 0 deletions

diff --git a/src/Write3D.c b/src/Write3D.c
new file mode 100644
index 0000000..6d72a93
--- /dev/null
+++ b/src/Write3D.c
@@ -0,0 +1,462 @@
+/*@@
+   @file      Write3D.c
+   @date      Fri Feb 21 15:27:03 1997
+   @author    
+   @desc 
+   A horrible blocking write and not-quite-so-horrible
+   slicer for xgraph output. Basically MPI based IO
+   bites the big one, and this is MPI based IO written
+   quickly and in the obvious inefficient fashion.
+   <p>
+   Oh, not it also contains zero-d I/O and a closer also.
+   <p>
+   This routine can write DFSD style HDF (not the default)
+   or Johns IEEEIO 3D files (the default). For 0 and 1D
+   I/O it writes xgraph files.
+   @enddesc 
+   @history
+   @hauthor Gabrielle Allen @hdate 26 Sep 1998
+   @hdesc Changed routine name (WriteGF->Write3D), Lots of tidying up and 
+          renaming of parameters
+   @hauthor Gabrielle Allen @hdate 17 Oct 1998
+   @hdesc Implemented new parameters IO_verbose,IO_parameter,IO_structures,IO_scalars
+   @hauthor Gabrielle Allen @hdate 19 Oct 1998      
+   @hdesc Changed names ready for thorn_IO
+   @hauthor Thomas Radke @hdate 16 Mar 1999
+   @hdesc Converted to Cactus 4.0
+   @hauthor Thomas Radke @hdate 17 Mar 1999
+   @hdesc Changed naming: IEEEIO -> IOFlexIO
+   @hendhistory
+   @version $Id$
+ @@*/
+
+static char *rcsid = "$Id$";
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "cctk.h"
+#include "flesh.h"
+#include "Comm.h"
+#include "Groups.h"
+#include "GHExtensions.h"
+#include "WarnLevel.h"
+#include "declare_parameters.h"
+#ifdef CACTUSBASE_PUGH
+#include "CactusBase/pugh/src/include/pugh.h"
+#endif
+#include "CactusBase/IOUtil/src/ioGH.h"
+#include "ioFlexGH.h"
+
+
+/* local function prototypes */
+void IOFlexIO_Write3D_filename (cGH *GH, char *filename, const char *name);
+void IOFlexIO_Write3D_openFile (cGH *GH, const char *filename, IOFile *iofile, int first);
+void IOFlexIO_Write3D_closeFile (cGH *GH, const char *filename, IOFile *iofile);
+/* external function prototypes */
+void IOFlexIO_DumpVar (cGH *GH, int index, int timelevel, IOFile iof);
+
+
+/*@@
+  @routine IOFlexIO_Write3D
+  @author  Paul Walker 
+  @date    Feb 1997
+  @desc
+  This routine does 3D IO for a grid function which has been
+  distributed with MPI.
+  <p>
+  3D MPI IO! Wow, you may say, that sounds pretty clever. Well it isn't
+  because <b><i>The method I use sucks rocks</i></b>. That is.
+  I spam the data from all processors onto processor 0. Luckily
+  it doesn't just put it all in one memory chunk, but recieves it
+  bit by bit and writes it out chunkwise, using IEEEIO. Alas, 
+  while this happens the other processors <i>sit in a barrier 
+  with their metaphoric thumbs up their asses</i>. 
+  <p>
+  <b>Even Worse</b> it is a blocking send which must come in order
+  from the other processors.
+  <p> 
+  This is inefficient to say the least. I/O costs can go up to 70-80%
+  using this technique on the T3E. So there are also a couple of ways to
+  get around it.
+  <p>
+  The easiest option is one-file-per-processor which is
+  good on some machines, (eg T3E) and bad on others (eg O2K, I think).
+  So you can activate this mode (or de-activate it on the T3E)
+  by setting the parameter onefileperproc to "yes" or "no".
+  This will create a bunch of files in their own subdirectory,
+  and they can be smooshed together using cactus/lib/etc/Recombiner.C
+  <p>
+  Also, you can downsample output. You can also output one file per
+  slice. Various combinations of these work or don't, depending.
+  <p>
+  Finally notice that @seeroutine main guarantees that this is never
+  called on a 1- or 2-D grid.
+  @enddesc
+  @calls    CCTK_GetGHExtensionHandle
+            CCTK_GetMyProc
+            CCTK_GetnProcs
+            CCTK_WARN
+  @calledby IOFlexIO_Output3DVarAs
+            IOFlexIO_TriggerOutput3D
+  @history 
+  @hdate Wed Sep  2 10:11:41 1998 @hauthor Tom Goodale
+  @hdesc Merged in Szu-Wen's Panda calls.
+  @hdate Sep 25 1998 1998 @hauthor Gabrielle Allen
+  @hdesc Split into subroutines
+  @endhistory 
+  @var     GH
+  @vdesc   Pointer to CCTK GH
+  @vtype   cGH
+  @vio     in
+  @vcomment
+  @endvar
+  @var     index
+  @vdesc   index of variable to output
+  @vtype   int
+  @vio     in
+  @vcomment
+  @endvar
+  @var     alias
+  @vdesc   alias name of variable to output
+  @vtype   const char *
+  @vio     in
+  @vcomment
+  @endvar
+  @var     first
+  @vdesc   flag to indicate whether file should be opened in "w" or "a" mode
+  @vtype   int (boolean)
+  @vio     in
+  @vcomment
+  @endvar
+  @@*/
+
+
+void IOFlexIO_Write3D (cGH *GH, int index, const char *alias, int first)
+{
+  DECLARE_PARAMETERS
+  int myproc, nprocs;
+  int timelevel;
+  ioGH *ioUtilGH;
+  flexioGH *myGH;
+  IOFile *iofile;
+  char *filename;
+
+
+  /* Get the handle for IOUtil and IOFlexIO extensions */
+  ioUtilGH = (ioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IO")];
+  myGH = (flexioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IOFlexIO")];
+
+  /* Get filename and reference to file handle */
+  filename = myGH->IEEEfname_3D [index];
+  iofile = &myGH->IEEEfile_3D [index];
+
+  if (IO_verbose) {
+    printf ("-------------------------------------------------------\n");
+    printf ("3D I/O on Grid Function %s\n", alias);
+    printf ("  downsample (x,y,z): (%d,%d,%d)  ioproc : %d\n", 
+            ioUtilGH->downsample_x, ioUtilGH->downsample_y,
+            ioUtilGH->downsample_z, ioUtilGH->ioproc);
+    fflush (stdout);
+  }
+
+  /* What processor are we on? */
+  myproc = CCTK_GetMyProc (GH);
+  nprocs = CCTK_GetnProcs (GH);
+
+
+  /* build the filename for output */
+  /* This code is semi-repeated in DumpGH.c */
+  /* The output directory is also created. */
+  if (first || onefileperslice)
+    IOFlexIO_Write3D_filename (GH, filename, alias);
+
+  /* Close the file if it is already open (if IEEEfile_3D is set). 
+   *  This shouldn't be needed, since we do it below, 
+   *  but better safe than sorry.
+   *  Also I have some vague recollection of needing it in some
+   *  very strange case ... So leave it here for now (PW 11.5.98)
+   */
+  if (*iofile && onefileperslice) {
+    if (IO_verbose)
+      printf ("Closing IEEEfile from previous iteration.\n");
+    CACTUS_IEEEIO_ERROR (IOclose (*iofile));
+    *iofile = NULL; /* prevent attempts to close it twice! */
+  }
+
+
+  /* open the output file (only if we are an output processor) */
+  if (myproc == ioUtilGH->ioproc)
+    IOFlexIO_Write3D_openFile (GH, filename, iofile, first);
+  
+  /* get the current timelevel */
+  timelevel = CCTK_GetNumTimeLevels (index) - 1;
+  if (timelevel > 0)
+    timelevel--;
+
+  /* output the data */
+  IOFlexIO_DumpVar (GH, index, timelevel, *iofile);
+
+  /* output the scalars and parameters */
+  /* Don't do this if the output is being byte compared, as the
+   * thorn lists might be different.
+   */
+  if (myproc == ioUtilGH->ioproc) {
+    /* output parameters necessary for filereader datafiles */
+    if (first)
+      IOFlexIO_IEEEIOStructDump (GH, *iofile);
+#if        0
+    if (ioUtilGH->parameters)
+      IOFlexIO_IEEEIOparamDump (*iofile);
+#endif
+    /* close the file */
+    IOFlexIO_Write3D_closeFile (GH, filename, iofile);
+  }
+
+  if (IO_verbose)
+    printf ("-------------------\n");
+}
+
+
+/*@@
+  @routine IOFlexIO_Write3D_closeFile
+  @author  Paul Walker
+  @date    Feb 1997
+  @desc
+  Closes or pauses the IEEEIO output file.
+  @enddesc
+  @history 
+  @hauthor Gabrielle Allen
+  @hdate Sep 1998
+  @hdesc Split into subroutine
+  @endhistory 
+  @@*/
+
+
+void IOFlexIO_Write3D_closeFile (cGH *GH, const char *filename, IOFile *iofile)
+{
+  DECLARE_PARAMETERS
+  ioGH *ioUtilGH;
+  flexioGH *myGH;
+
+  ioUtilGH = (ioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IO")];
+  myGH = (flexioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IOFlexIO")];
+
+  if (*iofile == NULL)
+    return;
+
+  if (onefileperslice) {    
+    if (IO_verbose)
+      printf ("Closing IEEEfile from this iteration\n");
+    CACTUS_IEEEIO_ERROR (IOclose (*iofile));
+    *iofile = NULL;
+    return;
+  }
+  
+  if (myGH->reuse_fh) {
+    if (IO_verbose)
+      printf ("Pausing file %s\n", filename);
+    IEEEbufferOff (*iofile); 
+    CACTUS_IEEEIO_ERROR (IOpause (*iofile));
+  } else {
+    if (IO_verbose)
+      printf ("Closing file %s\n", filename);
+    CACTUS_IEEEIO_ERROR (IOclose (*iofile));
+    *iofile = NULL;
+  }
+}
+
+
+/*@@
+  @routine IOFlexIO_Write3D_openFile
+  @author  Paul Walker
+  @date    Feb 1997
+  @desc
+  Opens or resumes the IEEEIO output file
+  @enddesc
+  @history 
+  @hauthor Gabrielle Allen
+  @hdate Sep 1998
+  @hdesc Split into subroutine
+  @endhistory 
+  @@*/
+
+void IOFlexIO_Write3D_openFile (cGH *GH, const char *filename, IOFile *iofile, int first)
+{
+  DECLARE_PARAMETERS
+  ioGH *ioUtilGH;
+  flexioGH *myGH;
+
+  ioUtilGH = (ioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IO")];
+  myGH = (flexioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IOFlexIO")];
+
+  /* If we are the first time through, or writing one file per slice, we need
+     to open a file in write mode. Or, if we are reusing file handles we will
+     need to resume the file */
+  
+  if (first || onefileperslice) {
+    
+    /* First time through or one file per slice; open anew */
+    if (IO_verbose)
+      printf ("Opening file %s\n", filename);
+    
+    *iofile = IEEEopen (filename, "w");
+    if (! IOisValid (*iofile)) {
+      char *msg = (char *) malloc (strlen (filename) + 80);
+
+      sprintf (msg, "The file %s could not be opened for writing.", filename);
+      CCTK_WARN (1, msg);
+      free (msg);
+    }
+  } 
+  
+  else if (myGH->reuse_fh) {
+    
+    /* resume the file (reopen descriptor)
+       This allows descriptor reuse without requiring expensive destruction
+       and reallocation of the associated datastructures */
+    
+    if (*iofile) {
+      
+      if (IO_verbose)
+        printf ("Resuming file %s\n", filename);
+
+      CACTUS_IEEEIO_ERROR (IOresume (*iofile));
+    }
+  } else {
+    /* File is closed, not onefileperslice and not first time.
+       Therefore we must be conserving file handles and failed
+       to use the pause()/resume methods(), So append.
+    */
+    if (IO_verbose)
+      printf ("Re-opening file %s in append mode\n", filename);
+    
+    *iofile = IEEEopen (filename, "a");
+    if (! IOisValid (*iofile)) {
+      char *msg = (char *) malloc (strlen (filename) + 80);
+
+      sprintf (msg, "The file %s could not be opened for appending.\n",
+               filename);
+      CCTK_WARN (1, msg);
+      free (msg);
+    }
+  }
+  
+  /* Turn on buffer cache (with default size chosen by IEEEIO lib) */
+  IEEEbufferOn (*iofile, 0);
+}
+
+
+/*@@
+  @routine IOFlexIO_Write3D_filename
+  @author  Paul Walker
+  @date    Feb 1997
+  @desc
+  Generates the filename for output
+  @enddesc
+  @history 
+  @hauthor Gabrielle Allen
+  @hdate Sep 1998
+  @hdesc Split into subroutine
+  @endhistory 
+  @@*/
+
+void IOFlexIO_Write3D_filename (cGH *GH, char *filename, const char *name)
+{
+  DECLARE_PARAMETERS
+  ioGH *ioUtilGH;       /* handle for IOUtil extensions */
+  pGH *pughGH;          /* handle for PUGH extensions */
+  int nprocs;
+  int myproc;
+  char extra [256];        /* Extra stuff in fname based on mode */
+  char extradir [256];        /* Extra stuff for an output dir */
+  char createdir [256];        /* Text for system call to create output directory */
+
+  ioUtilGH = (ioGH *) GH->extensions [CCTK_GetGHExtensionHandle ("IO")];
+  pughGH = (pGH *) GH->extensions [CCTK_GetGHExtensionHandle ("PUGH")];
+
+  extra [0] = '\0';
+  extradir [0] = '\0';
+  
+  nprocs = CCTK_GetnProcs (GH);
+  myproc = CCTK_GetMyProc (GH);
+
+#if        0
+  /* Say if we have a chunked data in the output file(s) */
+  if (! ioUtilGH->unchunked && nprocs > 1)
+    sprintf (extra, "%s_chunked", extra);
+#endif
+  
+  if (onefileperslice)
+    sprintf (extra, "%s.time_%7.3f", extra, GH->time);
+  
+ 
+  /* OUTPUT ONE FILE FOR EACH N PROCESSORS 
+   *  ------------------------------------- 
+   *  
+   *  If only one output file, the single file for each GF is written 
+   *  in the normal output dir (that is extradir = ""). Otherwise
+   *  a directory is created for each output GF to hold the multiple
+   *  file 
+   */
+
+  if (ioUtilGH->ioproc_every < nprocs) {
+    
+    /* Add the output processor number to the extra string */
+    sprintf (extra, "%s.file_%d", extra, myproc / ioUtilGH->ioproc_every);
+    
+    /* If necessary create the output directory */
+    if (myproc == 0) {
+      
+      sprintf (createdir, "mkdir -p %s/%s_3d", ioUtilGH->outpfx_3D, name);
+      
+      if (IO_verbose) {
+        printf ("Creating output directory with command\n %s\n", createdir);
+        fflush (stdout);
+      }
+      
+      if (system (createdir) != 0)
+        printf ("Could not create directory\n");
+    }
+    
+#ifdef MPI
+    /* Wait for all processors to catch up */
+    CACTUS_MPI_ERROR (MPI_Barrier (pughGH->PUGH_COMM_WORLD));
+#endif
+    
+    /* extradir is the relative output directory */
+    sprintf (extradir, "%s_3d/", name);
+    
+  }
+  
+  
+  /* CREATE THE COMPLETE OUTPUT FILENAME 
+     ----------------------------------- */
+  
+  sprintf (filename, "%s/%s%s_3d%s.ieee", ioUtilGH->outpfx_3D,
+           extradir, name, extra);
+
+  /* And be sure to replace any spaces in the filename with an _ */
+  do
+    if (*filename == ' ')
+      *filename = '_';
+  while (*++filename);
+
+}
+
+
+#if        0
+void FMODIFIER FORTRAN_NAME(io_write3d_,
+                            IO_WRITE3D,
+                            io_write3d)
+     (Double *x)
+{
+  pGF *GF;
+
+  GF = locateGFbyDataPtr(x);
+  IO_Write3D(GF->parentGH,GF);
+
+}
+
+#endif