Added routine IOUtil_CreateDirectory() which creates an output directory

on processor 0 and all other I/O processors. This should fix problems with
parallel I/O on non-shared filesystems such as a cluster or in metacomputing
runs between different machines.

Also closes PR CactusBase-758.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusBase/IOUtil/trunk@129 b32723a9-ab3a-4a60-88e2-2e5d99d7c17a

3 files changed, 112 insertions, 29 deletions

diff --git a/src/GHExtension.c b/src/GHExtension.c
index 66d868d..d7d90e9 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -5,8 +5,6 @@
    @desc 
               IOUtil GH extension stuff.
    @enddesc 
-   @history
-   @endhistory
    @version   $Id$
  @@*/
 
@@ -22,6 +20,7 @@
 #include "util_String.h"
 #include "StoreNamedData.h"
 
+#include "CactusBase/IOUtil/src/ioutil_Utils.h"
 #include "ioGH.h"
 
 
@@ -101,23 +100,6 @@ int IOUtil_InitGH (cGH *GH)
 
   myGH = (ioGH *) CCTK_GHExtension (GH, "IO");
 
-  /* Create the checkpoint directory */
-  if (CCTK_MyProc (GH) == 0)
-  {
-    i = CCTK_CreateDirectory (0755, checkpoint_dir);
-    if (i < 0)
-    {
-      CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Problem creating checkpoint directory '%s'",
-                  checkpoint_dir);
-    }
-    else if (i > 0)
-    {
-      CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Checkpoint directory '%s' already exists", checkpoint_dir);
-    }
-  }
-
   if (CCTK_Equals (out3D_mode, "proc"))
   {
     myGH->ioproc = CCTK_MyProc (GH);
@@ -168,6 +150,23 @@ int IOUtil_InitGH (cGH *GH)
     myGH->unchunked = 0;
   }
 
+  /* create the checkpoint directory */
+  i = IOUtil_CreateDirectory (GH, checkpoint_dir,
+                              ! CCTK_Equals (out3D_mode, "onefile"),
+                              myGH->ioproc);
+  if (i < 0)
+  {
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Problem creating checkpoint directory '%s'",
+                checkpoint_dir);
+  }
+  else if (i > 0 && CCTK_Equals (newverbose, "full"))
+  {
+    CCTK_VInfo (CCTK_THORNSTRING,
+                "IOUtil_InitGH: checkpoint directory '%s' already exists",
+                checkpoint_dir);
+  }
+
   /* save downsampling parameters in ioUtilGH because they are temporarily
      reset during checkpointing */
   /* for now we have only parameters for the first 3 dimensions
diff --git a/src/Utils.c b/src/Utils.c
index 7006acb..e5fd1b0 100644
--- a/src/Utils.c
+++ b/src/Utils.c
@@ -79,7 +79,6 @@ int IOUtil_1DLines (cGH *GH,
                     CCTK_REAL *const origin_phys[],
                     int *slice_center[])
 {
-  DECLARE_CCTK_PARAMETERS
   int dim, dir;
   char system_name[20];
   CCTK_REAL *lower_range, *upper_range;
@@ -121,21 +120,27 @@ int IOUtil_1DLines (cGH *GH,
     {
       if (dim == dir)
       {
+        /* line always starts at the first point */
         slice_center[dir][dim] = 0;
       }
       else if (origin_index && origin_index[dir][dim] >= 0)
       {
+/* FIXME: check upper index bounds also ?? */
         slice_center[dir][dim] = origin_index[dir][dim];
       }
       else if (lower_range[dim] > origin_phys[dir][dim] || 
                upper_range[dim] < origin_phys[dir][dim])
       {
-        CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-                    "IOUtil_1DLines: Coordinate in %c-direction (%f, %f) "
-                    "doesn't contain %f",
-                    'x' + dim, (double) lower_range[dim],
-                    (double) upper_range[dim], (double) origin_phys[dir][dim]);
-        slice_center[dir][dim] = 0;
+        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "IOUtil_1DLines: %c-coordinate for slice center of 1D "
+                    "lines in %c-direction (%f) is out of grid coordinates "
+                    "range (%f, %f)",
+                    'x' + dim, 'x' + dir, (double) origin_phys[dir][dim],
+                    (double) lower_range[dim], (double) upper_range[dim]);
+        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "IOUtil_1DLines: no 1D %c-line output will be written with "
+                    "this slice center default", 'x' + dir);
+        slice_center[dir][dim] = -1;
       }
       else 
       {
@@ -145,8 +150,8 @@ int IOUtil_1DLines (cGH *GH,
                 GH->cctk_delta_space[dim] - 1e-6);
 
 #ifdef DEBUG_IOUTIL
-        printf("spxyz for %d-coord of lines in %d-direction is %d\n",
-               dim,dir,slice_center[dir][dim]);
+        printf("spxyz for %c-coord of lines in %c-direction is %d\n",
+               'x' + dim,'x' + dir, slice_center[dir][dim]);
 #endif
       }
     }
@@ -208,7 +213,6 @@ int IOUtil_2DPlanes (cGH *GH,
                      const CCTK_REAL origin_phys[],
                      int slice_center[])
 {
-  DECLARE_CCTK_PARAMETERS
   int dir;
   char system_name[20];
   CCTK_REAL *lower_range, *upper_range;
@@ -371,3 +375,78 @@ void IOUtil_PrintTimings (const char *description,
                   "No timing output available.");
   }
 }
+
+
+ /*@@
+   @routine    IOUtil_CreateDirectory
+   @date       Fri 10 Aug 2001
+   @author     Thomas Radke
+   @desc
+               Creates an output directory path and makes sure it is visible
+               on all I/O processors.
+               It is assumed that processor 0 is always an I/O processor.
+               If there are other I/O processors, they will also try and
+               create the directory themselfs. This guarantees the directory
+               be created on all nodes in case the I/O processors don't share
+               a common filesystem.
+   @enddesc
+   @calls      CCTK_MyProc
+               CCTK_CreateDirectory
+               CCTK_Barrier
+
+   @var        GH
+   @vdesc      pointer to the GH extensions
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        dirname
+   @vdesc      the directory to create
+   @vtype      const char *
+   @vio        in
+   @endvar
+   @var        multiple_io_procs
+   @vdesc      flag indicating that there will be I/O from multiple I/O procs
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        ioproc
+   @vdesc      I/O processor associated with this processor
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for non-I/O processors, or
+               return code of @seeroutine CCTK_CreateDirectory
+   @endreturndesc
+@@*/
+int IOUtil_CreateDirectory (cGH *GH, const char *dirname,
+                            int multiple_io_procs, int ioproc)
+{
+  int myproc, retval;
+
+
+  /* default return value for non-I/O processors */
+  retval = 0;
+
+  /* first, processor 0 creates the directory */
+  myproc = CCTK_MyProc (GH);
+  if (myproc == 0)
+  {
+    retval = CCTK_CreateDirectory (0755, dirname);
+  }
+
+  if (multiple_io_procs)
+  {
+    /* now the other I/O processors create the directory
+       after syncing with processor 0 */
+    CCTK_Barrier (GH);
+    if (myproc == ioproc || ioproc != 0)
+    {
+      retval = CCTK_CreateDirectory (0755, dirname);
+    }
+  }
+
+  return (retval);
+}
diff --git a/src/ioutil_Utils.h b/src/ioutil_Utils.h
index 36c3878..c7e8b2e 100644
--- a/src/ioutil_Utils.h
+++ b/src/ioutil_Utils.h
@@ -32,6 +32,11 @@ int IOUtil_2DPlanes (cGH *GH,
                      const CCTK_REAL origin_phys[],
                      int slice_center[]);
 
+int IOUtil_CreateDirectory (cGH *GH,
+                            const char *dirname,
+                            int multiple_io_procs,
+                            int ioproc);
+
 #ifdef __cplusplus
 }
 #endif