Brought Panda stuff up-to-date with current IO interface.

Tested under Linux so far.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOPanda/trunk@11 38c3d835-c875-442e-b0fe-21c19ce1d001

11 files changed, 460 insertions, 225 deletions

diff --git a/src/DumpVar.c b/src/DumpVar.c
index 726eb0a..f5b319a 100644
--- a/src/DumpVar.c
+++ b/src/DumpVar.c
@@ -2,8 +2,7 @@
    @file      DumpVar.c
    @date      01 Oct 1999
    @author    Jonghyun Lee
-   @desc      Do the actual writing of a 3D grid function,
-              for output or for checkpointing
+   @desc      Do the actual writing of a 3D grid array.
    @enddesc 
    @history
    @hendhistory
@@ -19,12 +18,6 @@
 #include "ioPandaGH.h"
 
 
-#define IOTAGBASE 20000 /* This may break on more than 2000 processors */
-
-
-static char *char_time_date = NULL;
-
-
 void IOPanda_getDumpData (cGH *GH, int index, int timelevel, void **outme,
                          int *free_outme, CCTK_INT4 *geom, int element_size) 
 {
@@ -44,6 +37,9 @@ void IOPanda_getDumpData (cGH *GH, int index, int timelevel, void **outme,
   char_ptr = NULL;
   int_ptr = NULL;
 
+  /* get the processor ID */
+  myproc = CCTK_MyProc (GH);
+
   /* get the handle for IOUtil GH extensions */
   ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
 
diff --git a/src/GHExtension.c b/src/GHExtension.c
index cfac94b..735b407 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -15,35 +15,82 @@
 #include <stdio.h>
 
 #include "cctk.h"
-#include "cctk_DefineThorn.h"
 #include "cctk_Parameters.h"
 #include "CactusBase/IOUtil/src/ioGH.h"
-#include "CactusPUGH/PUGH/src/include/pugh.h"
 #include "ioPandaGH.h"
 
 void Panda_Create(int, int);
 
-void *IOPanda_SetupGH (tFleshConfig *config, int convergence_level, cGH *GH)
+
+ /*@@
+   @routine    IOPanda_SetupGH
+   @date       Fri 01 Oct 1999
+   @author     Jonghyun Lee
+   @desc
+               Allocates the IOPanda GH extension structure.
+   @enddesc
+   @calledby   CCTK scheduler at CCTK_INITIALIZE
+   @var        config
+   @vdesc      flesh configuration structure (unused)
+   @vtype      tFleshConfig *
+   @vio        in
+   @endvar
+   @var        convergence_level
+   @vdesc      convergence level (unused)
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        GH
+   @vdesc      pointer to grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @history
+
+   @endhistory
+@@*/
+void *IOPanda_SetupGH (tFleshConfig *config,
+                       int convergence_level,
+                       cGH *GH)
 {
-  int i, numvars;
+  int numvars;
   pandaGH *newGH;
 
+
   numvars = CCTK_NumVars ();
 
-  newGH = (pandaGH *) malloc (sizeof (pandaGH));
-  newGH->do_out3D = (char *) malloc (numvars * sizeof (char));
-  newGH->out3D_last = (int *) malloc (numvars * sizeof (int));
+  newGH             = (pandaGH *) malloc (sizeof (pandaGH));
+  newGH->do_out3D   = (char *)    malloc (numvars * sizeof (char));
+  newGH->out3D_last = (int *)     malloc (numvars * sizeof (int));
 
   return (newGH);
 }
 
+
+ /*@@
+   @routine    IOPanda_InitGH
+   @date       Fri 01 Oct 1999
+   @author     Jonghyun Lee
+   @desc
+               The GH initialization routine for IOFlexIO.
+               Necessary output dirs are created.
+   @enddesc
+   @calledby   CCTK scheduler at CCTK_INITIALIZE
+   @var        GH
+   @vdesc      pointer to grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @history
+
+   @endhistory
+@@*/
 int IOPanda_InitGH (cGH *GH)
 {
   DECLARE_CCTK_PARAMETERS
   int i;
   ioGH *ioUtilGH;
   pandaGH *myGH; 
-  const cParamData *paramdata;
 
 
   /* get the handles for IOUtil and IOPanda extensions */
@@ -53,37 +100,49 @@ int IOPanda_InitGH (cGH *GH)
   /* How often to output */
   myGH->out3D_every = out_every > 0 ? out_every : -1;
   if (out3D_every > 0)
+  {
     myGH->out3D_every = out3D_every;
+  }
 
-  IOUtil_ParseVarsForOutput (out3D_vars, myGH->do_out3D);
-
-  /* Deal with the output directories */
   /* Check whether "outdir3D" was set.
      If so take this dir otherwise default to "IO::outdir" */
-  paramdata = CCTK_ParameterData ("outdir3D", CCTK_THORNSTRING);
-  if (paramdata && paramdata->n_set > 0)
+  if (CCTK_ParameterQueryTimesSet ("outdir3D", CCTK_THORNSTRING) > 0)
+  {
     myGH->outdir3D = strdup (outdir3D);
+  }
   else
+  {
     myGH->outdir3D = strdup (outdir);
+  }
 
-  /* create the output dir */
-  if (CCTK_MyProc (GH) == 0) {
+  /* Create the output directory */
+  if (CCTK_MyProc (GH) == 0)
+  {
     FILE *fp;
 
-    i = CCTK_CreateDirectory(myGH->outdir3D,0755);
+
+    i = CCTK_CreateDirectory (0755, myGH->outdir3D);
     if (i < 0)
+    {
       CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-          "Problem creating IOPanda 3D output directory '%s'", myGH->outdir3D);
+                  "IOPanda_InitGH: Problem creating IOPanda 3D output "
+                  "directory '%s'", myGH->outdir3D);
+    }
     else if (i > 0)
+    {
       CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-            "IOPanda 3D output directory '%s' already exists", myGH->outdir3D);
+                  "IOPanda_InitGH: IOPanda 3D output directory '%s' already "
+                  "exists", myGH->outdir3D);
+    }
     fp = fopen ("FILEPREFIX", "w");
     fprintf (fp, "%s", myGH->outdir3D);
     fclose (fp);
   }
 
   for (i = 0; i < CCTK_NumVars (); i++)
+  {
     myGH->out3D_last [i] = -1;
+  }
 
   Panda_Create (ioUtilGH->ioproc_every, 1);
 
diff --git a/src/Output3D.c b/src/Output3D.c
index ad12776..4b3c5db 100644
--- a/src/Output3D.c
+++ b/src/Output3D.c
@@ -16,69 +16,80 @@
 #include "cctk_Parameters.h"
 #include "CactusBase/IOUtil/src/ioGH.h"
 #include "CactusPUGH/PUGH/src/include/pugh.h"
-#include "external/IEEEIO/src/IOProtos.h"
+#include "external/FlexIO/src/IOProtos.h"
 #include "ioPandaGH.h"
 #include "Panda/c_interface.h"
 
 
+/* function prototypes */
 int IOPanda_Output3DVarAs (cGH *GH, const char *var, const char *alias);
 int IOPanda_TimeFor3D (cGH *GH, int index);
-void IOPanda_Timestep (cGH *GH, int index, const char *alias);
-void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel, CCTK_INT *gsz, char *fname);
-void IOPanda_IEEEIOStructDump (cGH *GH, char *fname);
+void IOPanda_getDumpData (cGH *GH, int index, int timelevel, void **outme,
+                         int *free_outme, CCTK_INT4 *geom, int element_size);
+void PandaTimestep(ArrayInfo *);
+int PandaIsNewFile(char *);
+void Panda_WriteAttribute(char *, char *, int, int, void *);
+
+static int CheckOutputVar (int index);
+static void CheckSteerableParameters (pandaGH *myGH);
+static void SetOutputFlag (int index, const char *optstring, void *arg);
+static void IOPanda_Timestep (cGH *GH, int index, const char *alias);
+static void IOPanda_IEEEIOStructDump (cGH *GH, char *fname);
+static void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel, CCTK_INT *gsz, char *fname);
+static void IOPanda_AddChunkAttributes (cGH *GH, int index, CCTK_INT4 *geom, char *fname);
 
 /*@@
    @routine    IOPanda_Output3DGH
    @date       Sat March 6 1999
    @author     Gabrielle Allen
    @desc
-   Loops over all variables and outputs them if necessary
+               Loops over all variables and outputs them if necessary
    @enddesc
-   @calls    CCTK_GHExtensionHandle
-             CCTK_NumVars
-             CCTK_VarName
-             IOPanda_TimeFor3D
-             IOPanda_Timestep
-   @calledby
-   @history
-
-   @endhistory
-   @var     GH
-   @vdesc   Pointer to CCTK GH
-   @vtype   cGH
-   @vio     in
-   @vcomment
+   @var        GH
+   @vdesc      Pointer to CCTK GH
+   @vtype      cGH *
+   @vio        in
    @endvar
 @@*/
-
 int IOPanda_Output3DGH (cGH *GH)
 {
-  int i;
-  pandaGH *myGH;
   DECLARE_CCTK_PARAMETERS
+  int index;
+  char *fullname;
+  const char *name;
+  pandaGH *myGH;
+
 
   /* Get the GH extension for IOPanda */
   myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];
 
+  CheckSteerableParameters (myGH);
+
   if (myGH->out3D_every <= 0)
+  {
     return (0);
+  }
 
   /* Loop over all variables */
-  for (i = 0; i < CCTK_NumVars (); i++) {
-    if (IOPanda_TimeFor3D (GH, i)) {
-      if (verbose) {
-        char *fullname = CCTK_FullName (i);
-
-        printf ("IOPanda Output3DGH : \n");
-        printf ("  index/fullname = %d/%s\n", i, fullname);
+  for (index = 0; index < CCTK_NumVars (); index++)
+  {
+    if (IOPanda_TimeFor3D (GH, index))
+    {
+      name     = CCTK_VarName (index);
+
+      if (verbose)
+      {
+        fullname = CCTK_FullName (index);
+        CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_Output3DGH: "
+                    "(fullname, name) = (%s, %s)", fullname, name);
         free (fullname);
       }
 
       /* Do the 3D output */
-      IOPanda_Timestep (GH, i, CCTK_VarName (i));
+      IOPanda_Timestep (GH, index, name);
 
       /* Register variable as having 3D output this iteration */
-      myGH->out3D_last [i] = GH->cctk_iteration;
+      myGH->out3D_last [index] = GH->cctk_iteration;
     }
   }
 
@@ -91,46 +102,42 @@ int IOPanda_Output3DGH (cGH *GH)
    @date       Sat March 6 1999
    @author     Gabrielle Allen
    @desc
-   unconditional output of a variable using the IOPanda 3D output method
+               Unconditional output of a variable
+               using the IOPanda 3D output method
    @enddesc
-   @calls    CCTK_DecomposeName
-             CCTK_VarIndex
-             CCTK_GHExtensionHandle
-             IOPanda_Write3D
-   @calledby IOPanda_Output3DGH
-   @history
-
-   @endhistory
-   @var     GH
-   @vdesc   Pointer to CCTK GH
-   @vtype   cGH
-   @vio     in
-   @vcomment
+   @calledby   IOPanda_Output3DGH
+   @var        GH
+   @vdesc      Pointer to CCTK GH
+   @vtype      cGH
+   @vio        in
    @endvar
-   @var     fullname
-   @vdesc   complete name of variable to output
-   @vtype   const char *
-   @vio     in
-   @vcomment
+   @var        fullname
+   @vdesc      complete name of variable to output
+   @vtype      const char *
+   @vio        in
    @endvar
-   @var     alias
-   @vdesc   alias name of variable to output (used to generate output filename)
-   @vtype   const char *
-   @vio     in
-   @vcomment
+   @var        alias
+   @vdesc      alias name of variable to output
+               (used to generate output filename)
+   @vtype      const char *
+   @vio        in
    @endvar
 @@*/
-
-int IOPanda_Output3DVarAs (cGH *GH, const char *fullname, const char *alias)
+int IOPanda_Output3DVarAs (cGH *GH,
+                           const char *fullname,
+                           const char *alias)
 {
   DECLARE_CCTK_PARAMETERS
   int index;
 
+
   index = CCTK_VarIndex(fullname);
 
-  if (verbose) {
-    printf ("\nIn IOPanda Output3DVarAs\n-------------------\n");
-    printf ("  index/fullname/alias = %d/%s/%s\n", index, fullname, alias);
+  if (verbose)
+  {
+    CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_Output3DVarAs: "
+                "(fullname, alias, index) = (%s, %s, %d)",
+                fullname, alias, index);
   }
 
   /* Do the 3D output */
@@ -145,46 +152,55 @@ int IOPanda_Output3DVarAs (cGH *GH, const char *fullname, const char *alias)
    @date       Sat March 6 1999
    @author     Gabrielle Allen
    @desc
-   Decides if it is time to output a variable using the IOPanda 3D output
-   method
+               Decides if it is time to output a variable
+               using the IOPanda 3D output method
    @enddesc
-   @calledby IOPanda_Output3DGH
-   @history
-
-   @endhistory
-   @var     GH
-   @vdesc   Pointer to CCTK GH
-   @vtype   cGH
-   @vio     in
-   @vcomment
+   @calledby   IOPanda_Output3DGH
+   @var        GH
+   @vdesc      Pointer to CCTK GH
+   @vtype      cGH
+   @vio        in
    @endvar
-   @var     index
-   @vdesc   index of variable
-   @vtype   int
-   @vio     in
-   @vcomment
+   @var        index
+   @vdesc      index of variable
+   @vtype      int
+   @vio        in
    @endvar
 @@*/
-
-int IOPanda_TimeFor3D (cGH *GH, int index)
+int IOPanda_TimeFor3D (cGH *GH,
+                       int index)
 {
   pandaGH *myGH;
 
+
   /* Get the GH extension for IOPanda */
   myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];
 
+  CheckSteerableParameters (myGH);
+
+  /* Check if any output was requested */
+  if (myGH->out3D_every <= 0)
+  {
+    return (0);
+  }
+
   /* Check this variable should be output */
   if (! (myGH->do_out3D [index] && GH->cctk_iteration % myGH->out3D_every == 0))
+  {
     return (0);
+  }
 
   /* Check variable not already output this iteration */
-  if (myGH->out3D_last [index] == GH->cctk_iteration) {
-      char *fullname = CCTK_FullName (index);
+  if (myGH->out3D_last [index] == GH->cctk_iteration)
+  {
+    char *fullname = CCTK_FullName (index);
+
 
-      CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Already done IOPanda 3D output for '%s'", fullname);
-      free (fullname);
-      return (0);
+    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Already done IOPanda 3D output for variable '%s' in current "
+                "iteration (probably via triggers)", fullname);
+    free (fullname);
+    return (0);
   }
 
   return (1);
@@ -196,44 +212,36 @@ int IOPanda_TimeFor3D (cGH *GH, int index)
    @date       Sat March 6 1999
    @author     Gabrielle Allen
    @desc
-   Triggers the output a variable using the IOPanda 3D output
-   method
+               Triggers the output a variable using the IOPanda 3D output method
    @enddesc
-   @calls    CCTK_GHExtensionHandle
-             CCTK_VarName
-             IOPanda_Write3D
-   @calledby
-   @history
-
-   @endhistory
-   @var     GH
-   @vdesc   Pointer to CCTK GH
-   @vtype   cGH
-   @vio     in
-   @vcomment
+   @var        GH
+   @vdesc      Pointer to CCTK GH
+   @vtype      cGH
+   @vio        in
    @endvar
-   @var     index
-   @vdesc   index of variable to output
-   @vtype   int
-   @vio     in
-   @vcomment
+   @var        index
+   @vdesc      index of variable to output
+   @vtype      int
+   @vio        in
    @endvar
 @@*/
-
-int IOPanda_TriggerOutput3D (cGH *GH, int index)
+int IOPanda_TriggerOutput3D (cGH *GH,
+                             int index)
 {
   DECLARE_CCTK_PARAMETERS
   pandaGH *myGH;
   const char *varname;
 
+
   varname = CCTK_VarName (index);
 
   /* Get the GH extension for IOPanda */
   myGH = (pandaGH *) GH->extensions [CCTK_GHExtensionHandle ("IOPanda")];
 
-  if (verbose) {
-    printf("\nIn IOPanda TriggerOutput3D\n---------------------\n");
-    printf("  index/name = %d/%s\n", index, varname);
+  if (verbose)
+  {
+    CCTK_VInfo (CCTK_THORNSTRING, "TriggerOutput3D: "
+                "name, index = %s, %d", varname, index);
   }
 
   /* Do the 3D output */
@@ -245,7 +253,98 @@ int IOPanda_TriggerOutput3D (cGH *GH, int index)
   return (0);
 }
 
-void IOPanda_Timestep (cGH *GH, int index, const char *alias)
+
+/**************************** local functions ******************************/
+/* check if steerable parameters have changed */
+static void CheckSteerableParameters (pandaGH *myGH)
+{
+  DECLARE_CCTK_PARAMETERS
+  int times_set;
+  static int out3D_vars_lastset = -1;
+
+
+  /* How often to output */
+  myGH->out3D_every = out_every > 0 ? out_every : -1;
+  if (out3D_every > 0)
+  {
+    myGH->out3D_every = out3D_every;
+  }
+
+  /* re-parse the 'out3D_vars' parameter if it was changed */
+  times_set = CCTK_ParameterQueryTimesSet ("out3D_vars", CCTK_THORNSTRING);
+  if (times_set != out3D_vars_lastset)
+  {
+    memset (myGH->do_out3D, 0, CCTK_NumVars ());
+    CCTK_TraverseString (out3D_vars, SetOutputFlag, myGH->do_out3D,
+                         CCTK_GROUP_OR_VAR);
+
+    /* Save the last setting of 'out3D_vars' parameter */
+    out3D_vars_lastset = times_set;
+  }
+
+}
+
+
+/* check if this variable can be output (static conditions) */
+static int CheckOutputVar (int index)
+{
+  char *errormsg;
+  char *fullname;
+
+
+  errormsg = NULL;
+
+/*** FIXME: IEEEIO doesn't provide a COMPLEX datatype
+            so somehow CCTK_COMPLEX has to be mapped onto REALs or so.
+            We have to check this already here because if an IEEEIO file
+            is created and nothing is written to, it crashes at re-opening. ***/
+  if (CCTK_VarTypeI (index) == CCTK_VARIABLE_COMPLEX)
+  {
+    errormsg = "IOPanda 3D output for complex variable '%s' not yet supported";
+  }
+  else if (CCTK_GroupTypeFromVarI (index) == CCTK_SCALAR)
+  {
+    errormsg = "IOPanda 3D output for scalar variable '%s' not supported";
+  }
+
+  if (errormsg)
+  {
+    fullname = CCTK_FullName (index);
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                errormsg, fullname);
+    free (fullname);
+  }
+
+  return (errormsg != NULL);
+}
+
+
+/* callback for CCTK_TraverseString() to set the output flag
+   for the given variable */
+static void SetOutputFlag (int index,
+                           const char *optstring,
+                           void *arg)
+{
+  char *flags = (char *) arg;
+
+
+  /* Check the variable type */
+  if (CheckOutputVar (index) == 0)
+  {
+    flags[index] = 1;
+  }
+
+  if (optstring)
+  {
+    CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Optional string '%s' in variable name ignored", optstring);
+  }
+}
+
+
+static void IOPanda_Timestep (cGH *GH,
+                              int index,
+                              const char *alias)
 {
   DECLARE_CCTK_PARAMETERS
   void *data;
@@ -254,19 +353,16 @@ void IOPanda_Timestep (cGH *GH, int index, const char *alias)
   CCTK_INT4 bnd[9];
   int free_flag, timelevel;
   ArrayInfo ainfo;
-
   ioGH *ioUtilGH;
   pGH *pughGH;
 
-  if (CCTK_GroupTypeFromVarI (index) == GROUP_SCALAR) {
-    printf("##### %s is scalar\n", alias);
-    return;
-  } 
-  
+
   /* check if variable has storage assigned */
-  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index))) {
+  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index)))
+  {
     char *fullname = CCTK_FullName (index);
 
+
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "No IOPanda 3D output for '%s' (no storage)", fullname);
     free (fullname);
@@ -285,28 +381,35 @@ void IOPanda_Timestep (cGH *GH, int index, const char *alias)
   tmp1[2] = GH->cctk_gsh[0]; 
   ainfo.size_ = tmp1;
   
-  switch (CCTK_VarTypeI (index)) {
-  case CCTK_VARIABLE_CHAR:
-    ainfo.esize_ = CHAR;
-    break;
-  case CCTK_VARIABLE_INT:
+  switch (CCTK_VarTypeI (index))
+  {
+    case CCTK_VARIABLE_CHAR:
+      ainfo.esize_ = CHAR;
+      break;
+
+    case CCTK_VARIABLE_INT:
 #ifdef  CCTK_INTEGER_PRECISION_8
-    ainfo.esize_ = INT64;
+      ainfo.esize_ = INT64;
 #elif   CCTK_INTEGER_PRECISION_4
-    ainfo.esize_ = INT32;
+      ainfo.esize_ = INT32;
 #elif   CCTK_INTEGER_PRECISION_2
-    ainfo.esize_ = INT16;
+      ainfo.esize_ = INT16;
 #endif
-    break;
-  case CCTK_VARIABLE_REAL:
-    if (ioUtilGH->out_single) ainfo.esize_ = FLOAT32;
-    else {
+      break;
+
+    case CCTK_VARIABLE_REAL:
+      if (ioUtilGH->out_single)
+      {
+        ainfo.esize_ = FLOAT32;
+      }
+      else
+      {
 #ifdef  CCTK_REAL_PRECISION_8
-      ainfo.esize_ = FLOAT64;
+        ainfo.esize_ = FLOAT64;
 #elif   CCTK_REAL_PRECISION_4
-      ainfo.esize_ = FLOAT32;
+        ainfo.esize_ = FLOAT32;
 #endif    
-    }
+      }
   }
   
   /* FIXME: rank */
@@ -326,10 +429,12 @@ void IOPanda_Timestep (cGH *GH, int index, const char *alias)
   ainfo.disk_dist_ = dist2;
 
   timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
-  if (timelevel > 0) timelevel--;
-  
-  IOPanda_getDumpData(GH, index, timelevel, &data, &free_flag, bnd, 
-		      ainfo.esize_);
+  if (timelevel > 0)
+  {
+    timelevel--;
+  }
+
+  IOPanda_getDumpData(GH, index, timelevel, &data, &free_flag, bnd, ainfo.esize_);
   ainfo.data_ = (char *)data;
 
   /*** FIXME: asymmetric ghostzones */
@@ -337,24 +442,40 @@ void IOPanda_Timestep (cGH *GH, int index, const char *alias)
 
   PandaTimestep(&ainfo);
   if (free_flag)
+  {
     free (data);
+  }
 
   IOPanda_AddCommonAttributes(GH, index, timelevel, ainfo.size_, ainfo.name_);
 
+  if (! ioUtilGH->unchunked)
+  {
+    /* Write chunk attributes */
+    IOPanda_AddChunkAttributes (GH, index, bnd, ainfo.name_);
+  }
+
   if (PandaIsNewFile(ainfo.name_))
+  {
     IOPanda_IEEEIOStructDump(GH, ainfo.name_);
+  }
 }
 
-void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel,
-                                 CCTK_INT4 gsz [3], char *fname)
+
+static void IOPanda_AddCommonAttributes (cGH *GH,
+                                         int index,
+                                         int timelevel,
+                                         CCTK_INT4 gsz [3],
+                                         char *fname)
 {
   DECLARE_CCTK_PARAMETERS
   CCTK_REAL d3_to_IO [6]; /* buffer for writing doubles to IEEEIO */
   CCTK_INT4 i_to_IO;      /* buffer for writing an int to IEEEIO */
+  CCTK_REAL dummy;
   char *name, *gname;
   ioGH *ioUtilGH;
   char *char_time_date = "";
 
+
 #ifdef SGI
   time_t t = time(NULL);
   char_time_date = asctime (localtime (&t));
@@ -364,9 +485,7 @@ void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel,
   ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
 
   name = CCTK_FullName (index);
-  
   Panda_WriteAttribute (fname, "name", BYTE, strlen (name) + 1, name);
-  
   free (name);
 
   gname = CCTK_GroupNameFromVarI (index);
@@ -374,26 +493,25 @@ void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel,
   free (gname);
 
   i_to_IO = CCTK_GroupTypeFromVarI (index);
-  Panda_WriteAttribute (fname, "grouptype", INT32,
-                       1, &i_to_IO);
+  Panda_WriteAttribute (fname, "grouptype", INT32, 1, &i_to_IO);
 
   i_to_IO = CCTK_NumTimeLevelsFromVarI (index);
-  Panda_WriteAttribute (fname, "ntimelevels", INT32,
-                       1, &i_to_IO);
+  Panda_WriteAttribute (fname, "ntimelevels", INT32, 1, &i_to_IO);
 
   i_to_IO = timelevel;
-  Panda_WriteAttribute (fname, "timelevel", INT32,
-                       1, &i_to_IO);
+  Panda_WriteAttribute (fname, "timelevel", INT32, 1, &i_to_IO);
 
   if (char_time_date && out3D_datestamp)
+  {
     Panda_WriteAttribute (fname, "date", BYTE,
                          strlen (char_time_date) + 1, char_time_date);
+  }
 
   Panda_WriteAttribute (fname, "time", FLOAT64, 1,&GH->cctk_time);
 
-  d3_to_IO [0] = CCTK_CoordOrigin ("x");
-  d3_to_IO [1] = CCTK_CoordOrigin ("y");
-  d3_to_IO [2] = CCTK_CoordOrigin ("z");
+  d3_to_IO [0] = CCTK_CoordRange (GH, &d3_to_IO [0], &dummy, -1, "x", "cart3d");
+  d3_to_IO [1] = CCTK_CoordRange (GH, &d3_to_IO [1], &dummy, -1, "y", "cart3d");
+  d3_to_IO [2] = CCTK_CoordRange (GH, &d3_to_IO [2], &dummy, -1, "z", "cart3d");
   Panda_WriteAttribute (fname, "origin", FLOAT64,3,d3_to_IO);
   CCTK_CoordRange (GH, &d3_to_IO [0], &d3_to_IO [3], -1, "x", "cart3d");
   CCTK_CoordRange (GH, &d3_to_IO [1], &d3_to_IO [4], -1, "y", "cart3d");
@@ -409,7 +527,8 @@ void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel,
 
   if (ioUtilGH->downsample[0] > 1 ||
       ioUtilGH->downsample[1] > 1 ||
-      ioUtilGH->downsample[2] > 1) {
+      ioUtilGH->downsample[2] > 1)
+  {
     d3_to_IO [0] = GH->cctk_delta_space [0];
     d3_to_IO [1] = GH->cctk_delta_space [1];
     d3_to_IO [2] = GH->cctk_delta_space [2];
@@ -432,9 +551,39 @@ void IOPanda_AddCommonAttributes (cGH *GH, int index, int timelevel,
 }  
 
 
-void IOPanda_IEEEIOStructDump (cGH *GH, char *fname)
+static void IOPanda_AddChunkAttributes (cGH *GH,
+                                        int index,
+                                        CCTK_INT4 *geom,
+                                        char *fname)
 {
+  int dim;
+  char *name;
+  CCTK_INT4 i_to_IO;
+
 
+  /* there is nothing to do for a serial run */
+  if (CCTK_nProcs (GH) == 1)
+    return;
+
+  /* get the dimension of the variable */
+  dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (index));
+
+  Panda_WriteAttribute (fname, "chunk_origin", INT32, dim, &geom[0]);
+  Panda_WriteAttribute (fname, "subchunk_lb", INT32, dim, &geom[0]);
+  Panda_WriteAttribute (fname, "global_size", INT32, dim, &geom[2*dim]);
+
+  i_to_IO = GH->cctk_iteration;
+  Panda_WriteAttribute (fname, "chunk_dataset", INT32, 1, &i_to_IO);
+
+  name = CCTK_FullName (index);
+  Panda_WriteAttribute (fname, "name", CHAR, strlen (name)+1, name);
+  free (name);
+}
+
+
+static void IOPanda_IEEEIOStructDump (cGH *GH,
+                                      char *fname)
+{
   CCTK_INT4 i_temp;
   CCTK_REAL d_temp;
   ioGH *ioUtilGH;
@@ -443,18 +592,14 @@ void IOPanda_IEEEIOStructDump (cGH *GH, char *fname)
   ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
 
   i_temp = GH->cctk_iteration;
-  Panda_WriteAttribute (fname, "GH$iteration", INT32,
-                       1, &i_temp);
+  Panda_WriteAttribute (fname, "GH$iteration", INT32, 1, &i_temp);
 
   i_temp = ioUtilGH->ioproc_every;
-  Panda_WriteAttribute (fname, "GH$ioproc_every", INT32,
-                       1, &i_temp);
+  Panda_WriteAttribute (fname, "GH$ioproc_every", INT32, 1, &i_temp);
 
   i_temp = CCTK_nProcs (GH);
-  Panda_WriteAttribute (fname, "GH$nprocs", INT32,
-                       1, &i_temp);
+  Panda_WriteAttribute (fname, "GH$nprocs", INT32, 1, &i_temp);
 
   d_temp = GH->cctk_time;
-  Panda_WriteAttribute (fname, "GH$time", FLOAT64,
-                       1, &d_temp);
+  Panda_WriteAttribute (fname, "GH$time", FLOAT64, 1, &d_temp);
 }
diff --git a/src/Panda/Array.C b/src/Panda/Array.C
index e2fd7eb..757b07e 100644
--- a/src/Panda/Array.C
+++ b/src/Panda/Array.C
@@ -2,7 +2,7 @@
 #include "MPIFS.h"
 #include "Array.h"
 
-#include "external/IEEEIO/src/Arch.h"
+#include "external/FlexIO/src/Arch.h"
 
 extern "C" {
   int IOsizeOf(int);
diff --git a/src/Panda/Array.h b/src/Panda/Array.h
index 834fd36..4d4f104 100644
--- a/src/Panda/Array.h
+++ b/src/Panda/Array.h
@@ -5,7 +5,7 @@
 #include "ArrayDistribution.h"
 #include "Chunk.h"
 
-#include "external/IEEEIO/src/Arch.h"
+#include "external/FlexIO/src/Arch.h"
 
 //#include "../IEEEIO/IEEEIO.h"
 //#include "../IEEEIO/IOProtos.h"
diff --git a/src/Panda/Attribute.C b/src/Panda/Attribute.C
index 0c50f04..31947e1 100644
--- a/src/Panda/Attribute.C
+++ b/src/Panda/Attribute.C
@@ -22,17 +22,17 @@ Attribute::Attribute()
 
 void Attribute::init(char *name)
 {
-  int len = strlen(name);
+  int i, len = strlen(name);
   name_ = (char *)malloc(sizeof(char) * (len + 1));
-  for (int i=0; i<len; i++) name_[i] = name[i];
+  for (i=0; i<len; i++) name_[i] = name[i];
   name_[i] = '\0';
 }
 
 void Attribute::init(char *name, int esize, int count, void *data)
 {
-  int len = strlen(name);
+  int i, len = strlen(name);
   name_ = (char *)malloc(sizeof(char) * (len + 1));
-  for (int i=0; i<len; i++) name_[i] = name[i];
+  for (i=0; i<len; i++) name_[i] = name[i];
   name_[i] = '\0';
   esize_ = esize;
   count_ = count;
@@ -118,12 +118,19 @@ void Attribute::read(char *fname, char *n)
       MPIFS_global_obj->send_attr_schema(this, fname, READ_TIMESTEP);
     MPIFS_global_obj->receive_attr_schema(); 
 
-    int len = strlen(fname);
+    int i, len = strlen(fname);
     char *name = (char *)malloc(sizeof(char) * (len+1));
-    char *name1 = (char *)malloc(sizeof(char) * (len+6));
-    for (int i=0; i<len; i++) name[i] = fname[i];
+    char *name1 = (char *)malloc(sizeof(char) * (len+16));
+    for (i=0; i<len; i++) name[i] = fname[i];
     name[i] = '\0';
-    sprintf(name1, "%s.%d", name, MPIFS_global_obj->my_rank(IO_NODE));
+    if (MPIFS_global_obj->app_size(IO_NODE) > 1)
+    {
+      sprintf(name1, "%s.file_%d.ieee", name, MPIFS_global_obj->my_rank(IO_NODE));
+    }
+    else
+    {
+      sprintf(name1, "%s.ieee", name);
+    }
     fp = MPIFS_global_obj->open_file(name1, READ_TIMESTEP);   
     read_data(fp);
     if (MPIFS_global_obj->am_master_io_node()) {
@@ -150,12 +157,19 @@ void Attribute::write(char *fname, char *n, int esize, int count, void *data)
     MPIFS_global_obj->receive_attr_schema(); 
 
     IOFile fp;
-    int len = strlen(fname);
+    int i, len = strlen(fname);
     char *name = (char *)malloc(sizeof(char) * (len+1));
-    char *name1 = (char *)malloc(sizeof(char) * (len+6));
-    for (int i=0; i<len; i++) name[i] = fname[i];
+    char *name1 = (char *)malloc(sizeof(char) * (len+16));
+    for (i=0; i<len; i++) name[i] = fname[i];
     name[i] = '\0';
-    sprintf(name1, "%s.%d", name, MPIFS_global_obj->my_rank(IO_NODE));
+    if (MPIFS_global_obj->app_size(IO_NODE) > 1)
+    {
+      sprintf(name1, "%s.file_%d.ieee", name, MPIFS_global_obj->my_rank(IO_NODE));
+    }
+    else
+    {
+      sprintf(name1, "%s.ieee", name);
+    }
   
     fp = MPIFS_global_obj->open_file(name1, TIMESTEP);
   
diff --git a/src/Panda/Attribute.h b/src/Panda/Attribute.h
index d948316..6f96fe5 100644
--- a/src/Panda/Attribute.h
+++ b/src/Panda/Attribute.h
@@ -11,7 +11,7 @@ typedef union int_to_char {
 
 //#include "../IEEEIO/IEEEIO.h"
 //#include "../IEEEIO/IOProtos.h"
-#include "external/IEEEIO/src/Arch.h"
+#include "external/FlexIO/src/Arch.h"
 
 class Attribute {
   char 	*name_;
diff --git a/src/Panda/MPIFS.C b/src/Panda/MPIFS.C
index e8b56ae..560a0e2 100644
--- a/src/Panda/MPIFS.C
+++ b/src/Panda/MPIFS.C
@@ -8,7 +8,7 @@
 #define Max_Open_Files 1000
 
 
-#include "external/IEEEIO/src/Arch.h"
+#include "external/FlexIO/src/Arch.h"
 
 extern "C" {
   IOFile IEEEopen(char *,char *);
@@ -544,9 +544,10 @@ Boolean MPIFS::received_quit_message(int msg_src, int msg_tag,
 
 IOFile MPIFS::open_file(char *name, int op_type)
 {
+  int i;
   IOFile fp;
 
-  for (int i=0; i<num_open_files_; i++) 
+  for (i=0; i<num_open_files_; i++) 
     if (!strcmp(name, open_file_names_[i])) { 
       fp = open_file_ptrs_[i];
       free(name);
@@ -576,9 +577,16 @@ IOFile MPIFS::open_file(char *name, int op_type)
 
 Boolean MPIFS::is_new_file(char *name) {
   char name1[100];
-  if (node_type() == PART_TIME_COMPUTE || node_type() == COMPUTE_NODE) return;
-  sprintf(name1, "%s.%d", name, my_rank(IO_NODE));
+  if (node_type() == PART_TIME_COMPUTE || node_type() == COMPUTE_NODE) return NO;
 
+  if (app_size(IO_NODE) > 1)
+  {
+    sprintf(name1, "%s.file_%d.ieee", name, my_rank(IO_NODE));
+  }
+  else
+  {
+    sprintf(name1, "%s.ieee", name);
+  }
   for (int i=0; i<num_open_files_; i++) 
     if (!strcmp(name1, open_file_names_[i])) {
       if (is_new_file_[i] == YES) {
@@ -611,10 +619,17 @@ void MPIFS::start_attribute_io(int msg_src, int msg_tag, MPI_Status *status)
   for (i=0; i<4; i++) tmp.c[i] = *ptr++;
   int len = tmp.i;
   char *fname = (char *)malloc(sizeof(char) * (len + 1));
-  char *name1 = (char *)malloc(sizeof(char) * (len + 6));
+  char *name1 = (char *)malloc(sizeof(char) * (len + 16));
   for (i=0; i<len; i++) fname[i] = *ptr++;
   fname[i] = '\0';
-  sprintf(name1, "%s.%d", fname, world_rank_);
+  if (app_size(IO_NODE) > 1)
+  {
+    sprintf(name1, "%s.file_%d.ieee", fname, world_rank_);
+  }
+  else
+  {
+    sprintf(name1, "%s.ieee", fname);
+  }
   fp = open_file(name1, op_type);
   Attribute *attr = new Attribute(ptr, op_type);
   if (op_type == TIMESTEP) attr->write_data(fp);  
@@ -638,12 +653,19 @@ void MPIFS::start_collective_io(int msg_src, int msg_tag, MPI_Status *status)
 		  msg_tag, MPI_COMM_WORLD,status);
   Broadcast(IO_NODE, (void *)schema_buf, msg_len, MPI_INT, msg_tag);
 
-  int len = schema_buf[2];
+  int i, len = schema_buf[2];
   char *name = (char *)malloc(sizeof(char) * (len + 1));
-  char *name1 = (char *)malloc(sizeof(char) * (len + 6));
-  for (int i=0; i<len; i++) name[i] = schema_buf[3+i];
+  char *name1 = (char *)malloc(sizeof(char) * (len + 16));
+  for (i=0; i<len; i++) name[i] = schema_buf[3+i];
   name[i] = '\0';
-  sprintf(name1, "%s.%d", name, world_rank_);
+  if (app_size(IO_NODE) > 1)
+  {
+    sprintf(name1, "%s.file_%d.ieee", name, world_rank_);
+  }
+  else
+  {
+    sprintf(name1, "%s.ieee", name);
+  }
   fp = open_file(name1, schema_buf[1]);
   free(name);
 
@@ -677,12 +699,19 @@ void MPIFS::part_time_io_node_loop(int *schema_buf, int msg_len,
   Collective_IO *new_io;
   IOFile fp = 0;
 
-  int len = schema_buf[2];
+  int i, len = schema_buf[2];
   char *name = (char *)malloc(sizeof(char) * (len + 1));
-  char *name1 = (char *)malloc(sizeof(char) * (len + 6));
-  for (int i=0; i<len; i++) name[i] = schema_buf[3+i];
+  char *name1 = (char *)malloc(sizeof(char) * (len + 16));
+  for (i=0; i<len; i++) name[i] = schema_buf[3+i];
   name[i] = '\0';
-  sprintf(name1, "%s.%d", name, world_rank_);
+  if (app_size(IO_NODE) > 1)
+  {
+    sprintf(name1, "%s.file_%d.ieee", name, world_rank_);
+  }
+  else
+  {
+    sprintf(name1, "%s.ieee", name);
+  }
   fp = open_file(name1, schema_buf[1]);
   free(name);
 
diff --git a/src/Panda/Simple_IO.C b/src/Panda/Simple_IO.C
index a3c7bc0..54998eb 100644
--- a/src/Panda/Simple_IO.C
+++ b/src/Panda/Simple_IO.C
@@ -6,8 +6,8 @@
 #include "Array.h"
 #include "message.h"
 
-#include "external/IEEEIO/src/Arch.h"
-#include "external/IEEEIO/src/IOProtos.h"
+#include "external/FlexIO/src/Arch.h"
+#include "external/FlexIO/src/IOProtos.h"
 
 extern MPIFS* MPIFS_global_obj;
 extern int SUBCHUNK_SIZE;
diff --git a/src/Panda/c_interface.h b/src/Panda/c_interface.h
index b167f6f..aba8274 100644
--- a/src/Panda/c_interface.h
+++ b/src/Panda/c_interface.h
@@ -1,7 +1,7 @@
 #ifndef _included_C_Interface_h
 #define _included_C_Interface_h
 
-#include "external/IEEEIO/src/IEEEIO.h" 
+#include "external/FlexIO/src/IEEEIO.h" 
 
 
 typedef enum { NONE,
diff --git a/src/make.configuration.defn b/src/make.configuration.defn
index b051521..5fe526e 100644
--- a/src/make.configuration.defn
+++ b/src/make.configuration.defn
@@ -1,14 +1,6 @@
 # make.configuration.defn for IOPanda
 
-# make sure that IOPanda was configured in with MPI, PUGH, and IEEEIO
-
-ifeq ($(findstring external/IEEEIO,$(THORNS)),)
-.pseudo: MissingIEEEIOinIOPanda
-MissingIEEEIOinIOPanda:
-	@echo "IOPanda: requires IEEEIO"
-	@echo "IOPanda: Please add external/IEEEIO or remove IOPanda from Thornlist !"
-	exit 2
-endif
+# make sure that IOPanda was configured with PUGH and MPI
 
 ifeq ($(findstring CactusPUGH/PUGH,$(THORNS)),)
 .pseudo: MissingPUGHinIOPanda