Changes for new PUGH structures

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOFlexIO/trunk@98 ebee0441-1374-4afa-a3b5-247f3ba15b9a

1 files changed, 125 insertions, 68 deletions

diff --git a/src/Write2D.c b/src/Write2D.c
index f60555f..4242c12 100644
--- a/src/Write2D.c
+++ b/src/Write2D.c
@@ -91,10 +91,11 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
   MPI_Status ms;
   MPI_Datatype mpi_type;
 #endif
-
+  pGA *GA;
 
   /* first, check if variable has storage assigned */
-  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index))) {
+  if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (index))) 
+  {
     char *fullname = CCTK_FullName (index);
 
     CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
@@ -108,6 +109,7 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
   ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")];
   myGH = (flexioGH *) GH->extensions [CCTK_GHExtensionHandle ("IOFlexIO")];
 
+
   /* What processor are we on? */
   myproc = CCTK_MyProc (GH);
   nprocs = CCTK_nProcs (GH);
@@ -157,7 +159,8 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 
     /* see if output file for this alias name was already created */
     IEEEfile_2D = (IOFile *) GetNamedData (myGH->filenameList2D, alias);
-    if (IEEEfile_2D == NULL) {
+    if (IEEEfile_2D == NULL) 
+    {
       char *fname;
 
       IEEEfile_2D = (IOFile *) malloc (3 * sizeof (IOFile));
@@ -169,7 +172,8 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
       sprintf (fname, "%s/%s_2d_yz%s.ieee", myGH->outdir2D, alias,
                pughGH->identity_string);
       IEEEfile_2D [0] = IEEEopen (fname, "w");
-      if (! IOisValid (IEEEfile_2D [0])) {
+      if (! IOisValid (IEEEfile_2D [0])) 
+      {
         CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Cannot open 2D output file '%s'", fname);
         return;
@@ -205,22 +209,35 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 
   timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
   if (timelevel > 0)
+  {
     timelevel--;
+  }
+
+  /* Get the GA for this index */
+  GA = ((pGA ***)pughGH->variables)[index][timelevel];
+
   data = CCTK_VarDataPtrI (GH, timelevel, index);
 
-  for (dir = 0; dir < 3; dir++) {
+  for (dir = 0; dir < 3; dir++) 
+  {
     npoints = 0;
     /* Figure out the number of points */
-    if ((pughGH->ownership[pughGH->stagger][0][dir] <= myGH->sp2xyz[dir]) &&
-        (myGH->sp2xyz[dir] < pughGH->ownership[pughGH->stagger][1][dir]))
-      npoints = (pughGH->ownership[pughGH->stagger][1][(dir+1)%3] -
-                 pughGH->ownership[pughGH->stagger][0][(dir+1)%3]) * 
-                (pughGH->ownership[pughGH->stagger][1][(dir+2)%3] -
-                 pughGH->ownership[pughGH->stagger][0][(dir+2)%3]);
+    if ((pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][dir] 
+	 <= myGH->sp2xyz[dir]) &&
+        (myGH->sp2xyz[dir] 
+	 < pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][dir]))
+    {
+      npoints = 
+	(pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+1)%3] -
+	 pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+1)%3])*
+	(pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+2)%3] -
+	 pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+2)%3]);
+    }
 
     /* Set up the number of global points and the points in the "i"
        and "j" directions */
-    if (dir == 0) {
+    if (dir == 0) 
+    {
       ngpoints = GH->cctk_gsh[1] * GH->cctk_gsh[2];
       ni = GH->cctk_gsh[1];
       origin[0] = CCTK_CoordOrigin ("y");
@@ -228,7 +245,9 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
       nj = GH->cctk_gsh[2];
       origin[1] = CCTK_CoordOrigin ("z");
       delta[1] = pughGH->dz0;
-    } else if (dir == 1) {
+    } 
+    else if (dir == 1) 
+    {
       ngpoints = GH->cctk_gsh[0] * GH->cctk_gsh[2];
       ni = GH->cctk_gsh[0];
       origin[0] = CCTK_CoordOrigin ("x");
@@ -236,7 +255,9 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
       nj = GH->cctk_gsh[2];
       origin[1] = CCTK_CoordOrigin ("z");
       delta[1] = pughGH->dz0;
-    } else {
+    } 
+    else 
+    {
       ngpoints = GH->cctk_gsh[0] * GH->cctk_gsh[1];
       ni = GH->cctk_gsh[0];
       origin[0] = CCTK_CoordOrigin ("x");
@@ -246,7 +267,8 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
       delta[1] = pughGH->dy0;
     }
 
-    if (verbose) {
+    if (verbose) 
+    {
       printf ("Npoints in dir %d is %d -> ",dir,npoints);
       printf ("Global npoints is %d\n",ngpoints);
     }
@@ -276,28 +298,40 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
        mod naturally gives is yz zx xy so we need to flip the order.
        thus put the dir+1 % 3 in at lo, the dir+2 % 3 in at hi, and
        set lo and hi directionally as follows ... */
-    if (dir == 0) {lo = 0; hi = 2;}
-    if (dir == 1) {lo = 2; hi = 0;}
-    if (dir == 2) {lo = 0; hi = 2;}
-    if (npoints > 0) {
+    if (dir == 0) 
+    {
+      lo = 0; hi = 2;
+    }
+    if (dir == 1) 
+    {
+      lo = 2; hi = 0;
+    }
+    if (dir == 2) 
+    {
+      lo = 0; hi = 2;
+    }
+    if (npoints > 0) 
+    {
       /* See above comment ... */
       bnd [lo]   = GH->cctk_lbnd[(dir+1)%3] + 
-                   pughGH->ownership[pughGH->stagger][0][(dir+1)%3];
+                   pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+1)%3];
       bnd [lo+1] = GH->cctk_lbnd[(dir+1)%3] + 
-                   pughGH->ownership[pughGH->stagger][1][(dir+1)%3];
+                   pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+1)%3];
 
       bnd [hi]   = GH->cctk_lbnd[(dir+2)%3] + 
-                   pughGH->ownership[pughGH->stagger][0][(dir+2)%3];
+                   pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+2)%3];
       bnd [hi+1] = GH->cctk_lbnd[(dir+2)%3] + 
-                   pughGH->ownership[pughGH->stagger][1][(dir+2)%3];
+                   pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+2)%3];
 
-      locbnd [lo]   = pughGH->ownership[pughGH->stagger][0][(dir+1)%3];
-      locbnd [lo+1] = pughGH->ownership[pughGH->stagger][1][(dir+1)%3];
+      locbnd [lo]   = pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+1)%3];
+      locbnd [lo+1] = pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+1)%3];
 
-      locbnd [hi]   = pughGH->ownership[pughGH->stagger][0][(dir+2)%3];
-      locbnd [hi+1] = pughGH->ownership[pughGH->stagger][1][(dir+2)%3];
+      locbnd [hi]   = pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][0][(dir+2)%3];
+      locbnd [hi+1] = pughGH->GFExtras[GH->cctk_dim-1]->ownership[0][1][(dir+2)%3];
 
-    } else {
+    } 
+    else 
+    {
       bnd [lo] = 0;
       bnd [lo+1] = 0;
       bnd [hi] = 0;
@@ -305,36 +339,44 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
     }
 
     if (verbose)
+    {
       printf ("Local bounds %d: j -> %d %d  k -> %d %d\n",
-               dir, locbnd [0], locbnd [1], locbnd [2], locbnd [3]);
+	      dir, locbnd [0], locbnd [1], locbnd [2], locbnd [3]);
+    }
 
     /* Subsample the data */
-    if (npoints > 0) {
+    if (npoints > 0) 
+    {
       locdat = malloc (npoints * element_size);
       assert (locdat);
       l = 0;
-      if (variable_type == CCTK_VARIABLE_CHAR) {
+      if (variable_type == CCTK_VARIABLE_CHAR) 
+      {
         CCTK_CHAR *char_locdat = (CCTK_CHAR *) locdat;
 
         for (k = locbnd [2]; k < locbnd [3]; k++)
-          for (j = locbnd [0]; j < locbnd [1]; j++) {
+          for (j = locbnd [0]; j < locbnd [1]; j++) 
+	  {
             int pt = 0;
-            if (dir==0) pt = DATINDEX (pughGH, myGH->sp2xyz [dir], j, k);
-            if (dir==1) pt = DATINDEX (pughGH, j, myGH->sp2xyz [dir], k);
-            if (dir==2) pt = DATINDEX (pughGH, j, k, myGH->sp2xyz [dir]);
-            assert (l <= npoints && pt <= pughGH->npoints);
+            if (dir==0) pt = DATINDEX (GA->extras, myGH->sp2xyz [dir], j, k);
+            if (dir==1) pt = DATINDEX (GA->extras, j, myGH->sp2xyz [dir], k);
+            if (dir==2) pt = DATINDEX (GA->extras, j, k, myGH->sp2xyz [dir]);
+            assert (l <= npoints && pt <= GA->extras->npoints);
             char_locdat [l++] = ((CCTK_CHAR *) data) [pt];
           }
-      } else if (variable_type == CCTK_VARIABLE_INT) {
+      } 
+      else if (variable_type == CCTK_VARIABLE_INT) 
+      {
         CCTK_INT *int_locdat = (CCTK_INT *) locdat;
 
         for (k = locbnd [2]; k < locbnd [3]; k++)
-          for (j = locbnd [0]; j < locbnd [1]; j++) {
+          for (j = locbnd [0]; j < locbnd [1]; j++) 
+	  {
             int pt = 0;
-            if (dir==0) pt = DATINDEX (pughGH, myGH->sp2xyz [dir], j, k);
-            if (dir==1) pt = DATINDEX (pughGH, j, myGH->sp2xyz [dir], k);
-            if (dir==2) pt = DATINDEX (pughGH, j, k, myGH->sp2xyz [dir]);
-            assert (l <= npoints && pt <= pughGH->npoints);
+            if (dir==0) pt = DATINDEX (GA->extras, myGH->sp2xyz [dir], j, k);
+            if (dir==1) pt = DATINDEX (GA->extras, j, myGH->sp2xyz [dir], k);
+            if (dir==2) pt = DATINDEX (GA->extras, j, k, myGH->sp2xyz [dir]);
+            assert (l <= npoints && pt <= GA->extras->npoints);
             int_locdat [l++] = ((CCTK_INT *) data) [pt];
           }
       } else if (variable_type == CCTK_VARIABLE_REAL) {
@@ -342,16 +384,30 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
         CCTK_REAL  *real_locdat = (CCTK_REAL *) locdat;
 
         for (k = locbnd [2]; k < locbnd [3]; k++)
-          for (j = locbnd [0]; j < locbnd [1]; j++) {
+          for (j = locbnd [0]; j < locbnd [1]; j++) 
+	  {
             int pt = 0;
-            if (dir==0) pt = DATINDEX (pughGH, myGH->sp2xyz [dir], j, k);
-            if (dir==1) pt = DATINDEX (pughGH, j, myGH->sp2xyz [dir], k);
-            if (dir==2) pt = DATINDEX (pughGH, j, k, myGH->sp2xyz [dir]);
-            assert (l <= npoints && pt <= pughGH->npoints);
+            if (dir==0) 
+	    {
+	      pt = DATINDEX (GA->extras, myGH->sp2xyz [dir], j, k);
+	    }
+            if (dir==1) 
+	    {
+	      pt = DATINDEX (GA->extras, j, myGH->sp2xyz [dir], k);
+	    }
+            if (dir==2) 
+	    {
+	      pt = DATINDEX (GA->extras, j, k, myGH->sp2xyz [dir]);
+	    }
+            assert (l <= npoints && pt <= GA->extras->npoints);
             if (ioUtilGH->out_single)
+	    {
               real4_locdat [l++] = (CCTK_REAL4) (((CCTK_REAL *) data) [pt]);
+	    }
             else
+	    {
               real_locdat [l++] = ((CCTK_REAL *) data) [pt];
+	    }
           }
       }
     } 
@@ -359,7 +415,8 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
     /* Send the data */
 #ifdef MPI
     /* Only send if we have points and we are not on proc 0 */
-    if (npoints > 0 && myproc != 0) {
+    if (npoints > 0 && myproc != 0) 
+    {
       CACTUS_MPI_ERROR (MPI_Send (bnd, 4, PUGH_MPI_INT, 0, 2*myproc+1+BASE,
                         pughGH->PUGH_COMM_WORLD));
       CACTUS_MPI_ERROR (MPI_Send (locdat, (int) npoints, mpi_type, 0,
@@ -367,23 +424,30 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
     }
 
     /* Receive the data on processor 0 */
-    if (myproc == 0) {
+    if (myproc == 0) 
+    {
       alldat = malloc (ngpoints * element_size);
       recd   = malloc (maxrempt * element_size);
       assert (alldat && recd);
 
-      for (i = 0; i < nprocs; i++) {
+      for (i = 0; i < nprocs; i++) 
+      {
         void *ud;
 
         if (verbose)
           printf ("Proceesing proc %d\n",i);fflush(stdout);
 
-        if (nrempoints [i] > 0) {
+        if (nrempoints [i] > 0) 
+	{
           if (i == 0)
+	  {
             ud = locdat;
-          else {
+	  }
+          else 
+	  {
 
-            if (verbose) {
+            if (verbose) 
+	    {
               printf ("Receiving %d from %d\n", nrempoints [i], i);
               fflush (stdout);
             }
@@ -433,7 +497,8 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 #endif
 
     /* Proc 0 writes */
-    if (myproc == 0) {
+    if (myproc == 0) 
+    {
       int dims [2];
       CCTK_REAL min_ext [2], max_ext [2];
 
@@ -463,10 +528,14 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 
     /* Free memory for this time through */
     if (npoints > 0) 
+    {
       free (locdat);
+    }
 #ifdef MPI
     if (myproc == 0)
+    {
       free (alldat);
+    }
 #endif /* If not MPI then alldat and locdat are not distinct */
 
   }
@@ -475,7 +544,9 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 
 #if        0
   if (verbose)
+  {
     cactus_memtrace();
+  }
 #endif
 
   
@@ -483,17 +554,3 @@ void IOFlexIO_Write2D (cGH *GH, int index, const char *alias)
 }
  
 
-
-#if        0
-void FMODIFIER FORTRAN_NAME(io_write2d_,
-                            IO_WRITE2D,
-                            io_write2d)
-     (CCTK_REAL *x)
-{
-  pGF *GF;
-
-  GF = locateGFbyDataPtr(x);
-  IO_Write2D(GF->parentGH,GF);
-}
-
-#endif