Introduce cctk_ash, retire cctk_lssh

Introduce cctk_ash, describing the process-local array shape that has
been allocated. This may be larger than cctk_lsh, the process-local
shape that should be used.

Retire cctk_lssh and related infrastructure to handle staggered grid
functions.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@515 b61c5cb5-eaca-4651-9a7a-d64986f99364

15 files changed, 133 insertions, 284 deletions

diff --git a/src/Comm.c b/src/Comm.c
index 792882e..10aa26b 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -8,7 +8,7 @@
    @version   $Id$
  @@*/
 
-/*#define DEBUG_PUGH 1*/
+/* #define DEBUG_PUGH 1 */
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -1098,15 +1098,15 @@ static int PUGH_SyncSingleProc(pGH *pughGH, pComm *comm)
         if (pass_to > pass_total) pass_to = pass_total;
 
         /* get the index ranges for the nested loops */
-        istart_from   = GA->extras->overlap[GA->stagger[face>>1]][0][face];
-        iend_from     = GA->extras->overlap[GA->stagger[face>>1]][1][face];
+        istart_from   = GA->extras->overlap[0][face];
+        iend_from     = GA->extras->overlap[1][face];
         if (face & 1)
         {
-          istart_to   = GA->extras->ghosts[GA->stagger[face>>1]][0][face - 1];
+          istart_to   = GA->extras->ghosts[0][face - 1];
         }
         else
         {
-          istart_to   = GA->extras->ghosts[GA->stagger[face>>1]][0][face + 1];
+          istart_to   = GA->extras->ghosts[0][face + 1];
         }
 
         /* set iterators to the start vector */
diff --git a/src/FinishReceiveGA.c b/src/FinishReceiveGA.c
index 31b469b..1857fd3 100644
--- a/src/FinishReceiveGA.c
+++ b/src/FinishReceiveGA.c
@@ -10,7 +10,7 @@
    @version   $Id$
  @@*/
 
-/*#define DEBUG_PUGH*/
+/* #define DEBUG_PUGH 1 */
 
 #include <string.h>
 
@@ -84,8 +84,8 @@ void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm)
           comm->n_vars, GA->name);
 #endif
 
-  istart   = GA->extras->ghosts[GA->stagger[dir>>1]][0][dir];
-  iend     = GA->extras->ghosts[GA->stagger[dir>>1]][1][dir];
+  istart   = GA->extras->ghosts[0][dir];
+  iend     = GA->extras->ghosts[1][dir];
   iterator = GA->extras->iterator;
 
   /* set iterator to the start vector */
diff --git a/src/GHExtension.c b/src/GHExtension.c
index 8e14fd2..5c43c30 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -99,7 +99,6 @@ void *PUGH_SetupGH (tFleshConfig *config,
   PUGH_GFPeriodic (maxdim, perme);
 
   newGH = PUGH_SetupPGH (GH, maxdim, nsize, ghostsize,
-                         CCTK_StaggerVars() ? PUGH_STAGGER : PUGH_NO_STAGGER,
                          perme);
   if (!newGH)
   {
@@ -152,7 +151,7 @@ void *PUGH_SetupGH (tFleshConfig *config,
 
     PUGH_SetupGroup (newGH, groupsize, groupghostsize, pgroup.grouptype,
                      pgroup.vartype, pgroup.dim, pgroup.numvars,
-                     pgroup.vectorlength, pgroup.stagtype,
+                     pgroup.vectorlength,
                      pgroup.numtimelevels, pgroup.vectorgroup);
   }
 
@@ -178,7 +177,7 @@ void *PUGH_SetupGH (tFleshConfig *config,
 @@*/
 static void PUGH_InitGHBasics (cGH *GH)
 {
-  int  dim, stagger;
+  int  dim;
   pGH *mypGH;
   pGExtras *GFExtras;
 
@@ -210,14 +209,7 @@ static void PUGH_InitGHBasics (cGH *GH)
     GH->cctk_lbnd[dim]        = GFExtras->lb[mypGH->myproc][dim];
     GH->cctk_ubnd[dim]        = GFExtras->ub[mypGH->myproc][dim];
 
-    for (stagger = 0; stagger < CCTK_NSTAGGER; stagger++)
-    {
-      GH->cctk_lssh[CCTK_LSSH_IDX (stagger, dim)] = GH->cctk_lsh[dim];
-      if (GH->cctk_bbox[2*dim + 1] == 1 && stagger > 0)
-      {
-        GH->cctk_lssh[CCTK_LSSH_IDX (stagger, dim)]--;
-      }
-    }
+    GH->cctk_ash[dim] = GH->cctk_lsh[dim];
   }
 }
 
@@ -239,7 +231,7 @@ static void PUGH_InitGHBasics (cGH *GH)
 @@*/
 static void PUGH_InitGHVariables (cGH *GH)
 {
-  int  var, gtype, level, ntimelevels;
+  int  var, level, ntimelevels;
   pGH *mypGH;
 
 
@@ -247,7 +239,6 @@ static void PUGH_InitGHVariables (cGH *GH)
 
   for (var = 0; var < mypGH->nvariables; var++)
   {
-    gtype = CCTK_GroupTypeFromVarI (var);
     ntimelevels = CCTK_MaxTimeLevelsVI (var);
 
     for (level = 0; level < ntimelevels; level++)
diff --git a/src/Overloadables.c b/src/Overloadables.c
index c271e36..c6c355c 100644
--- a/src/Overloadables.c
+++ b/src/Overloadables.c
@@ -57,7 +57,7 @@ int PUGHi_NumTimeLevelsArray(const pGH *pughGH, int var, int timelevels);
 @@*/
 int PUGH_GroupDynamicData (const cGH *GH, int group, cGroupDynamicData *data)
 {
-  int gtype, var;
+  int var;
   pGH *pughGH;
   pGExtras *extras;
   int retval;
@@ -65,7 +65,6 @@ int PUGH_GroupDynamicData (const cGH *GH, int group, cGroupDynamicData *data)
 
   if (data)
   {
-    gtype = CCTK_GroupTypeI (group);
     /* Get the first variable in the group */
     var = CCTK_FirstVarIndexI (group);
     if (var<0)
@@ -81,9 +80,7 @@ int PUGH_GroupDynamicData (const cGH *GH, int group, cGroupDynamicData *data)
 
       data->dim         = extras->dim;
       data->lsh         = extras->lnsize;
-#ifdef CCTK_GROUPDYNAMICDATA_HAS_LSSH
-      data->lssh        = NULL;
-#endif
+      data->ash         = extras->lnsize;
       data->gsh         = extras->nsize;
       data->lbnd        = extras->lb[pughGH->myproc];
       data->ubnd        = extras->ub[pughGH->myproc];
diff --git a/src/PostReceiveGA.c b/src/PostReceiveGA.c
index ad4d24e..c8b4641 100644
--- a/src/PostReceiveGA.c
+++ b/src/PostReceiveGA.c
@@ -10,6 +10,8 @@
    @version $Header$
  @@*/
 
+/* #define DEBUG_PUGH 1 */
+
 #include <stdio.h>
 
 #include "pugh.h"
@@ -18,8 +20,6 @@ static const char *rcsid = "$Header$";
 
 CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c);
 
-/*#define DEBUG_PUGH*/
-
 
 #ifdef CCTK_MPI
 
@@ -102,19 +102,19 @@ void PostReceiveGA(pGH *pughGH, int dir, pComm *comm)
     switch (GA->vtype) 
     {
       case CCTK_VARIABLE_BYTE:
-        recv_dt = pughGH->recv_char_dt[GA->stagger[dir>>1]];
+        recv_dt = pughGH->recv_char_dt;
         break;
 
       case CCTK_VARIABLE_INT:
-        recv_dt = pughGH->recv_int_dt[GA->stagger[dir>>1]];
+        recv_dt = pughGH->recv_int_dt;
         break;
 
       case CCTK_VARIABLE_REAL:
-        recv_dt = pughGH->recv_real_dt[GA->stagger[dir>>1]];
+        recv_dt = pughGH->recv_real_dt;
         break;
 
       case CCTK_VARIABLE_COMPLEX:
-        recv_dt = pughGH->recv_complex_dt[GA->stagger[dir>>1]];
+        recv_dt = pughGH->recv_complex_dt;
         break;
 
       default:
diff --git a/src/PostSendGA.c b/src/PostSendGA.c
index 2692ac1..22f11b5 100644
--- a/src/PostSendGA.c
+++ b/src/PostSendGA.c
@@ -11,6 +11,8 @@
    @version $Header$
  @@*/
 
+/* #define DEBUG_PUGH 1 */
+
 #include <stdio.h>
 #include <string.h>
 
@@ -21,8 +23,6 @@ static const char *rcsid = "$Header$";
 
 CCTK_FILEVERSION(CactusPUGH_PUGH_PostSendGA_c);
 
-/*#define DEBUG_PUGH*/
-
 #ifdef CCTK_MPI
 
 void PostSendGA(pGH *pughGH, int dir, pComm *comm);
@@ -103,8 +103,8 @@ void PostSendGA(pGH *pughGH, int dir, pComm *comm)
 
   if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
   {
-    istart   = GA->extras->overlap[GA->stagger[dir>>1]][0][dir];
-    iend     = GA->extras->overlap[GA->stagger[dir>>1]][1][dir];
+    istart   = GA->extras->overlap[0][dir];
+    iend     = GA->extras->overlap[1][dir];
     iterator = GA->extras->iterator;
 
     /* set iterator to the start vector */
@@ -194,19 +194,19 @@ void PostSendGA(pGH *pughGH, int dir, pComm *comm)
     {
 
       case CCTK_VARIABLE_BYTE:
-        send_dt = pughGH->send_char_dt[GA->stagger[dir>>1]];
+        send_dt = pughGH->send_char_dt;
         break;
 
       case CCTK_VARIABLE_INT:
-        send_dt = pughGH->send_int_dt[GA->stagger[dir>>1]];
+        send_dt = pughGH->send_int_dt;
         break;
 
       case CCTK_VARIABLE_REAL:
-        send_dt = pughGH->send_real_dt[GA->stagger[dir>>1]];
+        send_dt = pughGH->send_real_dt;
         break;
 
       case CCTK_VARIABLE_COMPLEX:
-        send_dt = pughGH->send_complex_dt[GA->stagger[dir>>1]];
+        send_dt = pughGH->send_complex_dt;
         break;
 
       default:
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index bd00fd6..1a9c24c 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -33,7 +33,6 @@ static int PUGH_SetupGAGroup (pGH *newGH,
                               int  dim,
                               int  n_variables,
                               int  vectorlength,
-                              int  staggercode,
                               int  n_timelevels);
 
 
@@ -85,11 +84,6 @@ static int PUGH_SetupGAGroup (pGH *newGH,
    @vtype      int
    @vio        in
    @endvar
-   @var        staggercode
-   @vdesc      stagger code for variables in this group
-   @vtype      int
-   @vio        in
-   @endvar
    @var        n_timelevels
    @vdesc      number of timelevels in this group
    @vtype      int
@@ -110,7 +104,6 @@ static int PUGH_SetupGAGroup (pGH *newGH,
                               int  dim,
                               int  n_variables,
                               int  vectorlength,
-                              int  staggercode,
                               int  n_timelevels)
 {
   int i;
@@ -158,7 +151,7 @@ static int PUGH_SetupGAGroup (pGH *newGH,
       connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nsize,
 					     ghostsize, nprocs, perme);
 
-      extras = PUGH_SetupPGExtras (0, dim, perme, staggercode, nsize, ghostsize,
+      extras = PUGH_SetupPGExtras (0, dim, perme, nsize, ghostsize,
                                    newGH->nprocs, connectivity->nprocs, 
                                    connectivity->neighbours, newGH->myproc);
 
@@ -219,7 +212,6 @@ static int PUGH_SetupGAGroup (pGH *newGH,
                               newGH->narrays,
                               var_size,
                               vtype,
-                              staggercode,
                               vectorlength,
                               variable%vectorlength,
                               variable%vectorlength > 0 ? newGH->variables[newGH->nvariables - variable%vectorlength][level] : NULL);
@@ -292,11 +284,6 @@ static int PUGH_SetupGAGroup (pGH *newGH,
    @vtype      int
    @vio        in
    @endvar
-   @var        staggercode
-   @vdesc      stagger code for variables in this group
-   @vtype      int
-   @vio        in
-   @endvar
    @var        n_timelevels
    @vdesc      number of timelevels in this group
    @vtype      int
@@ -322,7 +309,6 @@ int PUGH_SetupGroup (pGH *newGH,
                      int  dim,
                      int  n_variables,
                      int  vectorlength,
-                     int  staggercode,
                      int  n_timelevels,
                      int  vectorgroup)
 {
@@ -331,7 +317,7 @@ int PUGH_SetupGroup (pGH *newGH,
 
   (void) (vectorgroup + 0);
   retval = PUGH_SetupGAGroup (newGH, nsize, nghostsize, gtype, vtype, dim,
-                              n_variables, vectorlength, staggercode,
+                              n_variables, vectorlength,
                               n_timelevels);
 
   return (retval);
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index cbccf71..5e81705 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -8,7 +8,7 @@
    @version   $Id$
  @@*/
 
-/*#define DEBUG_PUGH*/
+/* #define DEBUG_PUGH 1 */
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -84,11 +84,6 @@ static int topology_3d_z = 0;
    @vtype      int *
    @vio        in
    @endvar
-   @var        staggertype
-   @vdesc      staggering type for variables on this pGH
-   @vtype      int
-   @vio        in
-   @endvar
    @var        perme
    @vdesc      periodic boundary flags for every dimension
    @vtype      int *
@@ -104,7 +99,6 @@ pGH *PUGH_SetupPGH (void *callerid,
                     int dim,
                     int *nsize,
                     int *nghostzones,
-                    int staggertype,
                     int *perme)
 {
   DECLARE_CCTK_PARAMETERS
@@ -174,7 +168,7 @@ pGH *PUGH_SetupPGH (void *callerid,
       free(nprocs);
 
       pughGH->GFExtras[idim-1] =
-        PUGH_SetupPGExtras (1, idim, perme, staggertype,
+        PUGH_SetupPGExtras (1, idim, perme,
                             nsize, nghostzones,
                             pughGH->nprocs,
                             pughGH->Connectivity[idim-1]->nprocs,
@@ -652,40 +646,31 @@ int PUGH_nProcs (const cGH *GH)
 #if 0
 void pGH_DumpInfo(pGH *GH)
 {
-  int i,j,k;
+  int j,k;
   printf("INFO: \n");
   printf("   myproc: %d/%d \n",GH->myproc,GH->nprocs);
   printf("   ownership: \n");
-  for (i=0;i<PUGH_NSTAGGER;i++)
+  for (j=0;j<GH->dim;j++)
   {
-    for (j=0;j<GH->dim;j++)
-    {
-      printf("     GH->ownership[%d][0/1][%d]: %d %d \n",
-             i,j,GH->GFExtras[GH->dim-1]->ownership[i][0][j],GH->GFExtras[GH->dim-1]->ownership[i][1][j]);
-    }
+    printf("     GH->ownership[0/1][%d]: %d %d \n",
+           j,GH->GFExtras[GH->dim-1]->ownership[0][j],GH->GFExtras[GH->dim-1]->ownership[1][j]);
   }
 
-  for (i=0;i<PUGH_NSTAGGER;i++)
+  for (j=0;j<GH->dim;j++)
   {
-    for (j=0;j<GH->dim;j++)
+    for (k=0;k<2*GH->dim;k++)
     {
-      for (k=0;k<2*GH->dim;k++)
-      {
-        printf("     GH->GFExtras[GH->dim-1]->ghosts[%d][0/1][%d][%d]: %d %d \n",
-               i,k,j,GH->GFExtras[GH->dim-1]->ghosts[i][0][k][j],GH->GFExtras[GH->dim-1]->ghosts[i][1][k][j]);
-      }
+      printf("     GH->GFExtras[GH->dim-1]->ghosts[0/1][%d][%d]: %d %d \n",
+             k,j,GH->GFExtras[GH->dim-1]->ghosts[0][k][j],GH->GFExtras[GH->dim-1]->ghosts[1][k][j]);
     }
   }
 
-  for (i=0;i<PUGH_NSTAGGER;i++)
+  for (j=0;j<GH->dim;j++)
   {
-    for (j=0;j<GH->dim;j++)
+    for (k=0;k<2*GH->dim;k++)
     {
-      for (k=0;k<2*GH->dim;k++)
-      {
-        printf("     GH->GFExtras[GH->dim-1]->overlap[%d][0/1][%d][%d]: %d %d \n",
-               i,k,j,GH->GFExtras[GH->dim-1]->overlap[i][0][j][k],GH->GFExtras[GH->dim-1]->overlap[i][1][k][j]);
-      }
+      printf("     GH->GFExtras[GH->dim-1]->overlap[0/1][%d][%d]: %d %d \n",
+             k,j,GH->GFExtras[GH->dim-1]->overlap[0][j][k],GH->GFExtras[GH->dim-1]->overlap[1][k][j]);
     }
   }
 }
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 2375f2d..0828aa7 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -8,7 +8,7 @@
    @version $Header$
  @@*/
 
-/*#define DEBUG_PUGH*/
+/* #define DEBUG_PUGH 1 */
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -43,10 +43,8 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
                                        int nprocs,
                                        int **neighbours,
                                        pGExtras *this);
-static int PUGH_SetupPGExtrasStaggering(int dim, int this_proc, pGExtras *this);
 static int PUGH_SetupBoundingBox(int is_gf,
                                  int dim,
-                                 int stagger,
                                  int *nghosts,
                                  int total_procs,
                                  int *nprocs,
@@ -54,7 +52,6 @@ static int PUGH_SetupBoundingBox(int is_gf,
 static int PUGH_SetupRemoteSizes(int dim, int total_procs, pGExtras *this);
 static int PUGH_SetupPGExtrasSizes(int is_gf,
                                    int dim,
-                                   int stagger,
                                    int *sh,
                                    int *nghosts,
                                    int total_procs,
@@ -87,7 +84,6 @@ static int PUGH_DecomposeIJK(int dim, int ijk, int *nprocs, int *pos);
 pGExtras *PUGH_SetupPGExtras(int is_gf,
                              int dim,
                              int *perme,
-                             int stagger,
                              int *sh,
                              int *nghosts,
                              int total_procs,
@@ -106,11 +102,10 @@ pGExtras *PUGH_SetupPGExtras(int is_gf,
     {
       this->dim = dim;
 
-      PUGH_SetupPGExtrasSizes(is_gf, dim, stagger, sh, nghosts,
+      PUGH_SetupPGExtrasSizes(is_gf, dim, sh, nghosts,
                               total_procs, nprocs, this_proc,this);
       PUGH_SetupPGExtrasOwnership(dim, perme, this_proc, total_procs,
                                   neighbours, this);
-      PUGH_SetupPGExtrasStaggering(dim, this_proc, this);
     }
   }
 
@@ -129,21 +124,18 @@ void PUGH_DestroyPGExtras(pGExtras **PGExtras)
 {
   if(PGExtras && *PGExtras)
   {
-    int i, j;
+    int j;
 
-    for(i = 0 ; i < PUGH_NSTAGGER; i++)
+    for(j = 0; j < 2; j++)
     {
-      for(j = 0; j < 2; j++)
+      if((*PGExtras)->dim > 0)
       {
-        if((*PGExtras)->dim > 0)
-        {
-          free((*PGExtras)->ghosts[i][j][0]);
-          free((*PGExtras)->overlap[i][j][0]);
-        }
-        free((*PGExtras)->ownership[i][j]);
-        free((*PGExtras)->ghosts[i][j]);
-        free((*PGExtras)->overlap[i][j]);
+        free((*PGExtras)->ghosts[j][0]);
+        free((*PGExtras)->overlap[j][0]);
       }
+      free((*PGExtras)->ownership[j]);
+      free((*PGExtras)->ghosts[j]);
+      free((*PGExtras)->overlap[j]);
     }
     free((*PGExtras)->lb[0]);
     free((*PGExtras)->ub[0]);
@@ -457,7 +449,7 @@ static int PUGH_ComposeIJK(int dim, int *nprocs, int *pos)
 static int PUGH_SetupPGExtrasMemory(int dim, int total_procs, pGExtras *this)
 {
   int retcode;
-  int i,j,k;
+  int j,k;
 
   retcode = 0;
 
@@ -503,6 +495,7 @@ static int PUGH_SetupPGExtrasMemory(int dim, int total_procs, pGExtras *this)
            this->ub[0]     &&
            this->rnsize[0])
         {
+          int i;
           for (i = 1; i < total_procs; i++)
           {
             this->lb[i]     = this->lb[0]     + i*dim;
@@ -581,53 +574,50 @@ static int PUGH_SetupPGExtrasMemory(int dim, int total_procs, pGExtras *this)
        (dim == 0 || this->hyper_volume))
     {
       retcode = 0;
-      for (i = 0 ; i < PUGH_NSTAGGER; i++)
+      for (j = 0; j < 2; j++)
       {
-        for (j = 0; j < 2; j++)
+        this->ownership[j]    = malloc(dim*sizeof(int));
+        this->ghosts[j]       = malloc(2*dim*sizeof(int *));
+        this->overlap[j]      = malloc(2*dim*sizeof(int *));
+        if(dim > 0)
         {
-          this->ownership[i][j]    = malloc(dim*sizeof(int));
-          this->ghosts[i][j]       = malloc(2*dim*sizeof(int *));
-          this->overlap[i][j]      = malloc(2*dim*sizeof(int *));
-          if(dim > 0)
+          if(this->ghosts[j] &&
+             this->overlap[j])
           {
-            if(this->ghosts[i][j] &&
-               this->overlap[i][j])
+            this->ghosts[j][0]  = malloc(2*dim*dim*sizeof(int));
+            this->overlap[j][0] = malloc(2*dim*dim*sizeof(int));
+            for (k=1; k < 2*dim; k++)
             {
-              this->ghosts[i][j][0]  = malloc(2*dim*dim*sizeof(int));
-              this->overlap[i][j][0] = malloc(2*dim*dim*sizeof(int));
-              for (k=1; k < 2*dim; k++)
-              {
-                this->ghosts[i][j][k]  = this->ghosts[i][j][0] + k*dim;
-                this->overlap[i][j][k] = this->overlap[i][j][0] + k*dim;
-              }
+              this->ghosts[j][k]  = this->ghosts[j][0] + k*dim;
+              this->overlap[j][k] = this->overlap[j][0] + k*dim;
             }
-            else
-            {
-              free(this->ownership[i][j]);
-              this->ownership[i][j] = NULL;
+          }
+          else
+          {
+            free(this->ownership[j]);
+            this->ownership[j] = NULL;
 
-              free(this->ghosts[i][j]);
-              this->ghosts[i][j] = NULL;
+            free(this->ghosts[j]);
+            this->ghosts[j] = NULL;
 
-              free(this->overlap[i][j]);
-              this->overlap[i][j] = NULL;
-              retcode = 1;
-              break;
-            }
+            free(this->overlap[j]);
+            this->overlap[j] = NULL;
+            retcode = 1;
+            break;
           }
         }
         if(retcode)
         {
           for(j=1; j >=0 ; j--)
           {
-            free(this->ownership[i][j]);
-            this->ownership[i][j] = NULL;
+            free(this->ownership[j]);
+            this->ownership[j] = NULL;
 
-            free(this->ghosts[i][j]);
-            this->ghosts[i][j] = NULL;
+            free(this->ghosts[j]);
+            this->ghosts[j] = NULL;
 
-            free(this->overlap[i][j]);
-            this->overlap[i][j] = NULL;
+            free(this->overlap[j]);
+            this->overlap[j] = NULL;
           }
           break;
         }
@@ -635,31 +625,28 @@ static int PUGH_SetupPGExtrasMemory(int dim, int total_procs, pGExtras *this)
       if(retcode)
       {
         /* Loop back through the arrays freeing things */
-        for(i--; i >=0; i--)
+        for(j=1; j >=0 ; j--)
         {
-          for(j=1; j >=0 ; j--)
+          if(dim > 0)
           {
-            if(dim > 0)
-            {
-              free(this->ghosts[i][j][0]);
-              free(this->overlap[i][j][0]);
-            }
+            free(this->ghosts[j][0]);
+            free(this->overlap[j][0]);
+          }
 
-            free(this->ownership[i][j]);
-            this->ownership[i][j] = NULL;
+          free(this->ownership[j]);
+          this->ownership[j] = NULL;
 
-            free(this->ghosts[i][j]);
-            this->ghosts[i][j] = NULL;
+          free(this->ghosts[j]);
+          this->ghosts[j] = NULL;
 
-            free(this->overlap[i][j]);
-            this->overlap[i][j] = NULL;
-          }
-
-          free(this->ownership[i]);
-          free(this->ghosts[i]);
-          free(this->overlap[i]);
+          free(this->overlap[j]);
+          this->overlap[j] = NULL;
         }
 
+        free(this->ownership);
+        free(this->ghosts);
+        free(this->overlap);
+
         /* Free the stuff originally allocated */
 
         free(this->lb[0]);
@@ -711,7 +698,6 @@ static int PUGH_SetupPGExtrasMemory(int dim, int total_procs, pGExtras *this)
 @@*/
 static int PUGH_SetupPGExtrasSizes(int is_gf,
                                    int dim,
-                                   int stagger,
                                    int *sh,
                                    int *nghosts,
                                    int total_procs,
@@ -739,11 +725,6 @@ static int PUGH_SetupPGExtrasSizes(int is_gf,
       {
         this->nsize[dir] = nprocs[dir] * (-sh[dir]);
       }
-
-      if (stagger == PUGH_STAGGER)
-      {
-        this->nsize[dir] -= nprocs[dir]-1;
-      }
     }
     else
     {
@@ -758,7 +739,7 @@ static int PUGH_SetupPGExtrasSizes(int is_gf,
   }
 
   /* Setup the bounding box stuff */
-  PUGH_SetupBoundingBox(is_gf, dim, stagger, nghosts, total_procs, nprocs,this);
+  PUGH_SetupBoundingBox(is_gf, dim, nghosts, total_procs, nprocs,this);
 
   /* Set the remote sizes */
   PUGH_SetupRemoteSizes(dim, total_procs, this);
@@ -818,25 +799,24 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
 
 
   /* Ownership is pretty easy. Remember ownership is indexed as
-     [stagger][ijk][min/max]. See pGF_Reduction for a use of this,
+     [ijk][min/max]. See pGF_Reduction for a use of this,
      among others.
-     Note: Ownership is same for staggered and non-staggered grids
    */
   for (dir = 0 ; dir < dim; dir++)
   {
     /* Setup ownership and bbox using neighbours. */
     if (neighbours[this_proc][2*dir] == -1
         || neighbours[this_proc][2*dir+1] == -1) {
-      this->ownership[PUGH_VERTEXCTR][0][dir] =
+      this->ownership[0][dir] =
         (this->lb[this_proc][dir] == 0 ? 0 : this->nghostzones[dir]);
-      this->ownership[PUGH_VERTEXCTR][1][dir] =
+      this->ownership[1][dir] =
         (this->ub[this_proc][dir] == this->nsize[dir]-1 ?
          this->lnsize[dir] : this->lnsize[dir] - this->nghostzones[dir]);
       this->bbox[2*dir+0] = this->lb[this_proc][dir] == 0;
       this->bbox[2*dir+1] = this->ub[this_proc][dir] == this->nsize[dir]-1;
     } else {
-      this->ownership[PUGH_VERTEXCTR][0][dir] = this->nghostzones[dir];
-      this->ownership[PUGH_VERTEXCTR][1][dir] =
+      this->ownership[0][dir] = this->nghostzones[dir];
+      this->ownership[1][dir] =
         this->lnsize[dir] - this->nghostzones[dir];
       this->bbox[2*dir+0] = 0;
       this->bbox[2*dir+1] = 0;
@@ -849,8 +829,8 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
 	= (this->lb[this_proc][dir] == 0
 	   && this->ub[this_proc][dir] == this->nsize[dir]-1);
       if (! is_single_proc_in_dim
-          && this->ownership[PUGH_VERTEXCTR][1][dir] -
-             this->ownership[PUGH_VERTEXCTR][0][dir] < this->nghostzones[dir])
+          && this->ownership[1][dir] -
+             this->ownership[0][dir] < this->nghostzones[dir])
       {
         CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Not enough interior grid points in dimension %d ! "
@@ -869,8 +849,8 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
   {
     if (this->nsize[dir] == 1)
     {
-      this->ownership[PUGH_VERTEXCTR][0][dir] = 0;
-      this->ownership[PUGH_VERTEXCTR][1][dir] = 1;
+      this->ownership[0][dir] = 0;
+      this->ownership[1][dir] = 1;
     }
   }
 
@@ -904,8 +884,8 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
         iend   = tmp -     this->nghostzones[idir];
       }
 
-      this->overlap[PUGH_VERTEXCTR][0][dir][idir] = istart;
-      this->overlap[PUGH_VERTEXCTR][1][dir][idir] = iend;
+      this->overlap[0][dir][idir] = istart;
+      this->overlap[1][dir][idir] = iend;
     }
   }
 
@@ -938,69 +918,14 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
         iend   = tmp;
       }
 
-      this->ghosts[PUGH_VERTEXCTR][0][dir][idir] = istart;
-      this->ghosts[PUGH_VERTEXCTR][1][dir][idir] = iend;
+      this->ghosts[0][dir][idir] = istart;
+      this->ghosts[1][dir][idir] = iend;
     }
   }
 
   return 0;
 }
 
-static int PUGH_SetupPGExtrasStaggering(int dim, int this_proc, pGExtras *this)
-{
-  int s,d,k,dir;
-  int *upperbnd;
-
-  upperbnd = malloc(dim*sizeof(int));
-
-  for (d=0;d<dim;d++)
-  {
-    upperbnd[d] = (this->ub[this_proc][d] == this->nsize[d]-1) ? 1 : 0;
-  }
-
-  /* copy ownership/ghost/overlap from the default PUGH_VERTEXCTR */
-  for(s=1;s<PUGH_NSTAGGER;s++)
-  {
-    for(d=0;d<dim;d++)
-    {
-      for(k=0;k<2;k++)
-      {
-        this->ownership[s][k][d]    = this->ownership[0][k][d];
-        for (dir=0;dir<2*dim;dir++)
-        {
-          this->overlap[s][k][dir][d] = this->overlap[0][k][dir][d];
-          this->ghosts [s][k][dir][d] = this->ghosts [0][k][dir][d];
-        }
-      }
-
-      /* decrease ownership at upper bound */
-      if(upperbnd[d])
-      {
-        this->ownership[s][1][d] --;
-      }
-    }
-
-    /* correct overlaps */
-    for(dir=0;dir<2*dim;dir++)
-    {
-      for (d=0;d<dim;d++)
-      {
-        /* if we are at a upper physical bnd && at a upperface (1,3,5)
-           && we are in the dir direction decrease by one */
-        if((upperbnd[d])&&((dir+1)%2==0)&&(dir==2*d+1))
-        {
-          this->overlap[s][0][dir][d]--;
-          this->overlap[s][1][dir][d]--;
-        }
-      }
-    }
-  }
-
-  free(upperbnd);
-
-  return(0);
-}
-
  /*@@
    @routine    PUGH_SetupBoundingBox
    @date       Mon Nov  8 09:03:40 1999
@@ -1011,7 +936,6 @@ static int PUGH_SetupPGExtrasStaggering(int dim, int this_proc, pGExtras *this)
 @@*/
 static int PUGH_SetupBoundingBox(int is_gf,
                                  int dim,
-                                 int stagger,
                                  int *nghosts,
                                  int total_procs,
                                  int *nprocs,
@@ -1051,10 +975,6 @@ static int PUGH_SetupBoundingBox(int is_gf,
         else
         {
           this->lb[pnum][dir] = bounds[dir][pos[dir]] +1 - nghosts[dir];
-          if(stagger == PUGH_STAGGER)
-          {
-            this->lb[pnum][dir] --;
-          }
         }
 
         if (pos[dir] == nprocs[dir]-1)
@@ -1300,12 +1220,10 @@ pGA *PUGH_SetupGArray(void *parent,
                       int arrayid,
                       int varsize,
                       int vtype,
-                      int staggercode,
                       int vector_size,
                       int vector_entry,
                       pGA *vector_base)
 {
-  int dim;
   pGA *this;
 
 
@@ -1319,11 +1237,6 @@ pGA *PUGH_SetupGArray(void *parent,
     this->parent       = parent;
     this->varsize      = varsize;
     this->vtype        = vtype;
-    this->stagger      = calloc (this->extras->dim, sizeof (int));
-    for (dim = 0; dim < this->extras->dim; dim++)
-    {
-      this->stagger[dim] = CCTK_StaggerDirIndex (dim, staggercode);
-    }
     this->id           = id;
     this->arrayid      = arrayid;
     this->storage      = PUGH_NOSTORAGE;
diff --git a/src/Storage.c b/src/Storage.c
index 1386284..cbdf47d 100644
--- a/src/Storage.c
+++ b/src/Storage.c
@@ -8,6 +8,9 @@
    @version $Id$
  @@*/
 
+/* #define DEBUG_PUGH 1 */
+
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -18,11 +21,9 @@
 #include "pughi.h"
 #include "pugh_Comm.h"
 
-static const char *rcsid="$Header$";
+static const char *rcsid="$Header:$";
 CCTK_FILEVERSION(CactusPUGH_PUGH_Storage_c);
 
-/* #define DEBUG_PUGH 1 */
-
 
 /********************************************************************
  ********************    Static Variables   *************************
@@ -205,7 +206,6 @@ int PUGH_QueryGroupStorage (const cGH *GH, int group, const char *groupname)
 {
   int first_var;
   int storage;
-  int grouptype;
   int retval;
   pGH *pughGH;
 
@@ -222,7 +222,6 @@ int PUGH_QueryGroupStorage (const cGH *GH, int group, const char *groupname)
   if (first_var >= 0)
   {
     pughGH = PUGH_pGH (GH);
-    grouptype = CCTK_GroupTypeI (group);
     storage = ((pGA *) pughGH->variables[first_var][0])->storage;
 
     if (storage == PUGH_STORAGE)
diff --git a/src/include/pGH.h b/src/include/pGH.h
index f21db09..d8ae113 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -61,19 +61,19 @@ typedef struct PGH
 
 #ifdef PUGH_WITH_DERIVED_DATATYPES
   /* Derived data types for communication */
-  MPI_Datatype send_char_dt [PUGH_NSTAGGER][6];
-  MPI_Datatype recv_char_dt [PUGH_NSTAGGER][6];
-  MPI_Datatype send_int_dt  [PUGH_NSTAGGER][6];
-  MPI_Datatype recv_int_dt  [PUGH_NSTAGGER][6];
-  MPI_Datatype send_real_dt [PUGH_NSTAGGER][6];
-  MPI_Datatype recv_real_dt [PUGH_NSTAGGER][6];
-  MPI_Datatype send_complex_dt [PUGH_NSTAGGER][6];
-  MPI_Datatype recv_complex_dt [PUGH_NSTAGGER][6];
+  MPI_Datatype send_char_dt [6];
+  MPI_Datatype recv_char_dt [6];
+  MPI_Datatype send_int_dt  [6];
+  MPI_Datatype recv_int_dt  [6];
+  MPI_Datatype send_real_dt [6];
+  MPI_Datatype recv_real_dt [6];
+  MPI_Datatype send_complex_dt [6];
+  MPI_Datatype recv_complex_dt [6];
 #endif
 #endif
 
   /* Used for all grid functions */
-  pGExtras **GFExtras;          /* [dim] stagger ? */
+  pGExtras **GFExtras;          /* [dim] */
   pConnectivity **Connectivity; /* [dim] */
 
   char *identity_string;        /* identifier for this pGH */
diff --git a/src/include/pGV.h b/src/include/pGV.h
index 8219847..fad2fa3 100644
--- a/src/include/pGV.h
+++ b/src/include/pGV.h
@@ -51,17 +51,14 @@ typedef struct PGExtras
 
   /* Ghosts and overlaps. */
   int      *nghostzones;        /* Width of ghost zone */
-  int      *ownership[PUGH_NSTAGGER][2];        
-                                /* The box owned in each direction. */
-                                /* [stagger][min/max][dir] */
+  int      *ownership[2];       /* The box owned in each direction. */
+                                /* [min/max][dir] */
                                 
-  int     **ghosts[PUGH_NSTAGGER][2];   
-                                /* The ghost zones on each face. */
-                                /* [stagger][min/max][face][dir] */
+  int     **ghosts[2];          /* The ghost zones on each face. */
+                                /* [min/max][face][dir] */
                                 
-  int     **overlap[PUGH_NSTAGGER][2];  
-                                /* The overlap region owned on each face. */
-                                /* [stagger][min/max][face][ijk] */
+  int     **overlap[2];         /* The overlap region owned on each face. */
+                                /* [min/max][face][ijk] */
 
 } pGExtras;
 
@@ -96,7 +93,6 @@ typedef struct PGA
   void  *data;               /* See the note above. */
   int   storage;             /* Do we have storage or not? */
   int   npoints;             /* Number of points (for memory statistics) */
-  int   *stagger;            /* [dim] directional stagger indicies */
 
   void  *parent;             /* The GH to which I belong */
                              /* Note this is struct PGH which is
diff --git a/src/include/pugh.h b/src/include/pugh.h
index 4ca23f0..8c528af 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -131,7 +131,6 @@ int PUGH_SetupGroup (pGH *newGH,
                      int  dim,
                      int  n_variables,
                      int  vectorlength,
-                     int  staggercode,
                      int  n_timelevels,
                      int  vectorgroup);
 
@@ -139,7 +138,6 @@ pGH *PUGH_SetupPGH(void *callerid,
                    int dim,
                    int *nsize,
                    int *nghostzones,
-                   int staggertype,
                    int *perme);
 
 void PUGH_GFSize(int dim, int *nsize);
diff --git a/src/include/pugh_constants.h b/src/include/pugh_constants.h
index d9a919f..1b1fbba 100644
--- a/src/include/pugh_constants.h
+++ b/src/include/pugh_constants.h
@@ -20,20 +20,6 @@
 #define PUGH_NOSTORAGE        0
 #define PUGH_STORAGE          1
 
-/* Constants for the GF Type. Only one now ... */
-/* Number of staggerings available in the code */
-#define PUGH_NSTAGGER 4
-
-/* The index for each staggering type */
-/* This classifies each PGF member */
-#define PUGH_VERTEXCTR 0
-#define PUGH_FACECTR(i) i+1
-
-/* The index for each staggering type */
-/* This determines the PGH should be set up */
-#define PUGH_NO_STAGGER 0
-#define PUGH_STAGGER 1
-
 /* Comm types available */
 #define PUGH_NOCOMMBUFFERS    0
 #define PUGH_ALLOCATEDBUFFERS 1
diff --git a/src/include/pughi.h b/src/include/pughi.h
index a37b85a..b934690 100644
--- a/src/include/pughi.h
+++ b/src/include/pughi.h
@@ -34,7 +34,6 @@ pConnectivity *PUGH_SetupConnectivity(int dim,
 pGExtras *PUGH_SetupPGExtras(int is_gf,
                              int dim, 
                              int *perme,
-                             int stagger,
                              int *sh,
                              int *nghosts,
                              int total_procs,
@@ -73,7 +72,6 @@ pGA *PUGH_SetupGArray(void *parent,
                       int arrayid,
                       int varsize, 
                       int vtype, 
-                      int stagger,
                       int vector_size,
                       int vector_entry,
                       pGA *vector_base);