Major changes to PUGH as the dearly loved pGF is removed, and a grid function

is lowered to being simply a special case of a grid array, pGA. Grid arrays
can all have different sizes, which means that now the size of a GA is moved
from the GH to a structure on the GA, along with a lot of other connectivity
and additional information which was formerly on the GH.

If you need to care about these changes, take a look at the files in
src/include/pGH.h and src/include/pGA.h, to see the new structures
(I'll leave the old files pGF.h and SetupPGF.c around for a while to compare with).
Also, look at a PUGH dependent file which has already been converted, such
as CactusPUGHIO/IOASCII/src/Write1D.c.



Warning:

This commit breaks the development versions of IOHDF5 and IsoSurfacer, these
will be fixed real quick.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@182 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 52 insertions, 501 deletions

diff --git a/src/Comm.c b/src/Comm.c
index 5830219..47870bb 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -12,7 +12,7 @@
 
 #include <stdlib.h>
 #include <stdio.h>
-#include <stdarg.h>
+#include <stdarg.h> 
 #include <string.h>
 #include <assert.h>
 #include <unistd.h>
@@ -29,33 +29,16 @@ extern int PUGH_GHExtension;
 
 /* local function prototypes */
 
-int pugh_SyncGroupGF (cGH *GH, 
-                      int group, 
-                      int vtype, 
-                      int n_vars, 
-                      int timelevel);
 int pugh_SyncGroupGA (cGH *GH, 
                       int group, 
                       int vtype, 
                       int n_vars, 
                       int timelevel);
-int pugh_SyncGFs(pGH *pughGH, 
-		 int first_var, 
-		 int n_vars, 
-		 int timelevel);
 int pugh_SyncGAs(pGH *pughGH, 
                  int dim, 
                  int first_var, 
                  int n_vars, 
                  int timelevel);
-void DisableGFDataStorage(pGH *GH, pGF *GF);
-void EnableGFDataStorage(pGH *GH, pGF *GF);
-int pugh_DisableGFGroupStorage(cGH *GH,                              
-                               int group,
-                               int vtype,
-                               int dim,
-                               int n_variables,
-                               int n_timelevels);
 int pugh_DisableArrayGroupStorage(cGH *GH,                              
                                  int group,
                                  int vtype,
@@ -68,12 +51,6 @@ int pugh_EnableArrayGroupStorage(cGH *GH,
                                  int dim,
                                  int n_variables,
                                  int n_timelevels);
-int pugh_EnableGFGroupStorage(cGH *GH,                              
-                              int group,
-                              int vtype,
-                              int dim,
-                              int n_variables,
-                              int n_timelevels);
 int pugh_EnablePGAStorage(pGA *GA, 
                           int this_proc,
                           int zero_memory, 
@@ -81,13 +58,9 @@ int pugh_EnablePGAStorage(pGA *GA,
                           int padding_cacheline_bits,
                           int padding_size,
                           int padding_address_spacing);
-void pGF_FinishRecv(pGH *GH, pGF *GF, int dir);
-void pGF_PostRecv(pGH *GH, pGF *GF, int dir);
-void pGF_PostSend(pGH *GH, pGF *GF, int dir);
 void FinishReceiveGA(pGH *GH, pGA *GA, int dir);
 void PostReceiveGA(pGH *GH, pGA *GA, int dir);
 void PostSendGA(pGH *GH, pGA *GA, int dir);
-void SetGFComm(pGH *GH, pGF *res, int docomm);
 int pugh_GAComm(pGA *GA, int docomm);
 
 /*@@
@@ -154,7 +127,7 @@ int pugh_SyncGroup (cGH *GH, const char *groupname)
       break;
 
     case GROUP_GF     :
-      rc = pugh_SyncGroupGF (GH, 
+      rc = pugh_SyncGroupGA (GH, 
                              group, 
                              pgroup.vartype, 
                              pgroup.numvars, 
@@ -209,7 +182,6 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
   int first;
   int *sizep=NULL;
   pGH *pughGH;
-  pGF *pughGF;
   pGA *pughGA;
 
   if (groupname)
@@ -226,20 +198,20 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
   if (CCTK_GroupTypeI(group)==GROUP_GF)
   {
     /* get pointer to pGF for a timelevel of this variable */
-    pughGF = (pGF *)(pughGH->variables[first][0]);  
+    pughGA = (pGA *)(pughGH->variables[first][0]);  
 
-    if (pughGF->storage == PUGH_STORAGE)
+    if (pughGA->storage == PUGH_STORAGE)
     {
       switch (dir) 
       {
         case 0:
-          sizep = &pughGH->lnsize[0];
+          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[0];
           break;
         case 1:
-          sizep = &pughGH->lnsize[1];
+          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[1];
           break;
         case 2:
-          sizep = &pughGH->lnsize[2];
+          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[2];
           break;
         default :
           sizep = NULL;
@@ -254,7 +226,7 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
   }
   else if (CCTK_GroupTypeI(group)==GROUP_ARRAY)
   {
-    /* get pointer to pGF for a timelevel of this variable */
+    /* get pointer to pGA for a timelevel of this variable */
     pughGA = (pGA *)(pughGH->variables[first][0]);  
     
     if (pughGA->storage == PUGH_STORAGE)
@@ -281,7 +253,7 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
       sizep = &_cctk_one;
     }
   }
-  
+
   return (sizep);
 }
 
@@ -352,7 +324,7 @@ int pugh_QueryGroupStorage(cGH *GH, int group, const char *groupname)
   }
   else if (grouptype == CCTK_GF)
   {
-    storage = ((pGF *)(pughGH->variables[first][0]))->storage;  
+    storage = ((pGA *)(pughGH->variables[first][0]))->storage;  
   }
   else if (grouptype == CCTK_ARRAY)
   {
@@ -389,7 +361,7 @@ int pugh_QueryGroupStorage(cGH *GH, int group, const char *groupname)
   Enables storage for all variables in the group indicated by groupname.
   @enddesc
   @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-           pugh_EnableArrayGroupStorage pugh_EnableGFGroupStorage
+           pugh_EnableArrayGroupStorage
   @history
  
   @endhistory 
@@ -411,7 +383,6 @@ int pugh_EnableGroupStorage(cGH *GH, const char *groupname)
   cGroup pgroup;       /* pointer to group information */
   int rc;                  /* return code */
 
-
 #ifdef DEBUG_PUGH
   printf(" pugh_EnableGroupStorage: request for group -%s-\n",groupname);
   fflush(stdout);
@@ -439,12 +410,12 @@ int pugh_EnableGroupStorage(cGH *GH, const char *groupname)
       break;
 
     case CCTK_GF     :
-      rc = pugh_EnableGFGroupStorage (GH, 
-                                      group, 
-                                      pgroup.vartype, 
-                                      pgroup.dim, 
-                                      pgroup.numvars,
-                                      pgroup.numtimelevels);
+      rc = pugh_EnableArrayGroupStorage (GH, 
+					 group, 
+					 pgroup.vartype, 
+					 pgroup.dim, 
+					 pgroup.numvars,
+					 pgroup.numtimelevels);
       break;
 
     default :
@@ -465,7 +436,7 @@ int pugh_EnableGroupStorage(cGH *GH, const char *groupname)
   Disables storage for all variables in the group indicated by groupname.
   @enddesc
   @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-           pugh_DisableArrayGroupStorage pugh_DisableGFGroupStorage
+           pugh_DisableArrayGroupStorage
   @history
  
   @endhistory 
@@ -486,7 +457,13 @@ int pugh_DisableGroupStorage(cGH *GH, const char *groupname)
   int group;               /* group index */
   cGroup pgroup;        /* pointer to group information */
   int rc;                  /* return code */
+  pGH *pughGH;             /* PUGH extension reference */
+  int level;
+  int first_var,var;
 
+  /* get PUGH extension handle */
+  pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
+  
 #ifdef DEBUG_PUGH
   printf(" pugh_DisableGroupStorage: request for group -%s-\n",groupname);
   fflush(stdout);
@@ -494,6 +471,9 @@ int pugh_DisableGroupStorage(cGH *GH, const char *groupname)
 
   group = CCTK_GroupIndex(groupname);
 
+  /* get global index of first variable in group */
+  first_var = CCTK_FirstVarIndexI (group);
+
   /* get the group info from its index */
   CCTK_GroupData (group,&pgroup);
 
@@ -504,21 +484,25 @@ int pugh_DisableGroupStorage(cGH *GH, const char *groupname)
       break;
 
     case GROUP_ARRAY  :
-      rc = pugh_DisableArrayGroupStorage (GH, 
-                                          group, 
-                                          pgroup.vartype, 
-                                          pgroup.dim, 
-                                          pgroup.numvars,
-                                          pgroup.numtimelevels);
+      for (var = first_var; var < first_var+pgroup.numvars; var++)
+      {
+        for (level = 0; level < pgroup.numtimelevels; level++)
+        {
+          pugh_DisableGADataStorage ((pGA *)(pughGH->variables[var][level]));
+        }
+      }
+      rc = 1;
       break;
 
     case GROUP_GF     :
-      rc = pugh_DisableGFGroupStorage (GH, 
-                                       group, 
-                                       pgroup.vartype, 
-                                       pgroup.dim, 
-                                       pgroup.numvars,
-                                       pgroup.numtimelevels);
+      for (var = first_var; var < first_var+pgroup.numvars; var++)
+      {
+        for (level = 0; level < pgroup.numtimelevels; level++)
+        {
+          pugh_DisableGADataStorage ((pGA *)(pughGH->variables[var][level]));
+        }
+      }
+      rc = 1;
       break;
 
     default :
@@ -598,8 +582,8 @@ int pugh_EnableGroupComm (cGH *GH, const char *groupname)
       {
         for (level = 0; level < pgroup.numtimelevels; level++)
         {
-          SetGFComm (pughGH, (pGF *)(pughGH->variables[var][level]), 
-                     PUGH_ALLCOMM);
+          pugh_GAComm ((pGA *)(pughGH->variables[var][level]), 
+                             PUGH_ALLCOMM);
         }
       }
       rc = 1;
@@ -680,7 +664,10 @@ int pugh_DisableGroupComm(cGH *GH, const char *groupname)
     case GROUP_GF     :
       for (var = first_var; var < first_var+pgroup.numvars; var++)
         for (level = 0; level < pgroup.numtimelevels; level++)
-          SetGFComm (pughGH, (pGF *)(pughGH->variables[var][level]), PUGH_NOCOMM);
+        {
+          pugh_GAComm ((pGA *)(pughGH->variables[var][level]), 
+                             PUGH_NOCOMM);
+        }
       rc = 1;
       break;
 
@@ -768,35 +755,12 @@ int pugh_SyncGroupGA (cGH *GH,
 {
 
   pGH *pughGH;
-  pGA *GA;
-  int i;
-  int first_var;
 
   pughGH = pugh_pGH(GH);
-  first_var = CCTK_FirstVarIndexI (group);
-
-  /* Say which grid functions will actually be synchronised */
-  if (pughGH->sync_verbose && pughGH->myproc == 0) 
-  {
-    char *gname = CCTK_GroupName (group);
-
-    printf("Syncing %d arrays in group %s\n", n_vars, gname);
-    printf("  Syncing arrays ...");
-
-    for (i = first_var; i < first_var + n_vars; i++) 
-    {
-      GA = (pGA *) (pughGH->variables [i][timelevel]);
-      if (GA->storage && (GA->commflag != PUGH_NOCOMM || pughGH->forceSync))
-        printf (" %s", GA->name);
-    }
-    printf ("\n");
-    free (gname);
-  }
 
-  /* synchronize */
   return pugh_SyncGAs(pughGH, 
                       CCTK_GroupDimI(group),
-                      first_var, 
+                      CCTK_FirstVarIndexI(group), 
                       n_vars, 
                       timelevel);
 }
@@ -950,269 +914,6 @@ int pugh_SyncGAs(pGH *pughGH,
 }
 
 
-/*@@
-  @routine pugh_SyncGroup
-  @author Paul Walker
-  @date   March 1997
-  @desc
-  Synchronizes all the Grid Functions (which have their communications
-  on) in a given group which has previously been registered to 
-  the GF data base with StoreGFGroupData. The MPI section was
-  formally called SyncGH.c
-  @enddesc
-  @history
-  @hdate   Sep 27 1998 @hauthor Gabrielle Allen, Tom Goodale
-  @hdesc   Added the functionality to synchronise indices from a group 
-           rather than the whole GH. Changed the routine/file names.
-  @hdate   30 Mar 1999
-  @hauthor Thomas Radke
-  @hdesc   Modified for Cactus 4.0
-  @endhistory
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     group
-  @vdesc   global index of the GF group to be synchronized
-  @vtype   int
-  @vio     in
-  @endvar
-  @var     vtype
-  @vdesc   data type of GF
-  @vtype   int
-  @vio     in
-  @endvar
-  @var     n_vars
-  @vdesc   number of GF in this group
-  @vtype   int
-  @vio     in
-  @endvar
-  @var     timelevel
-  @vdesc   time level to be synchronized
-  @vtype   int
-  @vio     in
-  @endvar
-  @@*/
-
-/*  This routine works in one of two ways:
- *
- *  1. If using derived data types, it posts receives, posts sends
- *     waits for all sends and receives, and finishes the receives.
- *
- *  2. If using buffered communication, it posts recieves, posts
- *     sends, waits for receives only, and finishes them. We don't
- *     need to wait for sends until the next time through. This
- *     is why the wait for the send buffer is immediately before
- *     the post send. (Since the send buffers are initially set
- *     to MPI_REQUEST_NULL the first Wait and any waits in
- *     unused directions are a wash).
- *
- *  This routine is intertwined with the contents of routines
- *  pGF_PostRecv, pGF_PostSend, and pGF_FinishSend. 
- *
- *  Without MPI, this does nothing, of course.
- *
- *  Note this routine is also responsible for updating the
- *  commt global so we can get a measure of how much time we
- *  spend in communications overhead.
- *
- *  Also note that it is more efficient to sync a number of 
- *  GF together (as a group or a hierachy) rather than 
- *  singly. This is because syncing single functions waits for the
- *  receives on a single function before posting the sends on 
- *  all the other functions.
- */
-
-int pugh_SyncGroupGF (cGH *GH, int group, int vtype, int n_vars, int timelevel)
-{
-  pGH *pughGH;
-  pGF *GF;
-  int i;
-  int first_var;
-
-  pughGH = pugh_pGH(GH);
-  first_var = CCTK_FirstVarIndexI (group);
-
-  /* Say which grid functions will actually be synchronised */
-  if (pughGH->sync_verbose && pughGH->myproc == 0) 
-  {
-    char *gname = CCTK_GroupName (group);
-
-    printf("Syncing %d grid functions in group %s\n", n_vars, gname);
-    printf("  Syncing GFs ...");
-
-    for (i = first_var; i < first_var + n_vars; i++) 
-    {
-      GF = (pGF *) (pughGH->variables [i][timelevel]);
-      if (GF->storage && (GF->commflag != PUGH_NOCOMM || pughGH->forceSync))
-        printf (" %s", GF->name);
-    }
-    printf ("\n");
-    free (gname);
-  }
-
-  /* synchronize */
-  return pugh_SyncGFs(pughGH, first_var, n_vars, timelevel);
-}
-
-
-
-
-
-/* BB 11/99: PUGH entry point to synchronize index range of GFs
-             for backward compatibility to Cactus 3.2
-*/
-int pugh_SyncGFs(pGH *pughGH, int first_var, int n_vars, int timelevel)
-{
-#ifdef MPI
-  int Dir;
-  pGF *GF;
-#ifdef COMM_TIMING
-  double t1,t2;
-#endif
-  MPI_Request *sr = NULL;
-  MPI_Status mss;
-#endif
-
-  /* start the timer for communication time */
-#if 0
-  CactusStartTimer (&pughGH->comm_time);
-#endif
-
-#ifdef MPI
-  if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
-  {
-    int i;
-    /* 2 faces, send and receive is the 2 * 2 */
-    sr = (MPI_Request *)malloc(n_vars * 2 * 2 * sizeof(MPI_Request));
-    for (i=0;i<n_vars * 2 * 2; i++)
-      sr[i] = MPI_REQUEST_NULL;
-  }
-
-  for (Dir = 0; Dir < 3; Dir ++) 
-  {
-    int i;
-#ifdef COMM_TIMING
-    t1 = MPI_Wtime();
-#endif
-    for (i = first_var; i < first_var + n_vars; i++) 
-    {
-      GF = (pGF *) (pughGH->variables [i][timelevel]);
-      pGF_PostRecv(pughGH, GF, 2*Dir);
-      pGF_PostRecv(pughGH, GF, 2*Dir+1);
-    }
-    
-#ifdef COMM_TIMING
-    t2 = MPI_Wtime();
-    printf ("PR : %lf\n",t2-t1);
-#endif
-    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
-    {
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GF = (pGF *) (pughGH->variables [i][timelevel]);
-        /* Wait for the last send. Since these will be null if the
-           are not used, this is always safe.
-        */
-        MPI_Wait(&(GF->sreq[2*Dir]),&mss);
-        pGF_PostSend(pughGH, GF, 2*Dir);
-        MPI_Wait(&(GF->sreq[2*Dir+1]),&mss);
-        pGF_PostSend(pughGH, GF, 2*Dir+1);
-      }
-    } 
-    else 
-    {
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GF = (pGF *) (pughGH->variables [i][timelevel]);
-        pGF_PostSend(pughGH, GF, 2*Dir);
-        pGF_PostSend(pughGH, GF, 2*Dir+1);
-      }
-    }
-
-#ifdef COMM_TIMING
-    t1 = MPI_Wtime();
-    printf ("PS : %lf\n",t1-t2);
-#endif
-
-    /* Now comes the big difference between derived types and
-       allocated buffers. With derived types, we now have to
-       wait on all our recieve AND SEND buffers so we can
-       keep on using the send buffers ( as communications are
-       in-place). With the allocated we have to wait on each
-       recieve, but not on the send, since we don't need the
-       send buffer until we pack a send again (above)
-    */
-    
-    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
-    {
-      /* Do a wait any on the receives */
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GF = (pGF *) (pughGH->variables [i][timelevel]);
-        MPI_Wait(&(GF->rreq[2*Dir]),&mss);
-        pGF_FinishRecv(pughGH, GF, 2*Dir);
-        MPI_Wait(&(GF->rreq[2*Dir+1]),&mss);
-        pGF_FinishRecv(pughGH, GF, 2*Dir+1);
-      }
-    } 
-
-    if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
-    {
-      /* Load up the thing for the waitall */
-      for (i=0;i<n_vars;i++) 
-      {
-        int id = i*4;
-        
-        GF = (pGF *) (pughGH->variables [i][timelevel]);
-        if (GF->docomm[2*Dir] &&
-            GF->storage) 
-        {
-          sr[id] = GF->sreq[2*Dir];
-          sr[id+1] = GF->rreq[2*Dir];
-        } 
-        else 
-        {
-          sr[id] = MPI_REQUEST_NULL;
-          sr[id+1] = MPI_REQUEST_NULL;
-        }
-        
-        if (GF->docomm[2*Dir+1] &&
-            GF->storage) 
-        {
-          sr[id+2] = GF->sreq[2*Dir+1];
-          sr[id+3] = GF->rreq[2*Dir+1];
-        } 
-        else 
-        {
-          sr[id+2] = MPI_REQUEST_NULL;
-          sr[id+3] = MPI_REQUEST_NULL;
-        }
-      }
-      /* Now do a waitall */
-      MPI_Waitall(4*n_vars,sr,&mss);          
-    }
-
-    if (sr)
-      free(sr);
-#ifdef COMM_TIMING
-    t2 = MPI_Wtime();
-    printf ("FR : %lf\n",t2-t1);
-#endif
-  }
-#endif            /* MPI */
-
-  /* get the time spent in communication */
-
-#if 0
-  CactusStopTimer (&pughGH->comm_time);
-#endif
-
-  return (1);
-}
-
-
 int pugh_EnableArrayGroupStorage(cGH *GH,                              
                                  int group,
                                  int vtype,
@@ -1228,7 +929,8 @@ int pugh_EnableArrayGroupStorage(cGH *GH,
   pGH *mypGH;
   pGA *GA;
   int level;
-  int zero_memory=0;
+  /* FIXME: This should come from a parameter */
+  int zero_memory=1;
   int padding_active=0;
   int padding_cacheline_bits=0;
   int padding_size=0;
@@ -1300,79 +1002,6 @@ int pugh_EnableArrayGroupStorage(cGH *GH,
 }
 
 
-int pugh_EnableGFGroupStorage(cGH *GH,                              
-                              int group,
-                              int vtype,
-                              int dim,
-                              int n_variables,
-                              int n_timelevels)
-{
-  int nstorage;   /* Number of GF for which storage was set */
-  int nnostorage; /* Number of GF for which no storage was set */
-  int retval;
-  int first_var;
-  int var;
-  pGH *mypGH;
-  pGF *GF;
-  int level;
-
-  nstorage = 0;
-  nnostorage = 0;
-
-  mypGH = pugh_pGH(GH);
-  
-  first_var = CCTK_FirstVarIndexI(group);
-
-  for(var = first_var; var < first_var+n_variables; var++)
-  {
-    for(level = 0; level < n_timelevels; level++)
-    {
-      GF = (pGF *)(mypGH->variables[var][level]);
-
-      if(!GF->storage)
-      {
-#ifdef DEBUG_PUGH
-        {
-          const char *gfname;
-          gfname = CCTK_FullName(var);
-          printf("   Switching on storage in pugh_EnableGFDataStorage for %s\n",gfname);
-        }
-#endif
-        EnableGFDataStorage(mypGH,GF);
-        GH->data[var][level] = GF->data;
-        nnostorage++;
-      }
-      else
-      {
-        GH->data[var][level] = GF->data;
-        nstorage++;
-      }
-    }
-  }
-
-  if (nstorage > 0 && nnostorage > 0)
-  {
-    CCTK_WARN(0,"Group storage violation in pugh_EnableGroupGFStorage");
-    retval = -1;
-  }
-  else if (nstorage > 0)
-  {
-    retval = 1;
-  }
-  else if (nnostorage > 0)
-  {
-    retval = 0;
-  }
-  else
-  {
-    CCTK_WARN(0,"Problem in pugh_EnableGroupGFStorage");
-    retval = -1;
-  }
-
-  return retval;
-}
-
-
 int pugh_DisableArrayGroupStorage(cGH *GH,                              
                                  int group,
                                  int vtype,
@@ -1384,84 +1013,6 @@ int pugh_DisableArrayGroupStorage(cGH *GH,
 }
 
 
-int pugh_DisableGFGroupStorage(cGH *GH,                              
-                               int group,
-                               int vtype,
-                               int dim,
-                               int n_variables,
-                               int n_timelevels)
-{
-  int number_disabled;
-  int number_left;
-  int retval;
-  int first_var;
-  int var;
-  pGH *mypGH;
-  pGF *GF;
-  int level;
-
-  number_disabled = 0;
-  number_left = 0;
-
-  mypGH = pugh_pGH(GH);
-  
-  first_var = CCTK_FirstVarIndexI(group);
-
-  for(var = first_var; var < first_var+n_variables; var++)
-  {
-    for(level = 0; level < n_timelevels; level++)
-    {
-
-      GF = (pGF *)(mypGH->variables[var][level]);
-      if(GF->storage)
-      {
-#ifdef DEBUG_PUGH
-        {
-          const char *gfname;
-          gfname = CCTK_FullName(var);
-          printf("   Switching off storage in pugh_DisableGFDataStorage");
-          printf(" for %s\n",gfname);
-        }
-#endif
-        DisableGFDataStorage(mypGH,GF);
-        number_disabled++;        
-        GH->data[var][level] = GF->data;
-
-        retval = 1;
-      }
-      else
-      {
-        GH->data[var] = GF->data;
-        number_left++;
-        retval = 0;
-      }
-    }
-  }
-
-  if (number_disabled > 0 && number_left > 0)
-  {
-    CCTK_WARN(0,"Group storage violation in DisableGroupGFStorage");
-    retval = -1;
-  }
-  else if (number_disabled > 0)
-  {
-    retval = 1;
-  }
-  else if (number_left > 0)
-  {
-    retval = 0;
-  }
-  else
-  {
-    CCTK_WARN(0,"Problem in DisableGroupGFStorage");
-    retval = -1;
-  }
-
-  return retval;
-
-}
-
-
 int pugh_MyProc(cGH *GH)
 {
   int myproc;