Removing old files with pGFs

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@192 b61c5cb5-eaca-4651-9a7a-d64986f99364

17 files changed, 43 insertions, 2088 deletions

diff --git a/src/PUGH_Evolve.c b/src/Evolve.c
index 2904796..2904796 100644
--- a/src/PUGH_Evolve.c
+++ b/src/Evolve.c
diff --git a/src/PughUtils.c b/src/PughUtils.c
index 8b37cd5..10aab02 100644
--- a/src/PughUtils.c
+++ b/src/PughUtils.c
@@ -53,7 +53,10 @@ void pugh_Report(CCTK_CARGUMENTS)
   if (cctkGH->cctk_dim == 3)
   {
   
-    sprintf(message,"Size: %d %d %d",pughGH->GFExtras[2]->nsize[0],pughGH->GFExtras[2]->nsize[1],pughGH->GFExtras[2]->nsize[2]);
+    sprintf(message,"Size: %d %d %d",
+	    pughGH->GFExtras[2]->nsize[0],
+	    pughGH->GFExtras[2]->nsize[1],
+	    pughGH->GFExtras[2]->nsize[2]);
     CCTK_INFO(message);
     
 #ifdef MPI
@@ -63,25 +66,28 @@ void pugh_Report(CCTK_CARGUMENTS)
 	    pughGH->Connectivity[2]->nprocs[2]);
     CCTK_INFO(message);
 
-    if (CCTK_Equals(partition, "automatic")) { 
+    if (CCTK_Equals(partition, "automatic")) 
+    { 
       sprintf(message,"Local load: %d   [%d x %d x %d]",
               pughGH->GFExtras[2]->npoints,
               pughGH->GFExtras[2]->lnsize[0], 
 	      pughGH->GFExtras[2]->lnsize[1], 
 	      pughGH->GFExtras[2]->lnsize[2]);
       CCTK_INFO(message);                                                
-    } else { /* manual partition */
-      for (i=0; i<pughGH->nprocs; i++) { 
+    } 
+    else 
+    { /* manual partition */
+      for (i=0; i<pughGH->nprocs; i++) 
+      { 
         sprintf(message, 
-                "Local load on proc %d: %d [%d x %d x %d]",       
-                i,                                 
+                "Local load on proc %d: %d [%d x %d x %d]",i,      
                 pughGH->GFExtras[2]->rnpoints[i],                           
                 pughGH->GFExtras[2]->rnsize[i][0],                          
                 pughGH->GFExtras[2]->rnsize[i][1],                          
                 pughGH->GFExtras[2]->rnsize[i][2]);                         
         CCTK_INFO(message);                                                
-        }                       
-     }                                                    
+      }                       
+    }                                                    
 #endif
   }
   else if (cctkGH->cctk_dim == 2)
@@ -135,6 +141,8 @@ void pugh_Report(CCTK_CARGUMENTS)
 
   free(message);
 
+  return;
+
 }
 
  /*@@
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index 50708e3..02afaa1 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -16,38 +16,28 @@
 
 static char *rcisd = "$Header$";
 
-pGExtras *pugh_SetupPGExtras(int dim, 
+pGExtras *pugh_SetupPGExtras(int  dim, 
 			     int *perme,
-			     int stagger,
+			     int  stagger,
 			     int *sh,
 			     int *nghosts,
-			     int total_procs,
+			     int  total_procs,
 			     int *nprocs,
-			     int this_proc);
+			     int  this_proc);
 
-pConnectivity *pugh_SetupConnectivity(int dim, 
-				      int total_procs, 
+pConnectivity *pugh_SetupConnectivity(int  dim, 
+				      int  total_procs, 
 				      int *nprocs,
 				      int *perme);
 
-/*pGF *SetupPGF(pGH          *GH, 
-	      const char   *name, 
-	      int           dim, 
-	      int           varsize, 
-	      int           vtype, 
-	      int           staggercode);
-*/
-pGA *SetupPGA
-  (
-   void          *parent,
-   pGExtras      *extras,
-   pConnectivity *connectivity,
-   const char    *name,
-   int            id,
-   int            varsize, 
-   int            vtype, 
-   int            stagger
-  );      
+pGA *SetupPGA(void          *parent,
+	      pGExtras      *extras,
+	      pConnectivity *connectivity,
+	      const char    *name,
+	      int            id,
+	      int            varsize, 
+	      int            vtype, 
+	      int            stagger);      
 
   
  /*@@
@@ -95,7 +85,8 @@ int pugh_SetupScalarGroup
       {
         for(level = 0; level < n_timelevels; level++)
         {
-          newGH->variables[newGH->nvariables][level] = (void *)malloc(var_size);
+          newGH->variables[newGH->nvariables][level] = 
+	    (void *)malloc(var_size);
         }
         newGH->nvariables++;
       }
@@ -132,17 +123,14 @@ int pugh_SetupScalarGroup
 
 @@*/
 
-int pugh_SetupArrayGroup
-  (
-   pGH *newGH, 
-   int *nsize, 
-   int *ghostsize,
-   int vtype, 
-   int dim, 
-   int n_variables, 
-   int staggercode, 
-   int n_timelevels
-  )
+int pugh_SetupArrayGroup(pGH *newGH, 
+			 int *nsize, 
+			 int *ghostsize,
+			 int  vtype, 
+			 int  dim, 
+			 int  n_variables, 
+			 int  staggercode, 
+			 int  n_timelevels)
 {
 
   int           i;
@@ -185,7 +173,6 @@ int pugh_SetupArrayGroup
     CCTK_WARN(0,"Memory error in PUGH");
   }
 
-
   connectivity = pugh_SetupConnectivity(dim,newGH->nprocs,nprocs,perme);
 
   extras = pugh_SetupPGExtras(dim, 
diff --git a/src/SetupPGF.c b/src/SetupPGF.c
deleted file mode 100644
index e4b6402..0000000
--- a/src/SetupPGF.c
+++ /dev/null
@@ -1,527 +0,0 @@
- /*@@
-   @file      SetupPGF.c
-   @date      Fri Feb 21 11:04:20 1997
-   @author    
-   @desc 
-   Sets up (eg allocates) the pugh Grid Function. This is
-   crucial to understaiding the grid function struct.
-   @enddesc 
-   @version $Header$
- @@*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-
-#include "cctk.h"
-#include "cctk_Parameters.h"
-#include "cctk_Cache.h"
-
-#include "pugh.h"
-
-static char *rcsid = "$Header$";
-
-
-/* local function prototypes */
-void SetGFComm(pGH *GH, pGF *res, int docomm);
-int EnableGFDataStorage(pGH *GH, pGF *GF);
-int DisableGFDataStorage(pGH *GH, pGF *GF);
-
- /*@@
-   @routine    SetupPGF
-   @date       Fri Feb 21 11:05:04 1997
-   @author     
-   @desc 
-   Allocates the Grid Func structure (I wanted to say object ...)
-   There are several steps to this
-   <ul>
-     <li> Set up the name
-     <li> Set up the docomm flag and IO flags
-     <li> Allocate output file pointer arrays
-     <li> Allocate the data buffer
-     <li> Allocate the send buffer
-     <li> Add myself to the Grid Hierarchy
-   </ul>
-   But this is pretty straightforward code nonetheless.
-   <p>
-   Note that the input variable "storage" determines whether to
-   allocate the 3-D data or not. You can toggle this later
-   with @seeroutine DisableGFDataStorage and @seeroutine
-   EnableGFDataStorage
-   @enddesc 
-   @calls DisableGFDataStorage, EnableGFDataStorage
-@@*/
-
-
-pGF *SetupPGF(pGH *GH, const char *name, int dim, int varsize, int vtype, int stagger) 
-{
-  DECLARE_CCTK_PARAMETERS
-
-  pGF *res;                        /* The result */
-#ifdef MPI
-  int i;
-#endif
-#if 0
-  pGF **rlist;                        /* A temporary to add to the GH */
-  int dir,  sz;                      /* Counter thingies */
-  /* int dirp1, dirp2 */
-#endif
-
-  /* Fudge for CCTK. */
-  int storage = PUGH_NOSTORAGE;
-  int docomm = PUGH_NOCOMM;
-
-  /* Space for the struct */
-  res = (pGF *)malloc(sizeof(pGF));
-
-  /* Set my parent GH */
-  res->parentGH = GH;
-
-  /* Set up the name */
-  res->name = strdup (name);
-
-  /* Set up the dimension */
-  res->dim = dim;
-
-  res->stagger = stagger;
-  res->varsize = varsize;
-  res->vtype   = vtype;
-
-#if 0
-  
-   /* Finally add this to the list of grid funcs in the gh */
-  GH->ngridFuncs ++;
-  rlist = (pGF **)malloc(GH->ngridFuncs * sizeof(pGF *));
-  if (GH->ngridFuncs > 1) 
-  {
-    for (i=0;i<GH->ngridFuncs-1;i++)
-      rlist[i] = GH->gridFuncs[i];
-    free(GH->gridFuncs);
-  }
-  res->gfno = GH->ngridFuncs - 1;
-  rlist[GH->ngridFuncs-1] = res;
-  GH->gridFuncs = rlist;
-
-#endif
-
-  /* FIXME: special pad in enable GF storage. */
-  res->gfno = GH->nvariables-1;
-
-
- SetGFComm(GH, res, docomm);
-
-  /* Set up the comm layer */
-  res->padddata = NULL;
-  res->data = NULL;
-  res->send_buffer = NULL;
-  res->recv_buffer = NULL;
-
-  if (storage == PUGH_NOSTORAGE) 
-  {
-    res->storage = PUGH_STORAGE; /* Fake it out... */
-    DisableGFDataStorage(GH, res);
-  } 
-  else if (storage == PUGH_STORAGE) 
-  {
-    res->storage = PUGH_NOSTORAGE; /* Fake it out... */
-    EnableGFDataStorage(GH, res);
-  } 
-  else 
-  {
-    char *msg = (char *) malloc (80 + strlen (res->name));
-
-    sprintf (msg, "Storage keyword set incorrectly for %s [%d]\n"
-            "Perhaps misspelled in GridFuncList? Defaulting to storage active.",
-             res->name, storage);
-    CCTK_WARN (1, msg);
-    free (msg);
-    EnableGFDataStorage(GH, res);
-  }
-
-  /* FIXME I don't understand if the above needs to be there */
-  /* It can be useful for debugging to have memory on for all GFs */
-  if (enable_all_storage) EnableGFDataStorage(GH,res);
-
-#ifdef MPI
-  /* Null my send and recieve requests */
-  for (i=0;i<2*res->dim;i++) 
-  {
-    res->sreq[i] = MPI_REQUEST_NULL;
-    res->rreq[i] = MPI_REQUEST_NULL;
-  }
-#endif
-
-  /* And return myself... */
-  return res;
-
-}
-
- /*@@
-   @routine    DestroyPGF
-   @date       Thu Aug 21 11:44:22 1997
-   @author     Paul Walker
-   @desc 
-   Destroys a GF object.
-   @enddesc 
-@@*/
-
-void DestroyPGF(pGH *GH, pGF **GFin) 
-{
-  pGF *GF;
-  GF = *GFin;
-
-  if (GF->storage)
-  {
-    DisableGFDataStorage(GH, GF);
-  }
-
-  free(GF->name);
-  free(GF->padddata);
-
-  GF->data = NULL;
-
-  free(GF);
-  *GFin = NULL;
-
-}
-
-
- /*@@
-   @routine    EnableGFDataStorage
-   @date       Thu Apr  3 12:44:38 1997
-   @author     Paul Walker
-   @desc 
-   This routine toggles the data storage to the "on"
-   position. That means that data points to a 3D
-   array, rather than a single CCTK_REAL, and the
-   storage flag is set to one. This is used quite
-   a lot by thorns which need to toggle memory. for
-   instance, see the trK Driver in the util thron.
-   @enddesc 
-@@*/
-
-int EnableGFDataStorage(pGH *GH, pGF *GF)
-{
-  DECLARE_CCTK_PARAMETERS
-  int i, dir, sz, ldir;
-#if 0
-  int special_pad, cache_size, start;
-#endif
-  static int ive_blathered = 0;
-  char *temp_byte;
-
-  if (zero_memory && !ive_blathered)
-  {
-    CCTK_INFO("Zeroing memory for allocated GFs at alloc time");
-  }
-
-#if 0
-  if (padding_active && !ive_blathered) 
-  {
-    char *infomsg = (char *) malloc (200);
-    sprintf (infomsg,"PUGH Memory padding active. Stats:\n  cacheline_bits   : %d\n  size             : %d\n  spacing          : %d",padding_cacheline_bits, padding_size, padding_address_spacing);
-    CCTK_INFO(infomsg);
-    free(infomsg);
-  }
-#endif
-  
-  ive_blathered = 1;
-
-  if (GF->storage == PUGH_STORAGE) 
-  {
-    char *msg = (char *) malloc (80 + strlen (GF->name));
-
-    sprintf (msg, "Tried to enable %s when already enabled", GF->name);
-    CCTK_WARN (1, msg);
-    free (msg);
-    return (1);
-  }
-
-  /* Set up the send buffers. Note we only do this if
-     we are not using the derived types comm style.
-     We assume here that pointers have always the same size
-     (regardless of the datatype they point to).
-     */
-  assert(!GF->send_buffer);
-  GF->send_buffer = malloc (2*GF->dim * sizeof (void *));
-
-  assert(!GF->recv_buffer);
-  GF->recv_buffer = malloc (2*GF->dim * sizeof (void *));
-
-  
-  for (i=0;i<2*GF->dim;i++) 
-  {
-    dir = i/2;
-    
-    sz = GH->nghostzones[dir];
-    for (ldir=0;ldir<GF->dim;ldir++)
-    {
-      sz = sz*GH->lnsize[ldir];
-    }
-    sz = sz/GH->lnsize[dir];
-
-    if (GH->neighbors[GH->myproc][i] >= 0) 
-    {
-      GF->buffer_sz[i] = sz;
-      GF->send_buffer[i] = malloc (sz * GF->varsize);
-      GF->recv_buffer[i] = malloc (sz * GF->varsize);
-      assert(GF->recv_buffer[i]);
-      assert(GF->send_buffer[i]);
-    } 
-    else 
-    {
-      GF->buffer_sz[i] = 0;
-      GF->send_buffer[i] = NULL;
-      GF->recv_buffer[i] = NULL;
-    }
-  }
-
-  /* Set up the storage */
-  if (GF->padddata) 
-  {
-    free (GF->padddata);
-  }
-
-  if (!padding_active) 
-  {
-    if (zero_memory)
-    {
-      GF->padddata = calloc (GH->npoints, GF->varsize);
-    }
-    else
-    {
-      GF->padddata = malloc (GH->npoints * GF->varsize);
-    }
-    GF->data     = GF->padddata;        /* No padding case */
-  } 
-  else 
-  {
-#if 0
-    if (zero_memory)
-    {
-      GF->padddata  = calloc (GH->npoints + padding_size, GF->varsize);
-    }
-    else
-    {
-      GF->padddata  = malloc ((GH->npoints + padding_size) * GF->varsize);
-    }
-   
-    /* Align the actual starting address on a cache line.
-       More specifically align on a unique cache line for
-       each field. 
-       */
-    cache_size = 2 << (padding_cacheline_bits - 1);
-    start = ((long) (GF->padddata) / GF->varsize)%cache_size;
-
-    special_pad = ((GF->gfno) * padding_address_spacing + cache_size - start)%cache_size;
-
-    GF->data = (char *) GF->padddata + special_pad*GF->varsize;
-
-    if (special_pad > padding_size)
-    {
-      char msg [100];
-
-      sprintf (msg, "FATAL ERROR: padding size not large enough for cache type "
-               "specified %d %d %d\n", special_pad, padding_size, cache_size);
-      CCTK_WARN (0, msg);
-    }
-#endif
-
-    /* Use the Cactus Cache alignment function */
-    GF->data = Util_CacheMalloc(GF->gfno, 
-                                GH->npoints * GF->varsize, 
-                                &(GF->padddata));
-    
-    /* Zero the memory if desired. */
-    if(GF->data && zero_memory)
-    {
-      for(temp_byte = (char *)GF->data; 
-          temp_byte < ((char *)GF->data)+GH->npoints * GF->varsize; 
-          temp_byte++)
-      {
-        *temp_byte = 0;
-      }
-    }
-  }
-
-
-  if (!GF->padddata) 
-  {
-    char *msg = (char *) malloc (80 + strlen (GF->name));
-
-    sprintf (msg, "FATAL ERROR: Cannot allocate data for %s [%d]\n",
-                 GF->name, GF->gfno);
-    CCTK_WARN (0, msg);
-    free (msg);
-  }
-  GF->storage = PUGH_STORAGE;
-
-  return 1;
-}
-
- /*@@
-   @routine    DisableGFDataStorage
-   @date       Thu Apr  3 12:45:34 1997
-   @author     Paul Walker
-   @desc 
-   This routine disables the grid function storage.
-   That is, it un-allocates the 3D array and in its
-   place allocates a single CCTK_REAL with the value
-   0.0. This allows us to still have something to
-   pass around (an array of size (1,1,1) in fortran
-   speak) but also to not need all our 3D arrays 
-   "on" all the time.
-   @enddesc 
-@@*/
-
-
-int DisableGFDataStorage(pGH *GH, pGF *GF) 
-{
-  int i;
-
-  if (GF->storage == PUGH_NOSTORAGE) 
-  {
-    char *msg = (char *) malloc (80 + strlen (GF->name));
-
-    sprintf (msg, "Tried to disable %s when already disabled", GF->name);
-    CCTK_WARN (1, msg);
-    free (msg);
-    return (1);
-  }
-
-  if (GF->padddata) 
-  {
-    free(GF->padddata);
-    GF->padddata = NULL;
-    GF->data = NULL;
-  }
-
-  if (GF->send_buffer) 
-  {
-    for (i=0;i<2*GF->dim;i++)
-    {
-      if (GF->send_buffer[i]) 
-      {
-#ifdef MPI
-        if(GF->sreq[i] != MPI_REQUEST_NULL)
-        {
-          CACTUS_MPI_ERROR(MPI_Request_free(&(GF->sreq[i])));
-        }
-#endif
-        free(GF->send_buffer[i]);
-        GF->send_buffer[i] = NULL;
-      }
-    }
-    free(GF->send_buffer);
-    GF->send_buffer = NULL;
-  }
-
-  if (GF->recv_buffer) 
-  {
-    for (i=0;i<2*GF->dim;i++)
-    {
-      if (GF->recv_buffer[i]) 
-      {
-        free(GF->recv_buffer[i]);
-        GF->recv_buffer[i] = NULL;
-      }
-    }
-    free(GF->recv_buffer);
-    GF->recv_buffer = NULL;
-  }
-
-  GF->padddata = malloc (1);
-  GF->data = GF->padddata;
-#if 0
-  GF->data[0] = 0.0;                /* Very important! */
-#endif
-  GF->storage = PUGH_NOSTORAGE;
-
-  return (0);
-}
-
-
- /*@@
-   @routine    SetGFComm
-   @date       Thu Apr  3 12:46:23 1997
-   @author     Paul Walker
-   @desc 
-   This sets the docomm[3] array of the GF based
-   on the setting of the comm flag. Note the
-   comm flag constants are defined in
-   @seeheader pughDriver.h. As well as SetupPGF
-   this is called by thorns (eg, elliptic) which
-   need to toggle this state.
-   @enddesc 
-   @calledby   SetupPGF
-@@*/
-
-
-void SetGFComm(pGH *GH, pGF *res, int docomm) {
-  /* Copy docomm */
-  int i,idir;
-  res->commflag = docomm;
-
-  /* First set all communcation off */
-  for (i=0;i<2*res->dim;i++)
-  {
-    res->docomm[i] = 0;
-  }
-
-  if (docomm == PUGH_NOCOMM)
-  {
-    for (i=0;i<2*res->dim;i++)
-    {
-      res->docomm[i] = 0;
-    }
-  } 
-  else if (docomm == PUGH_ALLCOMM) 
-  {
-    for (i=0;i<2*res->dim;i++)
-    {
-      res->docomm[i] = 1;
-    }
-  } 
-  else if (docomm == PUGH_PLUSFACESCOMM) 
-  {
-    for (i=0;i<res->dim;i++)
-    {
-      res->docomm[2*i+1] = 1;
-    }
-  } 
-  else if (docomm == PUGH_MINUSFACESCOMM) 
-  {
-    for (i=0;i<res->dim;i++)
-    {
-      res->docomm[2*i] = 1;
-    }
-  } 
-  else 
-  {
-    for (idir=0;idir<res->dim;idir++)
-    {
-      if (docomm == PUGH_COMM(idir)) 
-      {
-        res->docomm[2*idir] = 1;
-        res->docomm[2*idir+1] = 1;
-      } 
-    }
-  } 
-
-  /* FIXME Add back the check that you have a valid COMM model: Gab */
-
-  /* Handle nsize = 1 type cases. This is only important for one
-     processor MPI periodic boundaries */
-
-  for (idir=0;idir<res->dim;idir++)
-  {
-    if (GH->nsize[idir] == 1) 
-    {
-      res->docomm[2*idir] = 0;
-      res->docomm[2*idir+1] = 0;
-    }
-  }
-    
-}
-
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 1ddca74..79fb081 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -1482,7 +1482,7 @@ int pugh_EnablePGAStorage(pGA *GA,
       else
       {
 
-#ifdef 0
+#if 0
         if(zero_memory)
         {
           GA->padddata = calloc(GA->extras->npoints + padding_size, GA->varsize);
diff --git a/src/include/pGF.h b/src/include/pGF.h
deleted file mode 100644
index ab9e0d4..0000000
--- a/src/include/pGF.h
+++ /dev/null
@@ -1,76 +0,0 @@
- /*@@
-   @header    pGF.h
-   @date      Fri Feb 21 10:16:32 1997
-   @author    Paul Walker
-   @desc 
-   The pugh Grid Function structure. This is the heart of
-   the data storage and the like. You can't have a GF
-   without a GH, so you'd better see @seefile pGH.h
-   also.
-   <p>
-   One wonderfully new feature that a pGF has is that it contains
-   both a CCTK_REAL* padddata and a CCTK_REAL* data. What is the difference?
-   Well data is the base address of the fortran data set. It points
-   into padddata. This allows us, if we want to, to explicitly move
-   the starting point of the array to align with cache lines.
-   Important on some architectures. So the deal is if we are
-   in padding mode (see @seeroutine EnableGFDataStorage for more
-   on that) padddata will be bigger than data, and data will point
-   to an address insinde padddata and such that lnx * lny * lnz
-   further down the data pointer is still in padddata. If we are
-   not in padding mode, data = padddata as pointers.
-   <p>
-   The only place we explicitly reference padddata is when we
-   create or de-allocate data. <b>All other operatoins shold
-   be done on *data</a>
-   @enddesc 
-   @version $Header$
- @@*/
-
-#ifndef _PGF_H_
-#define _PGF_H_ 1
-
-#define DATINDEX(GH,i,j,k) ((i) + GH->lnsize[0]*((j)+GH->lnsize[1]*(k)))
-/*#define DI(GH,i,j,k) ((i) + GH->lnx*((j)+GH->lny*(k)))*/
-/*#define GDATINDEX(GH,i,j,k) ((i) + GH->nx*((j)+GH->ny*(k)))*/
-
-#include "CactusTimers.h"
-
-typedef struct PGF 
-{
-  char  *name;               /* The name of the grid function */
-  int   gfno;                /* My ID number in my GH parent. */
-  int   dim;                 /* dimension of GF */
-  void  *padddata;           /* Storage for the data. */
-  void  *data;               /* See the note above. */
-  int   buffer_sz[6];        /* Size of the face ghost zones */
-  void  **send_buffer;       /* Storage for buffered Comm if */
-  void  **recv_buffer;       /* we don't use derived types */
-  int   commflag;            /* What is the comm flag set to? */
-  int   docomm[6];           /* Do we do comm or not? */
-  int   storage;             /* Do we have storage or not? */
-  int   stagger;             /* Only Vertex Centered now... */
-
-  struct PGH      *parentGH; /* The GH to which I belong */
-                             /* Note this is struct PGH whic is
-                                typedeffed to pGH in pGH.h, but
-                                that is included AFTER this so we 
-                                need the full name for the lookahead
-                                structure reference thingy.
-                             */
-
-  int   varsize;             /* The size of the data */
-  int   vtype;               /* The type of the data */
-
-#ifdef MPI
-  MPI_Request sreq[6], rreq[6];  /* Comm requests and statuses. */
-  MPI_Status ms;
-#endif
-
-  /*
-    #include "pGF_Extensions.h"
-  */
-
-} pGF;
-
-#endif /* _PGF_H_ */
diff --git a/src/include/pGH.h b/src/include/pGH.h
index ab2fe88..5bc3b5d 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -19,13 +19,11 @@ typedef struct PGH
 
   /* pGH identifier */
   void     *callerid;
-
   int       dim;                /* The dimension of the GH */
 
   /* Size of the processor group */
   int       nprocs;             /* Number of processors */
   int       myproc;             /* My processor */
-  /*int      *nproc;*/              /* Processor topology */
   int       commmodel;          /* Comm model is PUGH_ALLOCATEDBUFFERS */ 
                                 /* or PUGH_DERIVEDTYPES. Currently unused */
   
@@ -36,7 +34,7 @@ typedef struct PGH
   /* What time level we're on */
   int       timelevel;
 
-  int      *perme;               /* Periodic in each direction? */
+  int      *perme;              /* Periodic in each direction? */
   int       periodic;           /* Is the system periodic? */
 
   int       forceSync;          /* force synchronisation of GFs with storage */
@@ -45,7 +43,6 @@ typedef struct PGH
   /* Coordinate information */
   CCTK_REAL dx0, dy0, dz0, dt0; /* Delta of our system */
   CCTK_REAL lx0, ly0, lz0;      /* Processor-Local coordinate origins */
-  /*  CCTK_REAL phys_time;      */    /* physical time */
   int       GHiteration;        /* iteration number on processor */
 
   /* Indentification for a pGH */
diff --git a/src/include/pugh.h b/src/include/pugh.h
index af42aef..489ebe5 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -68,7 +68,6 @@
 #include "pugh_constants.h"
 
 #include "pGV.h"
-/*#include "pGF.h"*/
 #include "pGH.h"
 
 #ifdef MPI
@@ -137,8 +136,4 @@ int pugh_GetBounds(int dim,
 }
 #endif
 
-/*
-#include "pughDriver.h"
-*/
-
 #endif /* defined _PUGH_H_ */
diff --git a/src/include/pughDriver.h b/src/include/pughDriver.h
deleted file mode 100644
index 28c3a92..0000000
--- a/src/include/pughDriver.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*@@
-  @header pughDriver.h
-  @author Paul Walker
-  @date March 1997
-  @desc
-  This is stuff having to do with the driver, namely
-  the problem and globals, as well as definitions of what
-  the fort interfaces are. Also see @seefile pughProblem.h
-  for what the cactus-specific stuff is. The macros FORT_ARGS
-  and FORT_ARGS_PROTO passed from c routines must match 
-  FORT_GH_DECOMP and FORT_GHDECOMP_DECL received in fortran
-  routines defined in @seefile pughFortran.h
-  @enddesc
-  @history
-  @hauthor Gabrielle Allen
-  @hdate Sep 10 @hdesc Added the GH iteration number to fortran macros
-  @hauthor Gabrielle Allen @hdate 29 Sep 1998
-  @hdesc Removed levfac
-  @endhistory
-  @version $Id$
-  @@*/
-
-#ifdef 0
-extern pGHList *GHList;
-
-extern int nx0, ny0, nz0;
-extern int lb[3],ub[3],sh[3],bbox[6];
-extern int convergence;
-
-/* Timing variables */
-extern double commt[PUGH_NTIMERS];
-extern double elltime[PUGH_NTIMERS];
-
-/* How long to run */
-extern int itfirst, itlast, itout;	/* Iteration runs */
-extern int itout3, itout2, itout1, itout0, itoutarr, itinfo;
-
-extern int _show_storage;
-
-/* termination flags: pe-local and global: */
-extern int  cactus_terminate;
-
-/* FORT_ARGS must match FORT_ARGS_PROTO below, and FORT_GH_DECOMP, 
-   FORT_GHDECOMP_DECL in pughFortran.h */
-#define FORT_ARGS(xGH)  xGH, \
-    &(xGH->lnsize[0]),&(xGH->lnsize[1]), &(xGH->lnsize[2]),\
-    lb,ub,sh,bbox, \
-    &(xGH->nx),&(xGH->ny),&(xGH->nz),\
-    &(xGH->cx0),&(xGH->cy0),&(xGH->cz0), \
-    &(xGH->dx0),&(xGH->dy0),&(xGH->dz0),&(xGH->dt0), &(xGH->phys_time), \
-    &(xGH->level), &(xGH->nprocs), &(xGH->myproc), \
-    &(xGH->convlevel), &(xGH->stencil_width), &(xGH->GHiteration)
- 
-/* FORT_ARGS_PROTO must match FORT_ARGS above, and FORT_GH_DECOMP, 
-   FORT_GHDECOMP_DECL in pughFortran.h */
-#define FORT_ARGS_PROTO pGH *, \
-    int *, int *, int *, \
-    int *, int *, int *, int *, \
-    int *, int *, int *, \
-    CCTK_REAL *, CCTK_REAL *, CCTK_REAL *, \
-    CCTK_REAL *, CCTK_REAL *, CCTK_REAL *, CCTK_REAL *, CCTK_REAL *, \
-    int *, int *, int *, \
-    int *, int *, int *
-    
-
-#include "pughProblem.h"
-
-#endif
diff --git a/src/include/pughProblem.h b/src/include/pughProblem.h
deleted file mode 100644
index 2024a09..0000000
--- a/src/include/pughProblem.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*@@
-  @header pughProblem.h
-  @author Paul Walker
-  @date March 1997
-  @desc
-  This routine declares all the problem-specific globals
-  and teaches the driver how to make the grid functions
-  with the right names, and pass them to fortran. It
-  isn't that exciting, but it's pretty damned important
-  if you need to add new fields.
-  @enddesc
-  @version $Id$
-  @@*/
-
-#ifdef 0
-#include "gfIndices.h"
-#include "pughProblemDefs.h"
-
-/* Variables that ALL codes use. */
-#define USER_VARS \
-  int *_one; 
-
-/* Main externals */
-extern int *_one;
-
-/* Historical: Two macros for the same thing. PW. */
-#define PASS_FORT_FIELDS(xGH) \
-  PASS_FORT_FIELDS_SUBSET(xGH)
-
-
-/* If you add items to PASS_FORT_FIELDS make sure */
-/* you add the right type at the right position  */
-/* in PROTO_FORT_FIELDS */
-#define PROTO_FORT_FIELDS \
-    PROTO_FORT_FIELDS_SUBSET
-
-
-#ifdef WIN32
-#define FMODIFIER __stdcall
-#else
-#define FMODIFIER   
-#endif
-
-#ifdef THORN_CHECKPOINT
-  /* Is it possible to ifdef this? Or perhaps to rfr it? */
-#define f_recover FORTRAN_NAME(recover_,RECOVER,recover)
-void FMODIFIER f_recover(FORT_ARGS_PROTO,PROTO_FORT_FIELDS);
-#endif
-
-#include "cactus_constants.h"
-
-#endif
-
diff --git a/src/make.code.defn b/src/make.code.defn
index 650cb9a..d28f187 100644
--- a/src/make.code.defn
+++ b/src/make.code.defn
@@ -1,7 +1,7 @@
 # Main make.code.defn file for thorn pugh
 # : /usr/users/cactus/CCTK/lib/make/new_thorn.pl,v 1.1 1999/02/03 17:00:50 goodale Exp n
 # Source files in this directory
-SRCS =  PUGH_Evolve.c Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c  Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
+SRCS =  Evolve.c Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c  Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
 
 # Subdirectories containing source files
 SUBDIRS =
diff --git a/src/make.configuration.defn b/src/make.configuration.defn
deleted file mode 100644
index 6a7abe2..0000000
--- a/src/make.configuration.defn
+++ /dev/null
@@ -1 +0,0 @@
-# make.configuration.defn for PUGH
diff --git a/src/pGF_FinishRecv.c b/src/pGF_FinishRecv.c
deleted file mode 100644
index e34e9bf..0000000
--- a/src/pGF_FinishRecv.c
+++ /dev/null
@@ -1,230 +0,0 @@
- /*@@
-   @file      pGF_FinishRecv.c
-   @date      Thu Apr  3 11:37:28 1997
-   @author    Paul Walker
-   @desc 
-   The routine which finalize the MPI recieves for a grid
-   function. Critically linked with @seefile pGF_PostRecv.c
-   and @seefile pGF_PostSend.c
-   @enddesc 
-   @version $Header$
- @@*/
-
-#include <stdio.h>
-
-#include "cctk.h"
-#include "pugh.h"
-
-void  pGF_FinishRecv3(pGH *GH,pGF *GF,int dir);
-void  pGF_FinishRecv1(pGH *GH,pGF *GF,int dir);
-
-static char *rcsid = "$Header$";
-
- /*@@
-   @routine    pGF_FinishRecv
-   @date       Thu Apr  3 11:38:07 1997
-   @author     Paul Walker
-   @desc 
-   This routine finalizes the MPI communication through a face.
-   It is crucially linked with @seeroutine pGF_PostRecv and
-   @seeroutine pGF_PostSend.
-   <p>
-   <b>Important!</b>
-   This routine does <b>not</b> wait on the recieves.
-   Before it is called, you must do the MPI_Wait or
-   else you will not get the right answer. for an
-   example of this, see @seeroutine SyncGroupGF or
-   @seeroutine SyncGroupGF
-   @enddesc 
-   @calledby   SyncGroupGF
-   @history
-   @hdate Nov 4 1998 @hauthor Gabrielle Allen
-   @hdesc Allow for forced synchronization of all GFs with storage
-   @endhistory
-@@*/
-
-void pGF_FinishRecv(pGH *GH, pGF *GF, int dir)
-{
-#ifdef MPI
-
-  /* Return if GF has no storage */
-  if (!(GF->storage)) return;
-
-  /* Return if communication not required and no forced synchronisation */
-  if (!(GH->forceSync || GF->docomm[dir])) return;
-
-  if (GH->neighbors[GH->myproc][dir] >= 0) 
-  {
-    /* Here wait one at a time, so the others can arrive while
-       we copy the data back in... */
-    
-#ifdef DEBUG_PRINT
-    printf ("FINISHRECV: GF %s Side %d Proc %d request %d\n",
-            GF->name,dir,GH->myproc,GF->rreq[dir]);
-#endif
-
-    if (GH->dim == 3)
-    {
-      pGF_FinishRecv3(GH,GF,dir);
-    }
-    else if (GH->dim == 1)
-    {
-      pGF_FinishRecv1(GH,GF,dir);
-    }
-    else
-    {
-      CCTK_WARN(0,"PUGH does not support this dimension grid");
-    }
-
-  }
-
-#endif
-}
-
-void  pGF_FinishRecv3(pGH *GH,pGF *GF,int dir)
-{
-  int istart,iend,jstart,jend,kstart,kend;
-  int ii,jj,kk,xx;
-  CCTK_CHAR *char_data;
-  CCTK_INT *int_data;
-  CCTK_REAL *real_data;
-  CCTK_COMPLEX *complex_data;
-
-  /* Copy buffer onto GF Storage */
-  istart = GH->ghosts[GF->stagger][0][dir][0];
-  iend   = GH->ghosts[GF->stagger][1][dir][0];
-  jstart = GH->ghosts[GF->stagger][0][dir][1];
-  jend   = GH->ghosts[GF->stagger][1][dir][1];
-  kstart = GH->ghosts[GF->stagger][0][dir][2];
-  kend   = GH->ghosts[GF->stagger][1][dir][2];
-    
-  /* Great now copy. Note ordering is just link in PostSend */
-  xx=0;
-  switch (GF->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      char_data = (CCTK_CHAR *) GF->data;
-      for (kk=kstart; kk<kend; kk++) 
-      {
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            char_data [DATINDEX (GH,ii,jj,kk)] =
-              ((CCTK_CHAR *) GF->recv_buffer[dir]) [xx++];
-          }
-        }
-      }
-      break;
-      
-    case CCTK_VARIABLE_INT:
-      int_data = (CCTK_INT *) GF->data;
-      for (kk=kstart; kk<kend; kk++)
-      { 
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            int_data [DATINDEX (GH,ii,jj,kk)] =
-              ((CCTK_INT *) GF->recv_buffer[dir]) [xx++];
-          }
-        }
-      }
-      break;
-
-    case CCTK_VARIABLE_REAL:
-      real_data = (CCTK_REAL *) GF->data;
-      for (kk=kstart; kk<kend; kk++) 
-      {
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            real_data [DATINDEX (GH,ii,jj,kk)] =
-              ((CCTK_REAL *) GF->recv_buffer[dir]) [xx++];
-          }
-        }
-      }
-      break;
-      
-    case CCTK_VARIABLE_COMPLEX:
-      complex_data = (CCTK_COMPLEX *) GF->data;
-      for (kk=kstart; kk<kend; kk++)
-      { 
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            complex_data [DATINDEX (GH,ii,jj,kk)] =
-              ((CCTK_COMPLEX *) GF->recv_buffer[dir]) [xx++];
-          }
-        }
-      }
-      break;
-
-    default:
-      CCTK_WARN (1, "Unsupported variable type in pGF_FinishRecv3");
-      return;
-  }
-}
-
-
-
-void  pGF_FinishRecv1(pGH *GH,pGF *GF,int dir)
-{
-  int istart,iend;
-  int ii,xx;
-  CCTK_CHAR *char_data;
-  CCTK_INT *int_data;
-  CCTK_REAL *real_data;
-  CCTK_COMPLEX *complex_data;
-
-  /* Copy buffer onto GF Storage */
-  istart = GH->ghosts[GF->stagger][0][dir][0];
-  iend   = GH->ghosts[GF->stagger][1][dir][0];
-    
-  /* Great now copy. Note ordering is just link in PostSend */
-  xx=0;
-  switch (GF->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      char_data = (CCTK_CHAR *) GF->data;
-      for (ii=istart; ii<iend; ii++)
-      {
-        char_data [ii] =
-          ((CCTK_CHAR *) GF->recv_buffer[dir]) [xx++];
-      }
-      break;
-      
-    case CCTK_VARIABLE_INT:
-      int_data = (CCTK_INT *) GF->data;
-      for (ii=istart; ii<iend; ii++)
-      {
-        int_data [ii] =
-          ((CCTK_INT *) GF->recv_buffer[dir]) [xx++];
-      }
-      break;
-
-    case CCTK_VARIABLE_REAL:
-      real_data = (CCTK_REAL *) GF->data;
-      for (ii=istart; ii<iend; ii++)
-      {
-        real_data [ii] =
-          ((CCTK_REAL *) GF->recv_buffer[dir]) [xx++];
-      }
-      break;
-      
-    case CCTK_VARIABLE_COMPLEX:
-      complex_data = (CCTK_COMPLEX *) GF->data;
-      for (ii=istart; ii<iend; ii++)
-      {
-        complex_data [ii] =
-          ((CCTK_COMPLEX *) GF->recv_buffer[dir]) [xx++];
-      }
-      break;
-
-    default:
-      CCTK_WARN (1, "Unsupported variable type in pGF_FinishRecv3");
-      return;
-  }
-}
diff --git a/src/pGF_PostRecv.c b/src/pGF_PostRecv.c
deleted file mode 100644
index 26654cd..0000000
--- a/src/pGF_PostRecv.c
+++ /dev/null
@@ -1,102 +0,0 @@
- /*@@
-   @file      pGF_PostRecv.c
-   @date      Fri Feb 21 11:28:06 1997
-   @author    
-   @desc 
-   Routines which post all the IRecv commands for a GF. These
-   allow the asyncronous model proposed in, for example,
-   @seeroutine SyncGF to go!
-   @enddesc 
-   @version $Id$
- @@*/
-
-#include <stdio.h>
-
-#include "pugh.h"
-
-static char *rcsid = "$Id$";
-
-/*#define DEBUG_PRINT*/
-
-
- /*@@
-   @routine   pGF_PostRecv
-   @date      Thu Apr  3 12:10:46 1997
-   @author    Paul Walker
-   @desc 
-   This routine posts a recieve (MPI_Irecv) of the buffer
-   we want to get the send from another processor, which
-   will be sent by @seeroutine pGF_PostSend and then
-   finalized by @seeroutoine pGF_FinishRecv. 
-   <p>
-   Aside from a silly calculation to get a unique tag
-   based on neigbors and processors, this is a very
-   straightforward routine.
-   @enddesc 
-   @history
-   @hdate Nov 4 1998 @hauthor Gabrielle Allen
-   @hdesc Allow for forced synchronization of all GFs with storage
-   @endhistory
- @@*/
-
-void pGF_PostRecv(pGH *GH, pGF *GF, int dir)
-{
-#ifdef MPI
-  int rtag;
-  MPI_Datatype mpi_type;
-  MPI_Datatype *recv_dt;
-
-  /* Return if GF has no storage */
-  if (!(GF->storage)) return;
-
-  /* Return if communication not required and no forced synchronisation */
-  if (!(GH->forceSync || GF->docomm[dir])) return;
-
-  if (GH->neighbors[GH->myproc][dir] < 0) return;
-
-  switch (GF->vtype) {
-    case CCTK_VARIABLE_CHAR:
-      mpi_type = PUGH_MPI_CHAR;
-      recv_dt = GH->recv_char_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_INT:
-      mpi_type = PUGH_MPI_INT;
-      recv_dt = GH->recv_int_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_REAL:
-      mpi_type = PUGH_MPI_REAL;
-      recv_dt = GH->recv_real_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_COMPLEX:
-      mpi_type = GH->pugh_mpi_complex;
-      recv_dt = GH->recv_complex_dt [GF->stagger];
-      break;
-
-    default:
-      CCTK_WARN (1, "Unknown variable type in pGF_PostRecv");
-      return;
-  }
-
-  rtag = 1000 + dir + 2 * (GH->myproc + GH->nprocs * GF->gfno);
-  /* mod the tag to force MPI compliance */
-  rtag = rtag % 32768;
-#ifdef DEBUG_PRINT
-  printf ("RECV GF %s Side %d Proc %d rtag %d size %d from proc %d\n",
-          GF->name,dir,GH->myproc,rtag,
-          GF->buffer_sz[dir],GH->neighbors[GH->myproc][dir]);
-#endif
-  if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
-    CACTUS_MPI_ERROR (MPI_Irecv (GF->recv_buffer [dir],
-                      GF->buffer_sz [dir], mpi_type,
-                      GH->neighbors [GH->myproc][dir], rtag,
-                      GH->PUGH_COMM_WORLD, &(GF->rreq [dir])));
-  if (GH->commmodel == PUGH_DERIVEDTYPES)
-    CACTUS_MPI_ERROR (MPI_Irecv (GF->data,
-                      1, recv_dt [dir],
-                      GH->neighbors [GH->myproc][dir], rtag,
-                      GH->PUGH_COMM_WORLD, &(GF->rreq [dir])));
-#endif
-}
diff --git a/src/pGF_PostSend.c b/src/pGF_PostSend.c
deleted file mode 100644
index 2aed146..0000000
--- a/src/pGF_PostSend.c
+++ /dev/null
@@ -1,291 +0,0 @@
- /*@@
-   @file      pGF_PostSend.c
-   @date      Thu Apr  3 11:58:14 1997
-   @author    Paul Walker
-   @desc 
-   The send-arm of the three-way pugh communcations branch
-   which is @seefile pGF_PostRecv.v, @seefile pGF_PostSend.c
-   and @seefile pGF_FinishRecv.c. See the documentation for
-   @seeroutine pGF_PostSend for details on this routine.
-   @enddesc 
-   @version $Header$
- @@*/
-
-#include <stdio.h>
-
-#include "cctk.h"
-
-#include "pugh.h"
-
-static char *rcsid = "$Header$";
-
-void pGF_PostSend1(pGH *GH, pGF *GF, int dir);
-void pGF_PostSend3(pGH *GH, pGF *GF, int dir);
-
-/*#define DEBUG_PRINT*/
-
- /*@@
-   @routine    pGF_PostSend
-   @date       Thu Apr  3 11:58:59 1997
-   @author     Paul Walker
-   @desc 
-   This routine posts an asyncronous MPI send of a buffer
-   for a face (MPI_Isend). It does this by first packing up
-   a send buffer (allocated in @seeroutine SetupPGF) then
-   sending it out on the pipe.
-   <p>
-   Since this is an asynchronous communications model we
-   assume that a recieve has been posted for the send.
-   thus the correct calling order for this is as in
-   @seeroutine SyncGF, eg, after a Recv.
-   <p>
-   Note this does <b>not</b> wait on the send buffer from previous
-   communications. It is the users responsibility to wait on
-   that buffer.
-   @enddesc 
-   @calledby   SyncGroupGF
-   @history
-   @hdate Nov 4 1998 @hauthor Gabrielle Allen
-   @hdesc Allow for forced synchronization of all GFs with storage
-   @endhistory
-@@*/
-
-void pGF_PostSend(pGH *GH, pGF *GF, int dir)
-{
-#ifdef MPI
-  int stag, dircomp;
-  double ts, tw, tp;
-  MPI_Datatype mpi_type;
-  MPI_Datatype *send_dt;
-
-  /* Return if GF has no storage */
-  if (!(GF->storage)) return;
-
-  /* Return if communication not required and no forced synchronisation */
-  if (!(GH->forceSync || GF->docomm[dir])) return;
-
-  if (GH->neighbors[GH->myproc][dir] < 0) return;
-
-  switch (GF->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      mpi_type = PUGH_MPI_CHAR;
-      send_dt = GH->send_char_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_INT:
-      mpi_type = PUGH_MPI_INT;
-      send_dt = GH->send_int_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_REAL:
-      mpi_type = PUGH_MPI_REAL;
-      send_dt = GH->send_real_dt [GF->stagger];
-      break;
-
-    case CCTK_VARIABLE_COMPLEX:
-      mpi_type = GH->pugh_mpi_complex;
-      send_dt = GH->send_complex_dt [GF->stagger];
-      break;
-
-    default:
-      CCTK_WARN (1, "Unknown variable type in pGF_PostRecv");
-      return;
-  }
-
-  ts = MPI_Wtime();
-  dircomp = dir+1;                /* Complementary direction */
-  if (dircomp %2 == 0) dircomp = dir-1;
-  /* Note this is the complement of the rtag set in PostRecv */
-  stag = 1000 + dircomp + 2 * (GH->neighbors[GH->myproc][dir] + 
-                             GH->nprocs * GF->gfno);
-  /* mod the tag to force MPI compliance */
-  stag = stag % 32768;
-
-#ifdef DEBUG_PRINT
-  printf ("SEND: GF %s Side %d Proc %d stag %d Size %d to %d\n",
-          GF->name,dir,GH->myproc,stag,
-          GF->buffer_sz[dir],GH->neighbors[GH->myproc][dir]);
-#endif
-
-  if (GH->commmodel == PUGH_DERIVEDTYPES)
-  {
-    CACTUS_MPI_ERROR (MPI_Isend (GF->data,
-                      1, send_dt [dir],
-                      GH->neighbors [GH->myproc][dir], stag,
-                      GH->PUGH_COMM_WORLD, &(GF->sreq [dir])));
-  }
-
-  if (GH->commmodel == PUGH_ALLOCATEDBUFFERS) 
-  {
-    if (GH->dim == 3)
-    {
-      pGF_PostSend3(GH,GF,dir);
-    }
-    else if (GH->dim == 1)
-    {
-      pGF_PostSend1(GH,GF,dir);
-    }
-    else
-    {
-      CCTK_WARN(0,"PUGH does not yet support this dimension");
-    }
-
-    tp = MPI_Wtime();
-    
-    /* post send */
-    CACTUS_MPI_ERROR (MPI_Isend (GF->send_buffer [dir],
-                      GF->buffer_sz [dir], mpi_type,
-                      GH->neighbors [GH->myproc][dir], stag,
-                      GH->PUGH_COMM_WORLD, &(GF->sreq [dir])));
-    tw = MPI_Wtime();
-#ifdef COMM_TIMING
-    printf ("GF %s Dir %d Time %f\n", GF->name, dir, tw-ts);
-#endif
-  }
-#endif
-}
-
-void pGF_PostSend3(pGH *GH, pGF *GF, int dir)
-{
-
-  int ii,jj,kk,xx;
-  int istart, iend;
-  int jstart, jend;
-  int kstart, kend;
-  CCTK_CHAR *char_data;
-  CCTK_INT *int_data;
-  CCTK_REAL *real_data;
-  CCTK_COMPLEX *complex_data;
-
-  /* Copy my information into the send buffer */
-  istart = GH->overlap[GF->stagger][0][dir][0];
-  iend   = GH->overlap[GF->stagger][1][dir][0];
-  jstart = GH->overlap[GF->stagger][0][dir][1];
-  jend   = GH->overlap[GF->stagger][1][dir][1];
-  kstart = GH->overlap[GF->stagger][0][dir][2];
-  kend   = GH->overlap[GF->stagger][1][dir][2];
-    
-  /* Great now copy */
-  xx = 0;
-  switch (GF->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      char_data = (CCTK_CHAR *) GF->send_buffer [dir];
-      for (kk=kstart; kk<kend; kk++)
-      {
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            char_data [xx++] =
-              ((CCTK_CHAR *) GF->data) [DATINDEX (GH,ii,jj,kk)];
-          }
-        }
-      }
-      break;
-    
-    case CCTK_VARIABLE_INT:
-      int_data = (CCTK_INT *) GF->send_buffer [dir];
-      for (kk=kstart; kk<kend; kk++)
-      { 
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            int_data [xx++] =
-              ((CCTK_INT *) GF->data) [DATINDEX (GH,ii,jj,kk)];
-          }
-        }
-      }
-      break;
-    
-    case CCTK_VARIABLE_REAL:
-      real_data = (CCTK_REAL *) GF->send_buffer [dir];
-      for (kk=kstart; kk<kend; kk++)
-      { 
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            real_data [xx++] =
-              ((CCTK_REAL *) GF->data) [DATINDEX (GH,ii,jj,kk)];
-          }
-        }
-      }
-      break;
-    
-    case CCTK_VARIABLE_COMPLEX:
-      complex_data = (CCTK_COMPLEX *) GF->send_buffer [dir];
-      for (kk=kstart; kk<kend; kk++)
-      { 
-        for (jj=jstart; jj<jend; jj++)
-        {
-          for (ii=istart; ii<iend; ii++)
-          {
-            complex_data [xx++] =
-              ((CCTK_COMPLEX *) GF->data) [DATINDEX (GH,ii,jj,kk)];
-          }
-        }
-      }
-      break;
-  }
-}
-
-
-
-void pGF_PostSend1(pGH *GH, pGF *GF, int dir)
-{
-
-  int ii,xx;
-  int istart, iend;
-  CCTK_CHAR *char_data;
-  CCTK_INT *int_data;
-  CCTK_REAL *real_data;
-  CCTK_COMPLEX *complex_data;
-
-  /* Copy my information into the send buffer */
-  istart = GH->overlap[GF->stagger][0][dir][0];
-  iend   = GH->overlap[GF->stagger][1][dir][0];
-    
-  /* Great now copy */
-  xx = 0;
-  switch (GF->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      char_data = (CCTK_CHAR *) GF->send_buffer [dir];
-      for (ii=istart; ii<iend; ii++)
-      {
-        char_data [xx++] =
-          ((CCTK_CHAR *) GF->data) [ii];
-      }
-      break;
-    
-    case CCTK_VARIABLE_INT:
-      int_data = (CCTK_INT *) GF->send_buffer [dir];
-      for (ii=istart; ii<iend; ii++)
-      {
-        int_data [xx++] =
-          ((CCTK_INT *) GF->data) [ii];
-      }
-      break;
-    
-    case CCTK_VARIABLE_REAL:
-      real_data = (CCTK_REAL *) GF->send_buffer [dir];
-      for (ii=istart; ii<iend; ii++)
-      {
-        real_data [xx++] =
-          ((CCTK_REAL *) GF->data) [ii];
-      }
-      break;
-    
-    case CCTK_VARIABLE_COMPLEX:
-      complex_data = (CCTK_COMPLEX *) GF->send_buffer [dir];
-      for (ii=istart; ii<iend; ii++)
-      {
-        complex_data [xx++] =
-          ((CCTK_COMPLEX *) GF->data) [ii];
-      }
-      break;
-  }
-}
diff --git a/src/pugh_BoundingBox.c b/src/pugh_BoundingBox.c
deleted file mode 100644
index 93acbe4..0000000
--- a/src/pugh_BoundingBox.c
+++ /dev/null
@@ -1,515 +0,0 @@
- /*@@
-   @file      pugh_BoundingBox.c
-   @date      Sun Oct 24 10:43:18 CEST 1999
-   @author    Paul Walker, Gabrielle Allen
-   @desc 
-   Set up bounding box information for a GH for different dimension grids.
-   @enddesc 
-   @version $Header$
- @@*/
-
-static char *rcsid = "$Header$";
-
-/* #define DEBUG_BB */
-
-#include <stdlib.h>
-
-#include "pugh.h"
-
-#include "cctk.h"
-#include "cctk_Parameters.h"
-
-
-void BoundingBox1D(pGH *GH, int *nproc, int staggertype);
-void BoundingBox3D(pGH *GH, int *nproc, int staggertype);
-void BoundingBox1D_local(pGH *GH, int *nproc, int staggertype);
-void BoundingBox3D_local(pGH *GH, int *nproc, int staggertype);
-
-
-void BoundingBox(pGH *GH, int *nproc, int staggertype)
-{
-  switch (GH->dim) 
-  {
-    case 1:
-      if (nproc[0]>=0)
-      {
-        BoundingBox1D(GH,nproc,staggertype);
-      }
-      else
-      {
-          BoundingBox1D_local(GH,nproc,staggertype);
-      }
-      break;
-    case 3:
-      if (nproc[0]>=0)
-      {
-        BoundingBox3D(GH,nproc,staggertype);
-      }
-      else
-      {
-        BoundingBox3D_local(GH,nproc,staggertype);
-      }
-      break;
-case 0:
-	break;
-    default:
-      CCTK_WARN(0,"PUGH can't cope with this dimension grid yet");
-      return;
-  }
-}
-
-void BoundingBox3D(pGH *GH, int *nproc, int staggertype)
-{
-  DECLARE_CCTK_PARAMETERS                                                   
-
-  int i,j,k;
-  int *lX, *lY, *lZ;
-
-#ifdef DEBUG_BB                                                                
-  printf("Bounding_Box3D (%d:%d:%d)->%d\n",
-         nproc[0], nproc[1], nproc[2], GH->nprocs); 
-#endif                                                                  
-
-  if (GetSliceSizes(nproc[0], GH->nsize[0], partition_3d_X, &lX, 
-                   CCTK_Equals(partition, "manual"))) {
-#ifdef MPI 
-    MPI_Finalize();       
-#endif
-    exit(0); 
-  }
-                                                                           
-  if (GetSliceSizes(nproc[1], GH->nsize[1], partition_3d_Y, &lY,
-                    CCTK_Equals(partition, "manual"))) { 
-#ifdef MPI 
-    MPI_Finalize(); 
-#endif
-    exit(0);  
-  }                                                                           
- 
-  if (GetSliceSizes(nproc[2], GH->nsize[2], partition_3d_Z, &lZ,
-                    CCTK_Equals(partition, "manual"))) {
-#ifdef MPI 
-    MPI_Finalize(); 
-#endif
-    exit(0);   
-  }                                                                           
-
-  for (i=0;i<nproc[0];i++)
-  {
-    for (j=0;j<nproc[1];j++)
-    {
-      for (k=0;k<nproc[2];k++) 
-      {
-	int pnum = i + nproc[0]*(j+nproc[1]*k);
-	
-	/* Bounding box information */
-	if (i == 0)
-	{
-	  GH->lb[pnum][0] = 0;
-	}
-	else            
-	{
-	  GH->lb[pnum][0] = lX[i] +1 - GH->nghostzones[0];
-	  if(staggertype == PUGH_STAGGER)
-	  GH->lb[pnum][0] --;
-	}
-
-	if (i == nproc[0]-1) 
-	{
-	  GH->ub[pnum][0] = GH->nsize[0]-1;
-	}
-	else
-	{
-	  GH->ub[pnum][0] = lX[i+1] + GH->nghostzones[0];
-	}
-
-	if (j == 0)
-	{ 
-	  GH->lb[pnum][1] = 0;
-	}
-	else
-	{            
-	    GH->lb[pnum][1] = lY[j] +1 - GH->nghostzones[1];
-	    if(staggertype == PUGH_STAGGER)
-	    {
-	      GH->lb[pnum][1] --;
-	    }
-	}
-	if (j == nproc[1]-1) 
-	{
-	  GH->ub[pnum][1] = GH->nsize[1]-1;
-	}
-	else
-	{
-	  GH->ub[pnum][1] = lY[j+1] + GH->nghostzones[1];
-	}
-
-	if (k == 0)
-	{
-	  GH->lb[pnum][2] = 0;
-	}
-	else             
-	{
-	  GH->lb[pnum][2] = lZ[k] +1 - GH->nghostzones[2];
-	  if(staggertype == PUGH_STAGGER)
-	  {
-	    GH->lb[pnum][2] --;
-	  }
-	}
-	if (k == nproc[2]-1) 
-	{
-	  GH->ub[pnum][2] = GH->nsize[2]-1;
-	}
-	else
-	{ 
-	  GH->ub[pnum][2] = lZ[k+1] + GH->nghostzones[2];
-	}
-      }
-    }
-  }
-  free(lX); free(lY); free(lZ);
-
-#ifdef DEBUG_BB                        
-  for(i=0; i<GH->nprocs; i++)                                             
-    printf(" Bounding_Box3D(%d) X(%d,%d) Y(%d,%d) Z(%d,%d)\n"
-           ,i, GH->lb[i][0], GH->ub[i][0],
-           GH->lb[i][1], GH->ub[i][1],
-           GH->lb[i][2], GH->ub[i][2]);
-#endif                                     
-}
-
-
-void BoundingBox3D_local(pGH *GH, int *nproc, int staggertype)
-{
-  int i,j,k,idir;
-  int *step;
-
-#ifdef DEBUG_BB                                                                 
-  printf("Bounding_Box3D_local (%d:%d:%d)->%d\n",
-	 nproc[0], nproc[1], nproc[2], GH->nprocs);                             
-#endif                                                                          
-
-  step = (int *)malloc(GH->dim*sizeof(int));
-  for (idir=0;idir<GH->dim;idir++)
-  {
-    step[idir] = (GH->nsize[idir]-1) / nproc[idir];
-  }
-       
-  for (i=0;i<nproc[0];i++)
-  {
-    for (j=0;j<nproc[1];j++)
-    {
-      for (k=0;k<nproc[2];k++) 
-      {
-        int pnum = i + nproc[0]*(j+nproc[1]*k);
-          
-        if (i == 0)
-        {
-          GH->lb[pnum][0] = 0;
-        }
-        else        
-        {
-          GH->lb[pnum][0] = GH->ub[pnum-1][0] + 1 - 
-            2*(GH->nghostzones[0]);
-          if(staggertype == PUGH_STAGGER) GH->lb[pnum][0] --;
-        }
-        GH->ub[pnum][0] = GH->lb[pnum][0] + abs(GH->nsize[0]) - 1;
-  
-        if (j == 0)  
-        {
-          GH->lb[pnum][1] = 0;
-        }
-        else        
-        {
-          GH->lb[pnum][1] = GH->ub[pnum-nproc[0]][1] + 1 - 
-            2*(GH->nghostzones[1]);
-          if(staggertype == PUGH_STAGGER) GH->lb[pnum][1] --;
-        }
-        GH->ub[pnum][1] = GH->lb[pnum][1] + abs(GH->nsize[1]) - 1;
-          
-        if (k == 0)  
-        {
-          GH->lb[pnum][2] = 0;
-        }
-        else        
-        {
-          GH->lb[pnum][2] = GH->ub[pnum-nproc[0]*nproc[1]][2] + 1 - 
-            2*(GH->nghostzones[2]);
-          if(staggertype == PUGH_STAGGER) GH->lb[pnum][2] --;
-        }
-        GH->ub[pnum][2] = GH->lb[pnum][2] + abs(GH->nsize[2]) - 1;
-          
-      }
-    }
-  }
-  
-}
-
-
-
-void BoundingBox1D(pGH *GH, int *nproc, int staggertype)
-{
-  int i,idir;
-  int *step;
-
-  step = (int *)malloc(GH->dim*sizeof(int));
-  for (idir=0;idir<GH->dim;idir++)
-  {
-    step[idir] = (GH->nsize[idir]-1) / nproc[idir];
-  }
-      
-  for (i=0;i<nproc[0];i++)
-  {
-    if (i == 0)      
-    {
-      GH->lb[i][0] = 0;
-    }
-    else
-    {
-      GH->lb[i][0] = i*step[0] +1 - GH->nghostzones[0];
-      if(staggertype == PUGH_STAGGER)
-      {
-        GH->lb[i][0] --;
-      }
-    }
-    if (i == nproc[0]-1) 
-    {
-      GH->ub[i][0] = GH->nsize[0]-1;
-    }
-    else
-    {             
-      GH->ub[i][0] = (i+1)*step[0] + GH->nghostzones[0];
-    }
-
-  }
-
-  
-}
-
-
-
-void BoundingBox1D_local(pGH *GH, int *nproc, int staggertype)
-{
-  int i,idir;
-  int *step;
-
-  step = (int *)malloc(GH->dim*sizeof(int));
-  for (idir=0;idir<GH->dim;idir++)
-  {
-    step[idir] = (GH->nsize[idir]-1) / nproc[idir];
-  }
-       
-  for (i=0;i<nproc[0];i++)
-  {
-    if (i == 0)
-    {
-      GH->lb[i][0] = 0;
-    }
-    else        
-    {
-      GH->lb[i][0] = GH->ub[i-1][0] + 1 - 
-        2*(GH->nghostzones[0]);
-      if(staggertype == PUGH_STAGGER) GH->lb[i][0] --;
-    }
-    GH->ub[i][0] = GH->lb[i][0] + abs(GH->nsize[0]) - 1;
-  
-  }
-  
-}
-
-
-
-void Neighbours3D(pGH *GH, int *nproc, int *maxpoints, int *minpoints, double *avgpoints)
-{
-  int i,j,k,index,pnum;
-  int dim;
-
-  dim = 3;
-
-  for (i=0;i<nproc[0];i++)
-  {
-    for (j=0;j<nproc[1];j++)
-    {
-      for (k=0;k<nproc[2];k++) 
-      {
-        pnum = i + nproc[0]*(j+nproc[1]*k);
-
-        /* Neighbors are easy! Just recall that i dir is x dir etc ...*/
-        if (i == nproc[0]-1)
-        { 
-          if (GH->periodic)
-          {
-            /* Use the i = 0 processor */
-            GH->neighbors[pnum][XDP] = 0 + nproc[0]*(j+nproc[1]*k);
-          }
-          else
-          {
-            GH->neighbors[pnum][XDP] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][XDP] = (i+1) + nproc[0]*(j+nproc[1]*k);
-        }
-        if (i == 0)
-        {
-          if (GH->periodic) 
-          {
-            /* Use the nprocx processor */
-            GH->neighbors[pnum][XDM] = nproc[0]-1 + nproc[0]*(j+nproc[1]*k);
-          }
-          else
-          {
-            GH->neighbors[pnum][XDM] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][XDM] = (i-1) + nproc[0]*(j+nproc[1]*k);
-        }
-
-        if (j == nproc[1]-1)
-        {
-          if (GH->periodic)
-          {
-            /* Use the 0 processor */
-            GH->neighbors[pnum][YDP] = (i) + nproc[0]*(0+nproc[1]*k);
-          }
-          else
-          {
-            GH->neighbors[pnum][YDP] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][YDP] = (i) + nproc[0]*(j+1+nproc[1]*k);
-        }
-        if (j == 0)
-        { 
-          if (GH->periodic)
-          {
-            /* Use the nprocy-1 processor */
-            GH->neighbors[pnum][YDM] = (i) + nproc[0]*(nproc[1]-1+nproc[1]*k);
-          }
-          else
-          {
-            GH->neighbors[pnum][YDM] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][YDM] = (i) + nproc[0]*(j-1+nproc[1]*k);
-        }
-
-        if (k == nproc[2]-1)
-        {
-          if (GH->periodic)
-          {
-            /* Use the 0 processor */
-            GH->neighbors[pnum][ZDP] = (i) + nproc[0]*(j+nproc[1]*0);
-          }
-          else
-          {
-            GH->neighbors[pnum][ZDP] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][ZDP] = (i) + nproc[0]*(j+nproc[1]*(k+1));
-        }
-
-        if (k == 0)
-        {
-          if (GH->periodic)
-          {
-            /* Use the nprocz-1 processor */
-            GH->neighbors[pnum][ZDM] = (i) + nproc[0]*(j+nproc[1]*(nproc[2]-1));
-          }
-          else
-          {
-            GH->neighbors[pnum][ZDM] = -1;
-          }
-        }
-        else
-        {
-          GH->neighbors[pnum][ZDM] = (i) + nproc[0]*(j+nproc[1]*(k-1));
-        }
-
-        /* And save the number of points on each processor */
-        GH->rnpoints[pnum] = 1;
-        for (index=0;index<dim;index++)
-        {
-          GH->rnsize[pnum][index] = (GH->ub[pnum][index]-GH->lb[pnum][index]+1);
-          GH->rnpoints[pnum] = GH->rnpoints[pnum]*GH->rnsize[pnum][index];
-        }
-
-        if (GH->rnpoints[pnum] > *maxpoints) *maxpoints = GH->rnpoints[pnum];
-        if (GH->rnpoints[pnum] < *minpoints) *minpoints = GH->rnpoints[pnum];
-        *avgpoints += GH->rnpoints[pnum];
-
-      }
-    }
-  }
-}
-
-
-
-void Neighbours1D(pGH *GH, int *nproc, int *maxpoints, int *minpoints, double *avgpoints)
-{
-  int i,index;
-  int dim;
-
-  dim = 1;
-
-  for (i=0;i<nproc[0];i++)
-  {
-
-    /* Neighbors are easy! Just recall that i dir is x dir etc ...*/
-    if (i == nproc[0]-1) 
-    {
-      if (GH->periodic)
-      {
-        /* Use the i = 0 processor */
-        GH->neighbors[i][XDP] = 0;
-      }
-      else
-      {
-        GH->neighbors[i][XDP] = -1;
-      }
-    }
-    else
-    {
-      GH->neighbors[i][XDP] = (i+1);
-    }
-
-    if (i == 0)
-    {
-      if (GH->periodic) 
-      {
-        /* Use the nprocx processor */
-        GH->neighbors[i][XDM] = nproc[0]-1;
-      }
-      else
-      {
-        GH->neighbors[i][XDM] = -1;
-      }
-    }
-    else
-    {
-      GH->neighbors[i][XDM] = (i-1);
-    }
-
-    /* And save the number of points on each processor */
-    GH->rnpoints[i] = 1;
-    for (index=0;index<dim;index++)
-    {
-      GH->rnsize[i][index] = (GH->ub[i][index]-GH->lb[i][index]+1);
-      GH->rnpoints[i] = GH->rnpoints[i]*GH->rnsize[i][index];
-    }
-    
-    if (GH->rnpoints[i] > *maxpoints) *maxpoints = GH->rnpoints[i];
-    if (GH->rnpoints[i] < *minpoints) *minpoints = GH->rnpoints[i];
-    *avgpoints += GH->rnpoints[i];
-  }
-
-}
diff --git a/src/pugh_ProcTop.c b/src/pugh_ProcTop.c
deleted file mode 100644
index 4ea8da8..0000000
--- a/src/pugh_ProcTop.c
+++ /dev/null
@@ -1,169 +0,0 @@
- /*@@
-   @file      pugh_ProcTop.c
-   @date      Sat Oct 16 20:10:02 CEST 1999 
-   @author    Paul Walker
-   @desc 
-   Set up the processor topology for different GH dimensions
-   @enddesc 
-   @version $Header$
- @@*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-
-#include "cctk.h"
-#include "cctk_Parameters.h"
-
-#include "pugh.h"
-
-void ProcTop3D(int np, int *nproc);
-void ProcTop1D(int np, int *nproc);
-
-static char *rcsid = "$Header$";
-
-
- /*@@
-   @routine    ProcTop
-   @date       Thu May  1 10:13:40 1997
-   @author     Paul Walker
-   @desc 
-   Figures out a reasonable processor topology for a square
-   grid on n processors. That is, it figures out a,b,c such
-   that a*b*c = n and a,b, and c are in some senses minimal.
-   <p>
-   This algorithm will not guarantee the minimal condition,
-   or at least I can't prove it does, but it seems to on
-   a wide variety of processor numbers. So stop being a
-   weenie - not everything needs a proof!
-   <p>
-   Thie came to me in the shower, and is much better than
-   the old way I was doing it. I'm considering having my
-   working environment be the shower for the rest of time,
-   but there are some problems waterproofing my workstation.
-   @enddesc 
-   @calledby   SetupPGH
-@@*/
-
-void ProcTop(int np, int dim, int *nproc) 
-{
-
-  /* Set up processor topology information */
-  switch (dim) 
-  {
-    case 1:
-      ProcTop1D(np, nproc);
-      break;
-    case 3:
-      ProcTop3D(np, nproc);
-      break;
-    default:
-      CCTK_WARN(0,"PUGH can't cope with this dimension grid yet");
-      return;
-  }
-  
-}
-
-void ProcTop3D(int np, int *nproc)
-{
-  DECLARE_CCTK_PARAMETERS
-
-  int r,dim;
-  int *dir;
-
-  dim = 3;
-
-  dir = (int *)malloc(dim*sizeof(int));
-
-  if (CCTK_Equals(proc_topology,"manual")) 
-  {
-    dir[0] = proc_top_nz;
-    dir[1] = proc_top_ny;
-    dir[2] = proc_top_nx;
-    if (dir[0] * dir[1] * dir[2] != np) 
-    {
-      fprintf (stderr,
-               "Manual processor topology error. %d x %d x %d != %d\n",
-               dir[0],dir[1],dir[2],np);
-#ifdef MPI
-      CACTUS_MPI_ERROR(MPI_Finalize());
-#endif
-      exit(0);
-    }
-  } 
-  else 
-  {
-    dir[2] = (int)(pow(np,1.0/3.0)+0.0001);
-    while (np % dir[2] != 0) dir[2]--;
-    r = np / dir[2];
-    dir[1] = (int)(sqrt(r)+0.0001);
-    while (r % dir[1] != 0) dir[1]--;
-    dir[0] = r / dir[1];
-
-#ifdef THORN_CARTOON_2D
-    if (Contains("cartoon_active","yes")) 
-    {
-      dir[2] = (int)(sqrt(np)+0.0001);
-      while (np % dir[2] != 0) dir[2]--;
-      dir[1] = 1;
-      dir[0] = np / dir[2];
-    }
-#endif
-    
-    if (dir[0] * dir[1] * dir[2] != np) 
-    {
-      printf ("FATAL ERROR in Processor Topology\n");
-      printf ("This should never happen. Please report ASAP!\n");
-      exit(0);
-    }
-  }
-
-  /* NB: PW reverses these for better alignment of comm groups */
-  
-  nproc[2] = dir[0];
-  nproc[1] = dir[1];
-  nproc[0] = dir[2];
-
-  free(dir);
-}
-
-void ProcTop1D(int np, int *nproc)
-{
-
-  DECLARE_CCTK_PARAMETERS
-
-  int dim;
-  int *dir;
-
-  dim = 1;
-
-  dir = (int *)malloc(dim*sizeof(int));
-
-  if (CCTK_Equals(proc_topology,"manual")) 
-  {
-    dir[0] = proc_top_nx;
-    if (dir[0] != np) 
-    {
-      fprintf (stderr,
-               "Manual processor topology error. %d != %d\n",
-               dir[0],np);
-#ifdef MPI
-      CACTUS_MPI_ERROR(MPI_Finalize());
-#endif
-      exit(0);
-    }
-
-  } 
-  else 
-  {
-    dir[0] = np;
-  }
-
-  nproc[0] = dir[0];
-
-  free(dir);
-
-}
-
-
-