Enabled PUGH to synchronize groups of arrays in one communication call

rather than doing it sequentially on individual arrays.
Nothing changes for the CCTK routines overloaded by PUGH.

For BAM and other thorns which don't care about groups there is now
a separate interface to synchronize individual arrays.

Also eliminated the restriction to synchronize 1D and 3D arrays only
- now it should do arbitrary dims (not yet tested).


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@222 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 488 insertions, 778 deletions

diff --git a/src/Comm.c b/src/Comm.c
index 528ec63..31e2350 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -14,15 +14,11 @@
 #include <stdio.h>
 #include <stdarg.h> 
 #include <string.h>
-#include <assert.h>
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
 
 #include "cctk.h"
 
 #include "pugh.h"
+#include "pughi.h"
 #include "pugh_Comm.h"
 #include "cctk_Parameters.h"
 
@@ -30,41 +26,23 @@ static char *rcsid="$Header$";
 
 CCTK_FILEVERSION(CactusPUGH_PUGH_Comm_c)
 
-static int totalstorage=0; /* Storage for GAs in Bytes */
-static int totalnumber=0; /* Number of stored GAs */
-
-/* index into GH->extensions[] to get the PUGH extension handle */
-extern int pugh_GHExtension;
 
 /* local function prototypes */
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+                                      int first_var,
+                                      int commflag);
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+                                       int first_var);
+static int PUGH_EnableComm(pGH *pughGH,
+                           pComm *comm,
+                           int commflag);
+static int PUGH_DisableComm(pGH *pughGH,
+                            pComm *comm);
+static int PUGH_SyncGArrayGroup(pGH *pughGH,
+                                int first_var);
+static int PUGH_Sync(pGH *pughGH,
+                     pComm *comm);
 
-int PUGH_SyncGroupGA (cGH *GH, 
-                      int group, 
-                      int vtype, 
-                      int n_vars, 
-                      int timelevel);
-int PUGH_SyncGAs(pGH *pughGH, 
-                 int dim, 
-                 int first_var, 
-                 int n_vars, 
-                 int timelevel);
-int PUGH_EnableArrayGroupStorage(cGH *GH,                              
-                                 int group,
-                                 int vtype,
-                                 int dim,
-                                 int n_variables,
-                                 int n_timelevels);
-int PUGH_EnablePGAStorage(pGA *GA, 
-                          int this_proc,
-                          int zero_memory, 
-                          int padding_active,
-                          int padding_cacheline_bits,
-                          int padding_size,
-                          int padding_address_spacing);
-void FinishReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostSendGA(pGH *GH, pGA *GA, int dir);
-int PUGH_GAComm(pGA *GA, int docomm);
 
 /*@@
   @routine PUGH_SyncGroup
@@ -75,7 +53,7 @@ int PUGH_GAComm(pGA *GA, int docomm);
   Only groups of type GROUP_ARRAY and GROUP_GF can be synchronized.
   @enddesc
   @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-           PUGH_SyncGroupArray PUGH_SyncGroupGF
+           PUGH_SyncGArrayGroup
   @history
  
   @endhistory 
@@ -91,66 +69,48 @@ int PUGH_GAComm(pGA *GA, int docomm);
   @endvar 
   @@*/
 
-int PUGH_SyncGroup (cGH *GH, const char *groupname)
+int PUGH_SyncGroup(cGH *GH, const char *groupname)
 {
-  cGroup pgroup;       /* pointer to group information */
+  cGroup pgroup;           /* group information */
   int group;               /* group index */
-  int timelevel;           /* timelevel to be synchronized */
   int rc;                  /* return code */
 
-  group = CCTK_GroupIndex(groupname);
+#ifdef DEBUG_PUGH
+  printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
+  fflush (stdout);
+#endif
 
   /* get the group info from its index */
-  CCTK_GroupData (group,&pgroup);
+  group = CCTK_GroupIndex(groupname);
+  CCTK_GroupData(group, &pgroup);
 
-#if 0
-  /* Compute the timelevel to synchronize */
-  if (pgroup.numtimelevels > 1)
+  if (pgroup.grouptype == CCTK_SCALAR)
   {
-    timelevel = pgroup.numtimelevels - 2;
+    rc = 0;
+    CCTK_WARN(4, "Synchronising a scalar in PUGH");
   }
-  else
+  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
   {
-    timelevel = pgroup.numtimelevels - 1;
+    rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
   }
-#endif
-
-  timelevel = pgroup.numtimelevels - 1;
-
-  /* branch to synchronization function according to group type */
-  switch (pgroup.grouptype) 
+  else
   {
-    case GROUP_SCALAR :
-      rc = 0;
-      CCTK_WARN(4,"Synchronising a scalar in PUGH");
-      break;
-
-    case GROUP_ARRAY  :
-    case GROUP_GF     :
-      rc = PUGH_SyncGroupGA (GH, 
-                             group, 
-                             pgroup.vartype, 
-                             pgroup.numvars, 
-                             timelevel);
-      break;
-
-    default :
-      CCTK_WARN (1, "Unknown group type in PUGH_SyncGroup");
-      rc = 0;
-      break;
+    CCTK_WARN(1, "Unknown group type in PUGH_SyncGroup");
+    rc = 0;
   }
 
   return (rc);
 }
 
+
 /*@@
-  @routine PUGH_ArrayGroupSize
-  @author  Gabrielle Allen
-  @date    11 Apr 1999
+  @routine PUGH_EnableGroupComm
+  @author  Thomas Radke
+  @date    30 Mar 1999
   @desc
-  Returns size of arrays in a group, in a given direction
+  Enables communication for all variables in the group indicated by groupname.
   @enddesc
-  @calls   
+  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
   @history
  
   @endhistory 
@@ -163,109 +123,52 @@ int PUGH_SyncGroup (cGH *GH, const char *groupname)
   @vdesc   name of the group to be synchronized
   @vtype   const char *
   @vio     in
-  @vcomment either index or groupname must be present
   @endvar 
-  @var     group
-  @vdesc   index of group
-  @vtype   int
-  @vio     in
-  @vcomment either index or groupname must be given
-  @endvar
-  @var      dir
-  @vdesc    direction
-  @vtype    int
-  @vio      in
   @@*/
 
-const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
+int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
 {
-  int first;
-  int *sizep=NULL;
-  pGH *pughGH;
-  pGA *pughGA;
-
-  if (groupname)
-  {
-    group = CCTK_GroupIndex(groupname);
-  }
+  int group;               /* group index */
+  cGroup pgroup;           /* group information */
+  int rc;                  /* return code */
 
-  /* get PUGH extension handle */
-  pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+#ifdef DEBUG_PUGH
+  printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
+  fflush(stdout);
+#endif
 
-  /* get first variable in group */
-  first = CCTK_FirstVarIndexI(group);
+  /* get the group info from its index */
+  group = CCTK_GroupIndex(groupname);
+  CCTK_GroupData(group, &pgroup);
 
-  if (CCTK_GroupTypeI(group)==GROUP_GF)
+  if (pgroup.grouptype == CCTK_SCALAR)
   {
-    /* get pointer to pGF for a timelevel of this variable */
-    pughGA = (pGA *)(pughGH->variables[first][0]);  
-
-    if (pughGA->storage == PUGH_STORAGE)
-    {
-      switch (dir) 
-      {
-        case 0:
-          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[0];
-          break;
-        case 1:
-          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[1];
-          break;
-        case 2:
-          sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[2];
-          break;
-        default :
-          sizep = NULL;
-          printf("Wrong value for dir in PUGH_ArrayGroupSize\n");
-          break;
-      }
-    }
-    else
-    {
-      sizep = &_cctk_one;
-    }
+    rc = 1;
   }
-  else if (CCTK_GroupTypeI(group)==GROUP_ARRAY)
+  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
   {
-    /* get pointer to pGA for a timelevel of this variable */
-    pughGA = (pGA *)(pughGH->variables[first][0]);  
-    
-    if (pughGA->storage == PUGH_STORAGE)
-    {
-      switch (dir) 
-      {
-        case 0:
-          sizep = &pughGA->extras->lnsize[0];
-          break;
-        case 1:
-          sizep = &pughGA->extras->lnsize[1];
-          break;
-        case 2:
-          sizep = &pughGA->extras->lnsize[2];
-          break;
-        default :
-          sizep = NULL;
-          CCTK_WARN(1,"Wrong value for dir in PUGH_ArrayGroupSize");
-          break;
-      }
-    }
-    else
-    {
-      sizep = &_cctk_one;
-    }
+    rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
+                                    CCTK_FirstVarIndexI(group),
+                                    PUGH_ALLCOMM);
+  }
+  else
+  {
+    CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
+    rc = 0;
   }
 
-  return (sizep);
+  return (rc);
 }
 
 
 /*@@
-  @routine PUGH_QueryGroupStorage
-  @author  Gabrielle Allen
-  @date    13 Apr 1999
+  @routine PUGH_DisableGroupComm
+  @author  Thomas Radke
+  @date    30 Mar 1999
   @desc
-  Returns true if group has storage assigned, false otherwise
+  Disables communication for all variables in the group indicated by groupname.
   @enddesc
-  @calls   CCTK_DecomposeName
+  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
   @history
  
   @endhistory 
@@ -275,595 +178,523 @@ const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupna
   @vio     in
   @endvar 
   @var     groupname
-  @vdesc   name of the group to be queried
+  @vdesc   name of the group to be synchronized
   @vtype   const char *
   @vio     in
-  @vcomment either index or groupname must be present
   @endvar 
-  @var     group
-  @vdesc   index of group
-  @vtype   int
-  @vio     in
-  @vcomment either index or groupname must be given
-  @endvar
   @@*/
 
-int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
+int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
 {
+  int group;               /* group index */
+  cGroup pgroup;           /* pointer to group information */
+  int rc;                  /* return code */
 
-  int first;
-  int storage=PUGH_UNDEFINEDSTORAGE;
-  int retval;
-  int grouptype;
-  pGH *pughGH;
-
-  if (groupname)
-  {
-    group = CCTK_GroupIndex(groupname);
-  }
-
-  /* get first variable in group */
-  first = CCTK_FirstVarIndexI(group);
-  if (first < 0) 
-  {
-    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
-                "Invalid group ID %d in PUGH_QueryGroupStorage", group);
-  }
- 
-  /* get PUGH extension handle */
-  pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-
-  grouptype = CCTK_GroupTypeI(group);
+#ifdef DEBUG_PUGH
+  printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
+  fflush(stdout);
+#endif
 
-  if (grouptype == CCTK_SCALAR)
-  {
-    storage = PUGH_STORAGE;
-  }
-  else if (grouptype == CCTK_GF)
-  {
-    storage = ((pGA *)(pughGH->variables[first][0]))->storage;  
-  }
-  else if (grouptype == CCTK_ARRAY)
-  {
-    storage = ((pGA *)(pughGH->variables[first][0]))->storage;  
-  }
-  else
-  {
-    CCTK_WARN(0,"Unknown group type in PUGH_QueryGroupStorage");
-  }
+  /* get the group info from its index */
+  group = CCTK_GroupIndex(groupname);
+  CCTK_GroupData(group, &pgroup);
 
-  if (storage == PUGH_STORAGE)
+  if (pgroup.grouptype == CCTK_SCALAR)
   {
-    retval = 1;
+    rc = 1;
   }
-  else if (storage == PUGH_NOSTORAGE)
+  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
   {
-    retval = 0;
+    rc = PUGH_DisableGArrayGroupComm(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
   }
   else
   {
-    CCTK_WARN(0,"Inconsistency in PUGH_QueryGroupStorage");
-    retval = PUGH_ERROR;
+    CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
+    rc = 0;
   }
 
-  return (retval);
+  return (rc);
 }
 
 
-/*@@
-  @routine PUGH_EnableGroupStorage
-  @author  Tom Goodale
-  @date    30 Mar 1999
-  @desc
-  Enables storage for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-           PUGH_EnableArrayGroupStorage
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
-
-int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+   @routine    PUGH_EnableGArrayComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Enables communication for a single array.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+int PUGH_EnableGArrayComm(pGA *GA,
+                          int commflag)
 {
 
-  DECLARE_CCTK_PARAMETERS
-
-  int group;               /* group index */
-  cGroup pgroup;       /* pointer to group information */
-  int rc;                  /* return code */
-  pGH *pughGH;
-  int first_var;
-
 #ifdef DEBUG_PUGH
-  printf(" PUGH_EnableGroupStorage: request for group -%s-\n",groupname);
+  printf(" PUGH_EnableGArrayComm: request for var '%s' commflag %d\n",
+          GA->name, commflag);
   fflush(stdout);
 #endif
 
-  group = CCTK_GroupIndex(groupname);
-
-  /* get the group info from its index */
-  CCTK_GroupData (group, &pgroup);
-
-  /* branch to "enable storage" function according to group type */
-  switch (pgroup.grouptype) 
-  {
-    case CCTK_SCALAR :
-      rc = 1;
-      break;
-
-    case CCTK_ARRAY  :
-    case CCTK_GF     :
-      rc = PUGH_EnableArrayGroupStorage (GH, 
-                                         group, 
-                                         pgroup.vartype, 
-                                         pgroup.dim, 
-                                         pgroup.numvars,
-                                         pgroup.numtimelevels);
-      break;
-
-    default :
-      CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupStorage");
-      rc = 0;
-      break;
-  }
-
-  /* Report on memory usage */
-  if (storage_verbose)
-  {
-    /* get PUGH extension handle */
-    pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+  return (PUGH_EnableComm((pGH *) GA->parent, GA->comm, commflag));
+}
 
-    /* get global index of first variable in group */
-    first_var = CCTK_FirstVarIndexI (group);
 
-    if (rc == 0)
-    {
-      /* Memory toggled */
-      totalnumber = totalnumber + pgroup.numvars;
-      totalstorage = totalstorage +
-        ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
-        *CCTK_VarTypeSize(pgroup.vartype)
-        *pgroup.numtimelevels
-        *pgroup.numvars;
-      printf("Switched memory on for %s \n  [Num Arrays: %d Total Size: %d]\n",
-             groupname,totalnumber,totalstorage);
-    }
-    else if (rc == 1)
-    {
-      /* Memory already on */
-      printf("Memory already on for %s\n",groupname);
-    }
-  }
+ /*@@
+   @routine    PUGH_DisableGArrayComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Disables communication for a single array.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+int PUGH_DisableGArrayComm(pGA *GA)
+{
 
-  return (rc);
+#ifdef DEBUG_PUGH
+  printf(" PUGH_DisableGArrayComm: request for var '%s'\n", GA->name);
+  fflush(stdout);
+#endif
 
+  return (PUGH_DisableComm((pGH *) GA->parent, GA->comm));
 }
 
 
-/*@@
-  @routine PUGH_DisableGroupStorage
-  @author  Tom Goodale
-  @date    30 Mar 1999
-  @desc
-  Disables storage for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
-
-int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+   @routine    PUGH_SyncGArrayGroup
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Synchronizes a group of arrays
+               given the first variable within this group.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+int PUGH_SyncGArrayGroup(pGH *pughGH,
+                         int first_var)
 {
-  DECLARE_CCTK_PARAMETERS
+  pGA *firstGA;
+
+
+  firstGA = (pGA *) pughGH->variables [first_var][0];
 
-  int group;               /* group index */
-  cGroup pgroup;        /* pointer to group information */
-  int rc;                  /* return code */
-  pGH *pughGH;             /* PUGH extension reference */
-  int level;
-  int first_var,var;
-  int unchanged;           /* count how many aren't toggled */
-  int PUGH_DisableGADataStorage(pGA *GA);
-
-  /* get PUGH extension handle */
-  pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-  
 #ifdef DEBUG_PUGH
-  printf(" PUGH_DisableGroupStorage: request for group -%s-\n",groupname);
+  printf(" PUGH_SyncGArrayGroup: request for group with first var '%s'\n",
+         firstGA->name);
   fflush(stdout);
 #endif
 
-  group = CCTK_GroupIndex(groupname);
+  return (PUGH_Sync(pughGH, firstGA->groupcomm));
+}
 
-  /* get global index of first variable in group */
-  first_var = CCTK_FirstVarIndexI (group);
 
-  /* get the group info from its index */
-  CCTK_GroupData (group,&pgroup);
+ /*@@
+   @routine    PUGH_SyncGArray
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Synchronizes a single array variable given the GA structure
+               of this variable.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+int PUGH_SyncGArray(pGA *GA)
+{
 
-  unchanged = 0;
-  switch (pgroup.grouptype) 
-  {
-    case GROUP_SCALAR :
-      rc = 1;
-      break;
+#ifdef DEBUG_PUGH
+  printf(" PUGH_SyncGArray: request for var '%s'\n", GA->name);
+  fflush(stdout);
+#endif
 
-    case GROUP_ARRAY  :
-    case GROUP_GF     :
-      for (var = first_var; var < first_var+pgroup.numvars; var++)
-      {
-        for (level = 0; level < pgroup.numtimelevels; level++)
-        {
-          unchanged = unchanged + 
-            PUGH_DisableGADataStorage ((pGA *)(pughGH->variables[var][level]));
-        }
-      }
-      rc = 1;
-      break;
+  return (PUGH_Sync((pGH *) GA->parent, GA->comm));
+}
 
-    default :
-      CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
-      rc = 0;
-      break;
-  }
 
-  /* Report on memory usage */
-  if (storage_verbose)
-  {
-    if (unchanged == 0)
-    {
-      /* Memory toggled */
-      totalnumber = totalnumber - pgroup.numvars;
-      totalstorage = totalstorage - 
-        ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
-        *CCTK_VarTypeSize(pgroup.vartype)
-        *pgroup.numtimelevels
-        *pgroup.numvars;
-      printf("Switched memory off for %s \n  [Num Arrays: %d Total Size: %d]\n",
-             groupname,totalnumber,totalstorage);
-    }
-    else if (unchanged == pgroup.numvars)
-    {
-      /* Memory already off */
-      printf("Memory already off for %s\n",groupname);
-    }
-    else
-    {
-      CCTK_WARN(1,"Inconsistency in group memory assignment");
-    }
-  }
+int PUGH_Barrier(cGH *GH)
+{
+#ifdef CCTK_MPI
+  CACTUS_MPI_ERROR(MPI_Barrier(PUGH_pGH(GH)->PUGH_COMM_WORLD));
+#endif
 
-  return (rc);
+  return 0;
 }
 
 
-/*@@
-  @routine PUGH_EnableGroupComm
-  @author  Thomas Radke
-  @date    30 Mar 1999
-  @desc
-  Enables communication for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
+/*****************************************************************************/
+/* local functions                                                           */
+/*****************************************************************************/
 
-int PUGH_EnableGroupComm (cGH *GH, const char *groupname)
+ /*@@
+   @routine    PUGH_EnableGArrayGroupComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Enables communication for a group of arrays
+               given the first variable within this group.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+                                      int first_var,
+                                      int commflag)
 {
-  cGroup pgroup;       /* pointer to group information */
-  int group;               /* group index */
-  pGH *pughGH;             /* PUGH extension reference */
-  int var, level;          /* loop variables */
-  int first_var;           /* global index of first variable in group */
-  int rc;                  /* return code */
+  pGA *GA;                 /* first variable in group */
 
-  group = CCTK_GroupIndex(groupname);
 
-  /* get the group info from its index */
-  CCTK_GroupData (group,&pgroup);
+  GA = pughGH->variables [first_var][0];
 
-  /* get global index of first variable in group */
-  first_var = CCTK_FirstVarIndexI (group);
+#ifdef DEBUG_PUGH
+  printf(" PUGH_EnableGArrayGroupComm: request for group "
+         "with first var '%s', commflag %d\n", GA->name, commflag);
+  fflush(stdout);
+#endif
 
-  /* get PUGH extension handle */
-  pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+  return (PUGH_EnableComm(pughGH, GA->groupcomm, commflag));
+}
 
-  /* branch to "enable comm" function according to group type */
-  switch (pgroup.grouptype) 
-  {
-    case GROUP_SCALAR :
-      rc = 1;
-      break;
 
-    case GROUP_ARRAY  :
-    case GROUP_GF     :
-      for (var = first_var; var < first_var+pgroup.numvars; var++)
-      {
-        for (level = 0; level < pgroup.numtimelevels; level++)
-        {
-          PUGH_GAComm ((pGA *)(pughGH->variables[var][level]), 
-                             PUGH_ALLCOMM);
-        }
-      }
-      rc = 1;
-      break;
+ /*@@
+   @routine    PUGH_DisableGArrayGroupComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Disables communication for a group of arrays
+               given the first variable within this group.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+                                       int first_var)
+{
+  pGA *GA;                 /* first variable in group */
 
-    default :
-      CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupComm");
-      rc = 0;
-      break;
-  }
 
-  return (rc);
-}
+  GA = pughGH->variables [first_var][0];
 
+#ifdef DEBUG_PUGH
+  printf(" PUGH_DisableGArrayGroupComm: request for group "
+         "with first var '%s'\n", GA->name);
+  fflush(stdout);
+#endif
 
-/*@@
-  @routine PUGH_DisableGroupComm
-  @author  Thomas Radke
-  @date    30 Mar 1999
-  @desc
-  Disables communication for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
+  return (PUGH_DisableComm(pughGH, GA->groupcomm));
+}
 
-int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
+
+ /*@@
+   @routine    PUGH_EnableComm
+   @date       Sun Jan 23 12:46:23 2000
+   @author     Gabrielle Allen
+   @desc
+   This sets the docomm[2*dim] array of the GA based
+   on the setting of the comm flag and allocates the comm buffers.
+   @enddesc
+   @history
+               @date Mon Jun 05 2000 @author Thomas Radke
+               Moved buffer allocation from PUGH_EnableGArrayDataStorage
+   @endhistory
+@@*/
+static int PUGH_EnableComm(pGH *pughGH,
+                           pComm *comm,
+                           int commflag)
 {
-  int group;               /* group index */
-  cGroup pgroup;           /* pointer to group information */
-  pGH *pughGH;             /* PUGH extension reference */
-  int var, level;          /* loop variables */
-  int first_var;           /* global index of first variable in group */
-  int rc;                  /* return code */
+  int retval;              /* return value */
+#ifdef CCTK_MPI
+  pGA *GA;                 /* GA structure the pComm belongs to */
+  int i, idir;             /* loopers */
+  int dir;                 /* direction */
+  int sz;                  /* buffer size */
+#endif
 
-  group = CCTK_GroupIndex(groupname);
 
-  /* get the group info from its index */
-  CCTK_GroupData (group,&pgroup);
+  retval = 1;
 
-  /* get global index of first variable in group */
-  first_var = CCTK_FirstVarIndexI (group);
+#ifdef CCTK_MPI
+  /* check whether this comm buffer is already set up */
+  if (comm->commflag == commflag)
+  {
+    return (retval);
+  }
 
-  /* get PUGH extension handle */
-  pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+  /* get the GA structure the comm structure belongs to
+     For a group comm structure this GA is the first one within the group. */
+  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
 
-  /* branch to "disable comm" function according to group type */
-  switch (pgroup.grouptype) 
+  if (comm->commflag == PUGH_NOCOMM)
   {
-    case GROUP_SCALAR :
-      rc = 1;
-      break;
-
-    case GROUP_ARRAY  :
-    case GROUP_GF     :
-      for (var = first_var; var < first_var+pgroup.numvars; var++)
-        for (level = 0; level < pgroup.numtimelevels; level++)
+
+#ifdef DEBUG_PUGH
+    printf (" PUGH_EnableComm: allocating comm buffer for %d vars starting "
+            "with %d '%s'\n", comm->n_vars, comm->first_var, GA->name);
+    fflush (stdout);
+#endif
+
+    /* allocate memory for communication buffers: 2 faces per direction */
+    for (i = 0; i < 2 * GA->extras->dim; i++)
+    {
+      if (GA->connectivity->neighbours[pughGH->myproc][i] >= 0)
+      {
+        dir = i/2;
+
+        /* no ghostzones -> no comm buffers */
+        sz = comm->n_vars * GA->extras->nghostzones[dir];
+        if (sz > 0)
         {
-          PUGH_GAComm ((pGA *)(pughGH->variables[var][level]), PUGH_NOCOMM);
+          for (idir = 0; idir < GA->extras->dim; idir++)
+          {
+            if (idir != dir)
+            {
+              sz *= GA->extras->lnsize[idir];
+            }
+          }
+          comm->buffer_sz[i]   = sz;
+          comm->send_buffer[i] = malloc(sz * GA->varsize);
+          comm->recv_buffer[i] = malloc(sz * GA->varsize);
+
+          if (! (comm->send_buffer[i] && comm->recv_buffer[i]))
+          {
+            for (; i >=0 ; i--)
+            {
+              if (comm->send_buffer[i])
+              {
+                free(comm->send_buffer[i]);
+              }
+              if (comm->recv_buffer[i])
+              {
+                free(comm->recv_buffer[i]);
+              }
+              comm->buffer_sz[i] = 0;
+            }
+
+            CCTK_WARN(1, "Out of memory !");
+            retval = -1;
+            break;
+          }
         }
-      rc = 1;
-      break;
-
-    default :
-      CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupComm");
-      rc = 0;
-      break;
+        else
+        {
+          comm->buffer_sz[i]   = 0;
+          comm->send_buffer[i] = NULL;
+          comm->recv_buffer[i] = NULL;
+        }
+      }
+      else
+      {
+        comm->buffer_sz[i]   = 0;
+        comm->send_buffer[i] = NULL;
+        comm->recv_buffer[i] = NULL;
+      }
+    }
   }
 
-  return (rc);
-}
+  /* set up comm flags for each face */
+  if (retval >= 0)
+  {
+    /* Copy commflag */
+    comm->commflag = commflag;
 
+    /* First set all communcation off */
+    for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+      comm->docomm[idir] = 0;
 
-int PUGH_Barrier(cGH *GH)
-{
-#ifdef CCTK_MPI
-  pGH *pughGH;
+    if (commflag == PUGH_ALLCOMM)
+    {
+      for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+      {
+        comm->docomm[idir] = 1;
+      }
+    }
+    else if (commflag == PUGH_PLUSFACESCOMM)
+    {
+      for (idir = 0; idir < GA->extras->dim; idir++)
+      {
+        comm->docomm[2*idir+1] = 1;
+      }
+    }
+    else if (commflag == PUGH_MINUSFACESCOMM)
+    {
+      for (idir = 0; idir < GA->extras->dim; idir++)
+      {
+        comm->docomm[2*idir] = 1;
+      }
+    }
+    else
+    {
+      for (idir = 0; idir < GA->extras->dim; idir++)
+      {
+        if (commflag == PUGH_COMM(idir))
+        {
+          comm->docomm[2*idir] = 1;
+          comm->docomm[2*idir+1] = 1;
+        }
+      }
+    }
 
-  pughGH = PUGH_pGH(GH);
+    /* FIXME Add back the check that you have a valid COMM model: Gab */
 
-  CACTUS_MPI_ERROR (MPI_Barrier (pughGH->PUGH_COMM_WORLD));
-#endif
+    /* Handle nsize = 1 type cases. This is only important for one
+       processor MPI periodic boundaries */
 
-  return 0;
-}
+    for (idir = 0; idir < GA->extras->dim; idir++)
+    {
+      if (GA->extras->nsize[idir] == 1)
+      {
+        comm->docomm[2*idir] = 0;
+        comm->docomm[2*idir+1] = 0;
+      }
+    }
+  }
+#endif   /* CCTK_MPI */
 
-/*
-int PUGH_Reduce(cGH *GH,               
-                const char *operation,
-                int n_infields,
-                int n_outfields,
-                int out_type,
-                void **outarray,
-                ...)
-{
-  return 0;
+  return retval;
 }
-*/
 
 
-int PUGH_Interp(cGH *GH,
-                const char *operation,
-                int n_coords,
-                int n_infields,
-                int n_outfields,
-                int n_points,
-                int type,
-                ...)
+ /*@@
+   @routine    PUGH_DisableComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               This frees the communication buffers
+               of a given comm structure.
+   @enddesc
+   @history
+               Separated from routine PUGH_DisableGArrayDataStorage()
+   @endhistory
+@@*/
+static int PUGH_DisableComm(pGH *pughGH,
+                            pComm *comm)
 {
-  return 0;
-}
+#ifdef CCTK_MPI
+  int i;                  /* looper */
+  pGA *GA;                /* GA structure the comm structure belongs to */
 
 
-int PUGH_ParallelInit(cGH *GH)
-{
-  return 0;
-}
+  /* get the GA to which the comm structure belongs to
+     For a comm structure this is the first variable within this group. */
+  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
 
-int PUGH_Exit(int retval, cGH *GH)
-{
-#ifdef CCTK_MPI  
-  CACTUS_MPI_ERROR(MPI_Finalize());
+#ifdef DEBUG_PUGH
+  printf (" PUGH_DisableComm: freeing comm buffer for group of %d vars and "
+          "first var '%s'\n", comm->n_vars, GA->name);
+  fflush (stdout);
 #endif
-  exit(retval);
-  return(retval);
-}
-
-int PUGH_Abort(cGH *GH)
-{
-  assert(0);
-  exit(0);
-  return(0);
-}
-
-
-/*****************************************************************************/
-/* local functions                                                           */
-/*****************************************************************************/
-
 
+  /* free memory for communication buffers: 2 faces per direction */
+  for (i = 0; i < 2 * GA->extras->dim; i++)
+  {
+    if (comm->send_buffer[i])
+    {
+      if (comm->sreq[i] != MPI_REQUEST_NULL)
+      {
+        CACTUS_MPI_ERROR(MPI_Request_free(&comm->sreq[i]));
+      }
+      free(comm->send_buffer[i]);
+      comm->send_buffer[i] = NULL;
+    }
 
-int PUGH_SyncGroupGA (cGH *GH, 
-                      int group, 
-                      int vtype, 
-                      int n_vars, 
-                      int timelevel)
-{
+    if (comm->recv_buffer[i])
+    {
+      free(comm->recv_buffer[i]);
+      comm->recv_buffer[i] = NULL;
+    }
 
-  pGH *pughGH;
+    comm->buffer_sz[i] = 0;
+  }
 
-  pughGH = PUGH_pGH(GH);
+  comm->commflag = PUGH_NOCOMM;
+#endif   /* CCTK_MPI */
 
-  return PUGH_SyncGAs(pughGH, 
-                      CCTK_GroupDimI(group),
-                      CCTK_FirstVarIndexI(group), 
-                      n_vars, 
-                      timelevel);
+  return (1);
 }
 
 
-int PUGH_SyncGAs(pGH *pughGH, 
-                 int dim, 
-                 int first_var, 
-                 int n_vars, 
-                 int timelevel)
+ /*@@
+   @routine    PUGH_Sync
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Finally synchronizes a variable or group of variables
+               according to a given comm structure.
+   @enddesc
+   @history
+   @endhistory
+@@*/
+static int PUGH_Sync(pGH *pughGH,
+                     pComm *comm)
 {
 #ifdef CCTK_MPI
-  int Dir;
+  int dir;
   pGA *GA;
-#ifdef COMM_TIMING
-  double t1,t2;
-#endif
-  MPI_Request *sr = NULL;
   MPI_Status mss;
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+  int i;
+  MPI_Request *sr;
+#endif
+#ifdef COMM_TIMING
+  double t1, t2;
 #endif
 
-  /* start the timer for communication time */
+
 #if 0
-  CactusStartTimer (&pughGH->comm_time);
+  /* start the timer for communication time */
+  CCTK_TimerStartI(pughGH->comm_time);
 #endif
 
-#ifdef CCTK_MPI
+#ifdef PUGH_WITH_DERIVED_DATATYPES
   if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
   {
-    int i;
     /* 2 faces, send and receive is the 2 * 2 */
-    sr = (MPI_Request *)malloc(n_vars * 2 * 2 * sizeof(MPI_Request));
-    for (i=0;i<n_vars * 2 * 2; i++)
-      sr[i] = MPI_REQUEST_NULL;
+    sr = (MPI_Request *) malloc(comm->n_vars * 2 * 2 * sizeof(MPI_Request));
   }
+#endif
+
+  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
+
+#ifdef DEBUG_PUGH
+  printf (" PUGH_Sync: syncing group of %d vars with first var '%s'\n",
+          comm->n_vars, GA->name);
+  fflush (stdout);
+#endif
 
-  for (Dir = 0; Dir < dim; Dir ++) 
+  for (dir = 0; dir < GA->extras->dim; dir ++) 
   {
-    int i;
 #ifdef COMM_TIMING
     t1 = MPI_Wtime();
 #endif
-    for (i = first_var; i < first_var + n_vars; i++) 
-    {
-      GA = (pGA *) (pughGH->variables [i][timelevel]);
-      PostReceiveGA(pughGH, GA, 2*Dir);
-      PostReceiveGA(pughGH, GA, 2*Dir+1);
-    }
+    PostReceiveGA(pughGH, 2*dir, comm);
+    PostReceiveGA(pughGH, 2*dir+1, comm);
     
 #ifdef COMM_TIMING
     t2 = MPI_Wtime();
-    printf ("PR : %lf\n",t2-t1);
+    printf("PR : %f\n",t2-t1);
 #endif
-    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
+
+    /* For allocated buffers: wait for the last send.
+       Since these will be null if they
+       are not used, this is always safe.
+     */
+    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
     {
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GA = (pGA *) (pughGH->variables [i][timelevel]);
-        /* Wait for the last send. Since these will be null if they
-           are not used, this is always safe.
-        */
-        MPI_Wait(&(GA->sreq[2*Dir]),&mss);
-        PostSendGA(pughGH, GA, 2*Dir);
-        MPI_Wait(&(GA->sreq[2*Dir+1]),&mss);
-        PostSendGA(pughGH, GA, 2*Dir+1);
-      }
-    } 
-    else 
+      MPI_Wait(&comm->sreq[2*dir], &mss);
+    }
+    PostSendGA(pughGH, 2*dir, comm);
+    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
     {
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GA = (pGA *) (pughGH->variables [i][timelevel]);
-        PostSendGA(pughGH, GA, 2*Dir);
-        PostSendGA(pughGH, GA, 2*Dir+1);
-      }
+      MPI_Wait(&comm->sreq[2*dir+1], &mss);
     }
+    PostSendGA(pughGH, 2*dir+1, comm);
 
 #ifdef COMM_TIMING
     t1 = MPI_Wtime();
-    printf ("PS : %lf\n",t1-t2);
+    printf("PS : %f\n",t1-t2);
 #endif
 
     /* Now comes the big difference between derived types and
@@ -878,40 +709,37 @@ int PUGH_SyncGAs(pGH *pughGH,
     if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
     {
       /* Do a wait any on the receives */
-      for (i = first_var; i < first_var + n_vars; i++) 
-      {
-        GA = (pGA *) (pughGH->variables [i][timelevel]);
-        MPI_Wait(&(GA->rreq[2*Dir]),&mss);
-        FinishReceiveGA(pughGH, GA, 2*Dir);
-        MPI_Wait(&(GA->rreq[2*Dir+1]),&mss);
-        FinishReceiveGA(pughGH, GA, 2*Dir+1);
-      }
-    } 
-
-    if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
+      MPI_Wait(&comm->rreq[2*dir], &mss);
+      FinishReceiveGA(pughGH, 2*dir, comm);
+      MPI_Wait(&comm->rreq[2*dir+1], &mss);
+      FinishReceiveGA(pughGH, 2*dir+1, comm);
+    }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+    else if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
     {
       /* Load up the thing for the waitall */
-      for (i=0;i<n_vars;i++)
+      for (i = 0; i < comm->n_vars; i++)
       {
-        int id = i*4;
-        GA = (pGA *) (pughGH->variables [i][timelevel]);
-        if (GA->docomm[2*Dir] &&
+        int id = i * 2 * 2;
+        pGA *GA = (pGA *) pughGH->variables [i][comm->sync_timelevel];
+
+        if (GA->comm->docomm[2*dir] &&
             GA->storage)
         {
-          sr[id] = GA->sreq[2*Dir];
-          sr[id+1] = GA->rreq[2*Dir];
+          sr[id] = GA->comm->sreq[2*dir];
+          sr[id+1] = GA->comm->rreq[2*dir];
         }
         else
         {
           sr[id] = MPI_REQUEST_NULL;
           sr[id+1] = MPI_REQUEST_NULL;
         }
-        
-        if (GA->docomm[2*Dir+1] &&
+
+        if (GA->comm->docomm[2*dir+1] &&
             GA->storage)
         {
-          sr[id+2] = GA->sreq[2*Dir+1];
-          sr[id+3] = GA->rreq[2*Dir+1];
+          sr[id+2] = GA->comm->sreq[2*dir+1];
+          sr[id+3] = GA->comm->rreq[2*dir+1];
         }
         else
         {
@@ -920,144 +748,26 @@ int PUGH_SyncGAs(pGH *pughGH,
         }
       }
       /* Now do a waitall */
-      MPI_Waitall(4*n_vars,sr,&mss);
-          
+      MPI_Waitall(4*comm->n_vars, sr, &mss);
     }
+#endif
 
-    if (sr)
-      free(sr);
 #ifdef COMM_TIMING
     t2 = MPI_Wtime();
-    printf ("FR : %lf\n",t2-t1);
+    printf("FR : %f\n",t2-t1);
 #endif
   }
-#endif            /* MPI */
-
-  /* get the time spent in communication */
-
-#if 0
-  CactusStopTimer (&pughGH->comm_time);
-#endif
-
-  return (1);
-}
-
-
-int PUGH_EnableArrayGroupStorage(cGH *GH,                              
-                                 int group,
-                                 int vtype,
-                                 int dim,
-                                 int n_variables,
-                                 int n_timelevels)
-{
-  int nstorage;   /* Number of Arrays for which storage was set */
-  int nnostorage; /* Number of Arrays for which no storage was set */
-  int retval;
-  int first_var;
-  int var;
-  pGH *mypGH;
-  pGA *GA;
-  int level;
-  /* FIXME: This should come from a parameter */
-  int zero_memory=1;
-  int padding_active=0;
-  int padding_cacheline_bits=0;
-  int padding_size=0;
-  int padding_address_spacing=0;
-
-  nstorage = 0;
-  nnostorage = 0;
-
-  mypGH = PUGH_pGH(GH);
 
-  first_var = CCTK_FirstVarIndexI(group);
-
-  for(var = first_var; var < first_var+n_variables; var++)
-  {
-    for(level = 0; level < n_timelevels; level++)
-    {
-      GA = (pGA *)(mypGH->variables[var][level]);
-
-      if(!GA->storage)
-      {
-#ifdef DEBUG_PUGH
-        {
-          const char *name;
-          name = CCTK_FullName(var);
-          printf("Storage on in PUGH_EnableArrayGroupStorage for %s\n",name);
-        }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+  free(sr);
 #endif
-        PUGH_EnablePGAStorage(GA, 
-                              mypGH->myproc,
-                              zero_memory, 
-                              padding_active,
-                              padding_cacheline_bits,
-                              padding_size,
-           padding_address_spacing);
-
-        GH->data[var][level] = GA->data;
-
-        nnostorage++;
-      }
-      else
-      {
-        GH->data[var][level] = GA->data;
-        nstorage++;
-      }
-    }
-  }
 
-  if (nstorage > 0 && nnostorage > 0)
-  {
-    CCTK_WARN(0,"Group storage violation in PUGH_EnableArrayGroupStorage");
-    retval = -1;
-  }
-  else if (nstorage > 0)
-  {
-    retval = 1;
-  }
-  else if (nnostorage > 0)
-  {
-    retval = 0;
-  }
-  else
-  {
-    CCTK_WARN(0,"Problem in PUGH_EnableArrayGroupStorage");
-    retval = -1;
-  }
-
-  return retval;
-
-}
-
-
-int PUGH_MyProc(cGH *GH)
-{
-  int myproc;
-
-#ifdef CCTK_MPI
-  pGH *pughGH;
-  pughGH = PUGH_pGH(GH);
-  CACTUS_MPI_ERROR(MPI_Comm_rank(pughGH->PUGH_COMM_WORLD, &myproc));
-#else
-  myproc=0;
+#if 0
+  /* get the time spent in communication */
+  CCTK_TimerStopI(pughGH->comm_time);
 #endif
 
-  return myproc;
-}
-
-
-int PUGH_nProcs(cGH *GH)
-{
-  int nprocs;
-
-#ifdef CCTK_MPI
-  pGH *pughGH;
-  pughGH = PUGH_pGH(GH);
-  CACTUS_MPI_ERROR(MPI_Comm_size(pughGH->PUGH_COMM_WORLD, &nprocs));
-#else
-  nprocs=1;
-#endif
+#endif            /* CCTK_MPI */
 
-  return nprocs;
+  return (0);
 }