Fixed a bug when freeing the comm structure of individual grid arrays with

multiple timelevels.
Completed grdoc.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@374 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 369 insertions, 212 deletions

diff --git a/src/Comm.c b/src/Comm.c
index 833ad9e..db01a5a 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -2,17 +2,16 @@
    @file      Comm.c
    @date      Thu Feb  4 11:34:29 1999
    @author    Tom Goodale
-   @desc 
-   Pugh communication functions
-   @enddesc 
-   @version $Header$
+   @desc
+              PUGH communication functions
+   @enddesc
+   @version   $Id$
  @@*/
 
 /*#define DEBUG_PUGH 1*/
 
 #include <stdlib.h>
 #include <stdio.h>
-#include <stdarg.h> 
 #include <string.h>
 
 #include "cctk.h"
@@ -20,7 +19,6 @@
 #include "pugh.h"
 #include "pughi.h"
 #include "pugh_Comm.h"
-#include "cctk_Parameters.h"
 
 static const char *rcsid="$Header$";
 
@@ -28,52 +26,48 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_Comm_c)
 
 
 /* local function prototypes */
-static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
-                                      int first_var,
-                                      int commflag);
-static int PUGH_EnableComm(pGH *pughGH,
-                           pComm *comm,
-                           int commflag);
-static int PUGH_DisableComm(pGH *pughGH,
-                            pComm *comm);
-static int PUGH_SyncGArrayGroup(pGH *pughGH,
-                                int first_var);
-static int PUGH_Sync(pGH *pughGH,
-                     pComm *comm);
-static int PUGH_SyncSingleProc(pGH *pughGH,
-                               pComm *comm);
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH, int first_var, int commflag);
+static int PUGH_EnableComm(pGH *pughGH, pComm *comm, int commflag);
+static int PUGH_DisableComm(pGA *GA, pComm *comm);
+static int PUGH_SyncGArrayGroup(pGH *pughGH, int first_var);
+static int PUGH_Sync(pGH *pughGH, pComm *comm);
+static int PUGH_SyncSingleProc(pGH *pughGH, pComm *comm);
 
 
 /*@@
-  @routine PUGH_SyncGroup
-  @author  Thomas Radke
-  @date    30 Mar 1999
-  @desc
-  Synchronizes all variables in the group indicated by groupname.
-  Only groups of type GROUP_ARRAY and GROUP_GF can be synchronized.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-           PUGH_SyncGArrayGroup
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
-
+   @routine    PUGH_SyncGroup
+   @author     Thomas Radke
+   @date       30 Mar 1999
+   @desc
+               Synchronizes all variables in the group indicated by groupname.
+               Only groups of type GROUP_ARRAY and GROUP_GF can be synchronized.
+   @enddesc
+   @calls      PUGH_SyncGArrayGroup
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        groupname
+   @vdesc      name of the group to be synchronized
+   @vtype      const char *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+                0 for grid scalar groups<BR>
+                  return code of @seeroutine PUGH_SyncGArrayGroup
+                  for grid array groups<BR>
+               -1 for an unknown group or group type
+   @endreturndesc
+@@*/
 int PUGH_SyncGroup(cGH *GH, const char *groupname)
 {
-  cGroup pgroup;           /* group information */
-  int group;               /* group index */
-  int rc;                  /* return code */
+  cGroup pgroup;
+  int group, retval;
+
 
 #ifdef DEBUG_PUGH
   printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
@@ -86,7 +80,7 @@ int PUGH_SyncGroup(cGH *GH, const char *groupname)
   {
     CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "PUGH_SyncGroup: Unknown group: %s", groupname);
-    rc = -1;
+    retval = -1;
   }
   else
   {
@@ -94,53 +88,59 @@ int PUGH_SyncGroup(cGH *GH, const char *groupname)
 
     if (pgroup.grouptype == CCTK_SCALAR)
     {
-      rc = 0;
+      retval = 0;
       CCTK_VWarn(4, __LINE__, __FILE__, CCTK_THORNSTRING,
-		 "PUGH_SyncGroup: Synchronising scalar group: %s",groupname);
+                 "PUGH_SyncGroup: Synchronising scalar group: %s",groupname);
     }
     else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
     {
-      rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
+      retval = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
     }
     else
     {
       CCTK_WARN(1, "PUGH_SyncGroup: Unknown group type");
-      rc = 0;
+      retval = -1;
     }
   }
 
-  return (rc);
+  return (retval);
 }
 
 
 /*@@
-  @routine PUGH_EnableGroupComm
-  @author  Thomas Radke
-  @date    30 Mar 1999
-  @desc
-  Enables communication for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-  @history
+   @routine PUGH_EnableGroupComm
+   @author     Thomas Radke
+   @date       30 Mar 1999
+   @desc
+               Enables communication for all variables in the group
+               indicated by groupname.
+   @enddesc
+   @calls      PUGH_EnableGArrayGroupComm
  
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
-
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        groupname
+   @vdesc      name of the group to be synchronized
+   @vtype      const char *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+                1 for grid scalar groups<BR>
+                  return code of @seeroutine PUGH_EnableGArrayGroupComm
+                  for grid array groups<BR>
+               -1 for an unknown group type
+   @endreturndesc
+ @@*/
 int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
 {
-  int group;               /* group index */
-  cGroup pgroup;           /* group information */
-  int rc;                  /* return code */
+  int group, retval;
+  cGroup pgroup;
+
 
 #ifdef DEBUG_PUGH
   printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
@@ -153,55 +153,60 @@ int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
 
   if (pgroup.grouptype == CCTK_SCALAR)
   {
-    rc = 1;
+    retval = 1;
   }
   else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
   {
-    rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
+    retval = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
                                     CCTK_FirstVarIndexI(group),
                                     PUGH_ALLCOMM);
   }
   else
   {
     CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
-    rc = 0;
+    retval = -1;
   }
 
-  return (rc);
+  return (retval);
 }
 
 
 /*@@
-  @routine PUGH_DisableGroupComm
-  @author  Thomas Radke
-  @date    30 Mar 1999
-  @desc
-  Disables communication for all variables in the group indicated by groupname.
-  @enddesc
-  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
-  @history
- 
-  @endhistory 
-  @var     GH
-  @vdesc   Pointer to CCTK grid hierarchy
-  @vtype   cGH
-  @vio     in
-  @endvar 
-  @var     groupname
-  @vdesc   name of the group to be synchronized
-  @vtype   const char *
-  @vio     in
-  @endvar 
-  @@*/
-
+   @routine    PUGH_DisableGroupComm
+   @author     Thomas Radke
+   @date       30 Mar 1999
+   @desc
+               Disables communication for all variables in the group
+               indicated by groupname.
+   @enddesc
+   @calls      PUGH_DisableGArrayGroupComm
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      cGH *
+   @vio        in
+   @endvar
+   @var        groupname
+   @vdesc      name of the group to be synchronized
+   @vtype      const char *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+                1 for grid scalar groups<BR>
+                  return code of @seeroutine PUGH_DisableGArrayGroupComm
+                  for grid array groups<BR>
+               -1 for an unknown group type
+   @endreturndesc
+ @@*/
 int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
 {
-  int group;               /* group index */
-  cGroup pgroup;           /* pointer to group information */
-  int rc;                  /* return code */
-
+  int group, first_var, retval;
+  cGroup pgroup;
+  pGA *GA;
   pGH *pughGH;
-  int var;
+
 
 #ifdef DEBUG_PUGH
   printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
@@ -214,38 +219,52 @@ int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
 
   if (pgroup.grouptype == CCTK_SCALAR)
   {
-    rc = 1;
+    retval = 1;
   }
   else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
   {
-    pughGH=PUGH_pGH(GH);
-    var = CCTK_FirstVarIndexI(group);
+    first_var = CCTK_FirstVarIndexI(group);
+    pughGH = PUGH_pGH(GH);
+    GA = (pGA *) pughGH->variables[first_var][0];
 
     /* FIXME:  workaround.  This one is really bad ! */
-    rc = PUGH_DisableGArrayGroupComm(pughGH, var,(((pGA ***)pughGH->variables)[var][0])->groupcomm);
+    retval = PUGH_DisableGArrayGroupComm(pughGH, first_var, GA->groupcomm);
   }
   else
   {
     CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
-    rc = 0;
+    retval = -1;
   }
 
-  return (rc);
+  return (retval);
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_EnableGArrayComm
    @date       Mon Jun 05 2000
    @author     Thomas Radke
    @desc
                Enables communication for a single array.
    @enddesc
-   @history
-   @endhistory
-@@*/
-int PUGH_EnableGArrayComm(pGA *GA,
-                          int commflag)
+
+   @var        GA
+   @vdesc      Pointer to grid array structure
+   @vtype      pGA *
+   @vio        in
+   @endvar
+   @var        commflag
+   @vdesc      flag indicating in which directions to communicate
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_EnableComm
+   @endreturndesc
+ @@*/
+int PUGH_EnableGArrayComm(pGA *GA, int commflag)
 {
 
 #ifdef DEBUG_PUGH
@@ -258,29 +277,37 @@ int PUGH_EnableGArrayComm(pGA *GA,
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_DisableGArrayComm
    @date       Mon Jun 05 2000
    @author     Thomas Radke
    @desc
                Disables communication for a single array.
    @enddesc
-   @history
-   @endhistory
-@@*/
+
+   @var        GA
+   @vdesc      Pointer to grid array structure
+   @vtype      pGA *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_DisableComm
+   @endreturndesc
+ @@*/
 int PUGH_DisableGArrayComm(pGA *GA)
 {
-
 #ifdef DEBUG_PUGH
   printf(" PUGH_DisableGArrayComm: request for var '%s'\n", GA->name);
   fflush(stdout);
 #endif
 
-  return (PUGH_DisableComm((pGH *) GA->parent, GA->comm));
+  return (PUGH_DisableComm(GA, GA->comm));
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_SyncGArrayGroup
    @date       Mon Jun 05 2000
    @author     Thomas Radke
@@ -288,14 +315,28 @@ int PUGH_DisableGArrayComm(pGA *GA)
                Synchronizes a group of arrays
                given the first variable within this group.
    @enddesc
-   @history
-   @endhistory
-@@*/
-int PUGH_SyncGArrayGroup(pGH *pughGH,
-                         int first_var)
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid hierarchy extension
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        first_var
+   @vdesc      grid index of first variable in the group
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_Sync
+   @endreturndesc
+ @@*/
+int PUGH_SyncGArrayGroup(pGH *pughGH, int first_var)
 {
   pGA *firstGA;
 
+
   firstGA = (pGA *) pughGH->variables [first_var][0];
 
 #ifdef DEBUG_PUGH
@@ -308,7 +349,7 @@ int PUGH_SyncGArrayGroup(pGH *pughGH,
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_SyncGArray
    @date       Mon Jun 05 2000
    @author     Thomas Radke
@@ -316,9 +357,18 @@ int PUGH_SyncGArrayGroup(pGH *pughGH,
                Synchronizes a single array variable given the GA structure
                of this variable.
    @enddesc
-   @history
-   @endhistory
-@@*/
+
+   @var        GA
+   @vdesc      Pointer to grid array structure
+   @vtype      pGA *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_Sync
+   @endreturndesc
+ @@*/
 int PUGH_SyncGArray(pGA *GA)
 {
 
@@ -331,6 +381,72 @@ int PUGH_SyncGArray(pGA *GA)
 }
 
 
+/*@@
+   @routine    PUGH_DisableGArrayGroupComm
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Disables communication for a group of arrays
+               given the first variable within this group.
+   @enddesc
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid hierarchy extension
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        first_var
+   @vdesc      grid index of first variable in the group
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        groupcomm
+   @vdesc      pointer to comm structure for this group
+   @vtype      pComm *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_DisableComm
+   @endreturndesc
+@@*/
+int PUGH_DisableGArrayGroupComm(pGH *pughGH, int first_var, pComm *groupcomm)
+{
+  pGA *GA;                 /* first variable in group */
+
+
+  GA = (pGA *) pughGH->variables[first_var][0];
+
+#ifdef DEBUG_PUGH
+  printf(" PUGH_DisableGArrayGroupComm: request for group "
+         "with first var '%s'\n", GA->name);
+  fflush(stdout);
+#endif
+
+  return (PUGH_DisableComm(GA, groupcomm));
+}
+
+
+/*@@
+   @routine    PUGH_Barrier
+   @date       Mon Jun 05 2000
+   @author     Thomas Radke
+   @desc
+               Synchronize all processors at a given point of execution.
+   @enddesc
+
+   @var        GH
+   @vdesc      Pointer to CCTK grid hierarchy
+   @vtype      const cGH *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for success
+   @endreturndesc
+ @@*/
 int PUGH_Barrier(const cGH *GH)
 {
 #ifdef CCTK_MPI
@@ -347,7 +463,7 @@ int PUGH_Barrier(const cGH *GH)
 /* local functions                                                           */
 /*****************************************************************************/
 
- /*@@
+/*@@
    @routine    PUGH_EnableGArrayGroupComm
    @date       Mon Jun 05 2000
    @author     Thomas Radke
@@ -355,12 +471,29 @@ int PUGH_Barrier(const cGH *GH)
                Enables communication for a group of arrays
                given the first variable within this group.
    @enddesc
-   @history
-   @endhistory
-@@*/
-static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
-                                      int first_var,
-                                      int commflag)
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid hierarchy extension
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        first_var
+   @vdesc      grid index of first variable in the group
+   @vtype      int
+   @vio        in
+   @endvar
+   @var        commflag
+   @vdesc      flag indicating in which directions to communicate
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               return code of @seeroutine PUGH_EnableComm
+   @endreturndesc
+ @@*/
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH, int first_var, int commflag)
 {
   pGA *GA;                 /* first variable in group */
 
@@ -377,53 +510,42 @@ static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
 }
 
 
- /*@@
-   @routine    PUGH_DisableGArrayGroupComm
-   @date       Mon Jun 05 2000
-   @author     Thomas Radke
-   @desc
-               Disables communication for a group of arrays
-               given the first variable within this group.
-   @enddesc
-   @history
-   @endhistory
-@@*/
-int PUGH_DisableGArrayGroupComm(pGH *pughGH,
-                                int first_var,
-                                pComm *groupcomm)
-{
-#ifdef DEBUG_PUGH
-  pGA *GA;                 /* first variable in group */
-
-  GA = (pGA *) pughGH->variables[first_var][0];
-  printf(" PUGH_DisableGArrayGroupComm: request for group "
-         "with first var '%s'\n", GA->name);
-  fflush(stdout);
-#endif
-
-  /* get rid of compiler warning about unused parameters */
-  first_var = first_var;
-
-  return (PUGH_DisableComm(pughGH, groupcomm));
-}
-
-
- /*@@
+/*@@
    @routine    PUGH_EnableComm
    @date       Sun Jan 23 12:46:23 2000
    @author     Gabrielle Allen
    @desc
-   This sets the docomm[2*dim] array of the GA based
-   on the setting of the comm flag and allocates the comm buffers.
+               This sets the docomm[2*dim] array of the GA based
+               on the setting of the comm flag and allocates the comm buffers.
    @enddesc
    @history
                @date Mon Jun 05 2000 @author Thomas Radke
                Moved buffer allocation from PUGH_EnableGArrayDataStorage
    @endhistory
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid hierarchy extension
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        comm
+   @vdesc      Pointer to comm structure
+   @vtype      pComm *
+   @vio        in
+   @endvar
+   @var        commflag
+   @vdesc      flag indicating in which directions to communicate
+   @vtype      int
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               1 if communication was enabled,<BR>
+               or negative otherwise
+   @endreturndesc
 @@*/
-static int PUGH_EnableComm(pGH *pughGH,
-                           pComm *comm,
-                           int commflag)
+static int PUGH_EnableComm(pGH *pughGH, pComm *comm, int commflag)
 {
   int retval;              /* return value */
 #ifdef CCTK_MPI
@@ -522,7 +644,9 @@ static int PUGH_EnableComm(pGH *pughGH,
 
     /* First set all communcation off */
     for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+    {
       comm->docomm[idir] = 0;
+    }
 
     if (commflag == PUGH_ALLCOMM)
     {
@@ -561,7 +685,6 @@ static int PUGH_EnableComm(pGH *pughGH,
 
     /* Handle nsize = 1 type cases. This is only important for one
        processor MPI periodic boundaries */
-
     for (idir = 0; idir < GA->extras->dim; idir++)
     {
       if (GA->extras->nsize[idir] == 1)
@@ -585,30 +708,39 @@ static int PUGH_EnableComm(pGH *pughGH,
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_DisableComm
    @date       Mon Jun 05 2000
    @author     Thomas Radke
    @desc
-               This frees the communication buffers
-               of a given comm structure.
+               This frees the communication buffers of a given comm structure.
    @enddesc
    @history
                Separated from routine PUGH_DisableGArrayDataStorage()
    @endhistory
+
+   @var        GA
+   @vdesc      Pointer to grid array structure
+   @vtype      pGA *
+   @vio        in
+   @endvar
+   @var        comm
+   @vdesc      Pointer to comm structure
+   @vtype      pComm *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               1 for communication was disabled
+   @endreturndesc
 @@*/
-static int PUGH_DisableComm(pGH *pughGH,
-                            pComm *comm)
+static int PUGH_DisableComm(pGA *GA, pComm *comm)
 {
 #ifdef CCTK_MPI
   int i;                  /* looper */
-  pGA *GA;                /* GA structure the comm structure belongs to */
 
 
-  /* get the GA to which the comm structure belongs to
-     For a comm structure this is the first variable within this group. */
-  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
-
 #ifdef DEBUG_PUGH
   printf (" PUGH_DisableComm: freeing comm buffer for group of %d vars and "
           "first var '%s'\n", comm->n_vars, GA->name);
@@ -641,14 +773,13 @@ static int PUGH_DisableComm(pGH *pughGH,
   {
     CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
                 "PUGH_DisableComm: Communication already disabled for "
-		"group of %d vars "
-                "and first var '%s'", comm->n_vars, GA->name);
+                "group of %d vars and first var '%s'", comm->n_vars, GA->name);
   }
 
 #else
 
   /* get rid of compiler warning about unused parameters */
-  pughGH = pughGH;
+  GA = GA;
   comm = comm;
 
 #endif   /* CCTK_MPI */
@@ -657,7 +788,7 @@ static int PUGH_DisableComm(pGH *pughGH,
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_Sync
    @date       Mon Jun 05 2000
    @author     Thomas Radke
@@ -666,11 +797,24 @@ static int PUGH_DisableComm(pGH *pughGH,
                according to a given comm structure.
    @enddesc
    @calls      PUGH_SyncSingleProc
-   @history
-   @endhistory
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid extensions
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        comm
+   @vdesc      Pointer to comm structure
+   @vtype      pComm *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for success
+   @endreturndesc
 @@*/
-static int PUGH_Sync(pGH *pughGH,
-                     pComm *comm)
+static int PUGH_Sync(pGH *pughGH, pComm *comm)
 {
 #ifdef CCTK_MPI
   int dir;
@@ -686,7 +830,7 @@ static int PUGH_Sync(pGH *pughGH,
 #endif
 
 
-  /* single-processor case in handled in separate routine */
+  /* single-processor case is handled in separate routine */
   if (pughGH->nprocs == 1)
   {
     return (PUGH_SyncSingleProc (pughGH, comm));
@@ -703,7 +847,7 @@ static int PUGH_Sync(pGH *pughGH,
   GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
 
 #ifdef PUGH_WITH_DERIVED_DATATYPES
-  if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
+  if (pughGH->commmodel == PUGH_DERIVEDTYPES)
   {
     /* 2 faces, send and receive is the 2 * 2 */
     sr = (MPI_Request *) malloc(comm->n_vars * 2 * 2 * sizeof(MPI_Request));
@@ -716,7 +860,7 @@ static int PUGH_Sync(pGH *pughGH,
   fflush (stdout);
 #endif
 
-  for (dir = 0; dir < GA->extras->dim; dir ++) 
+  for (dir = 0; dir < GA->extras->dim; dir ++)
   {
 
 #ifdef COMM_TIMING
@@ -725,7 +869,7 @@ static int PUGH_Sync(pGH *pughGH,
 
     PostReceiveGA(pughGH, 2*dir, comm);
     PostReceiveGA(pughGH, 2*dir+1, comm);
-    
+
 #ifdef COMM_TIMING
     t2 = MPI_Wtime();
     printf("PR : %f\n",t2-t1);
@@ -747,8 +891,8 @@ static int PUGH_Sync(pGH *pughGH,
        recieve, but not on the send, since we don't need the
        send buffer until we pack a send again (above)
     */
-    
-    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS) 
+
+    if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
     {
       /* Do a wait any on the receives */
       MPI_Wait(&comm->rreq[2*dir], &mss);
@@ -757,7 +901,7 @@ static int PUGH_Sync(pGH *pughGH,
       FinishReceiveGA(pughGH, 2*dir+1, comm);
     }
 #ifdef PUGH_WITH_DERIVED_DATATYPES
-    else if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
+    else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
     {
       /* Load up the thing for the waitall */
       for (i = 0; i < comm->n_vars; i++)
@@ -802,7 +946,7 @@ static int PUGH_Sync(pGH *pughGH,
   }
 
 #ifdef PUGH_WITH_DERIVED_DATATYPES
-  if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
+  if (pughGH->commmodel == PUGH_DERIVEDTYPES)
   {
     free(sr);
   }
@@ -826,7 +970,7 @@ static int PUGH_Sync(pGH *pughGH,
 }
 
 
- /*@@
+/*@@
    @routine    PUGH_SyncSingleProc
    @date       Sun Jan 14 2001
    @author     Thomas Radke
@@ -834,11 +978,24 @@ static int PUGH_Sync(pGH *pughGH,
                Finally synchronizes a variable or group of variables
                in the single-processor case.
    @enddesc
-   @history
-   @endhistory
+
+   @var        pughGH
+   @vdesc      Pointer to PUGH grid extensions
+   @vtype      pGH *
+   @vio        in
+   @endvar
+   @var        comm
+   @vdesc      Pointer to comm structure
+   @vtype      pComm *
+   @vio        in
+   @endvar
+
+   @returntype int
+   @returndesc
+               0 for success
+   @endreturndesc
 @@*/
-static int PUGH_SyncSingleProc(pGH *pughGH,
-                               pComm *comm)
+static int PUGH_SyncSingleProc(pGH *pughGH, pComm *comm)
 {
   pGA *GA;
   int i, face, dim, copy_bytes, offset_from, offset_to;