Enabled PUGH to synchronize groups of arrays in one communication call

rather than doing it sequentially on individual arrays.
Nothing changes for the CCTK routines overloaded by PUGH.

For BAM and other thorns which don't care about groups there is now
a separate interface to synchronize individual arrays.

Also eliminated the restriction to synchronize 1D and 3D arrays only
- now it should do arbitrary dims (not yet tested).


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@222 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 72 insertions, 58 deletions

diff --git a/src/PostReceiveGA.c b/src/PostReceiveGA.c
index 3a7864b..caacff0 100644
--- a/src/PostReceiveGA.c
+++ b/src/PostReceiveGA.c
@@ -3,9 +3,9 @@
    @date      Wed Feb 2 11:28:06 1997
    @author    Gabrielle Allen
    @desc 
-   Routines which post all the IRecv commands for a GA. These
-   allow the asyncronous model proposed in, for example,
-   @seeroutine SyncGA to go!
+   Routines which post all the IRecv commands for a single sync.
+   These allow the asyncronous model proposed in, for example,
+   @seeroutine PUGH_Sync to go!
    @enddesc 
    @version $Header$
  @@*/
@@ -21,6 +21,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
 /*#define DEBUG_PUGH*/
 
 
+#ifdef CCTK_MPI
+
  /*@@
    @routine   PostReceiveGA
    @date      Thu Apr  3 12:10:46 1997
@@ -28,8 +30,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
    @desc 
    This routine posts a recieve (MPI_Irecv) of the buffer
    we want to get the send from another processor, which
-   will be sent by @seeroutine pGA_PostSend and then
-   finalized by @seeroutoine pGA_FinishRecv. 
+   will be sent by @seeroutine PostSendGA and then
+   finalized by @seeroutoine FinishRecvGA. 
    <p>
    Aside from a silly calculation to get a unique tag
    based on neigbors and processors, this is a very
@@ -41,83 +43,95 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
    @endhistory
  @@*/
 
-void PostReceiveGA(pGH *GH, pGA *GA, int dir)
+void PostReceiveGA(pGH *pughGH, int dir, pComm *comm)
 {
-#ifdef CCTK_MPI
+  pGA *GA;
   int rtag;
-  MPI_Datatype mpi_type;
-  MPI_Datatype *recv_dt;
+  int neighbour;
 
-  if (!(GA->storage)) 
+  GA = (pGA *) pughGH->variables[comm->first_var][0];
+
+  /* return if no storage assigned */
+  if (! GA->storage)
   {
     CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
                "Trying to synchronize variable '%s' with no storage", GA->name);
     return;
   }
 
-  /* Return if communication not required and no forced synchronisation */
-  if (!(GH->forceSync || GA->docomm[dir])) 
+  /* return if communication not required and no forced synchronisation */
+  if (! (pughGH->forceSync || comm->docomm[dir]))
   {
     return;
   }
 
-  if (GA->connectivity->neighbours[GH->myproc][dir] < 0) 
+  /* return if there is no neighbour in the given direction */
+  neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
+  if (neighbour < 0)
   {
     return;
   }
 
-  switch (GA->vtype) 
-  {
-    case CCTK_VARIABLE_CHAR:
-      mpi_type = PUGH_MPI_CHAR;
-      recv_dt = GH->recv_char_dt [GA->stagger];
-      break;
-
-    case CCTK_VARIABLE_INT:
-      mpi_type = PUGH_MPI_INT;
-      recv_dt = GH->recv_int_dt [GA->stagger];
-      break;
-
-    case CCTK_VARIABLE_REAL:
-      mpi_type = PUGH_MPI_REAL;
-      recv_dt = GH->recv_real_dt [GA->stagger];
-      break;
-
-    case CCTK_VARIABLE_COMPLEX:
-      mpi_type = GH->PUGH_mpi_complex;
-      recv_dt = GH->recv_complex_dt [GA->stagger];
-      break;
-
-    default:
-      CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
-      return;
-  }
-
-  rtag = 1000 + dir + 2 * (GH->myproc + GH->nprocs * GA->id);
+  /* note this is the complement of the stag set in PostSendGA */
+  rtag = 1000 + dir + 2 * (pughGH->myproc + pughGH->nprocs * GA->id);
   /* mod the tag to force MPI compliance */
   rtag = rtag % 32768;
+
 #ifdef DEBUG_PUGH
-  printf ("RECV GA %s Side %d Proc %d rtag %d size %d from proc %d\n",
-          GA->name,
-          dir,
-          GH->myproc,
-          rtag,
-          GA->buffer_sz[dir],
-          GA->connectivity->neighbours[GH->myproc][dir]);
+  printf ("PostReceiveGA: into direction %d from proc %d with rtag %d size %d "
+          "for %d vars starting from '%s'\n",
+          dir, neighbour, rtag, comm->buffer_sz[dir], comm->n_vars, GA->name);
 #endif
-  if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
+
+  if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
   {
-    CACTUS_MPI_ERROR (MPI_Irecv (GA->recv_buffer [dir],
-                      GA->buffer_sz [dir], mpi_type,
-                      GA->connectivity->neighbours [GH->myproc][dir], rtag,
-                      GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+    CACTUS_MPI_ERROR (MPI_Irecv (comm->recv_buffer[dir],
+                                 comm->buffer_sz[dir], comm->mpi_type,
+                                 neighbour, rtag,
+                                 pughGH->PUGH_COMM_WORLD, &comm->rreq[dir]));
   }
-  if (GH->commmodel == PUGH_DERIVEDTYPES)
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+  else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
   {
-    CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
-                      1, recv_dt [dir],
-                      GA->connectivity->neighbours [GH->myproc][dir], rtag,
-                      GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+    int var;
+    MPI_Datatype *recv_dt;
+
+
+    switch (GA->vtype) 
+    {
+      case CCTK_VARIABLE_CHAR:
+        recv_dt = pughGH->recv_char_dt[GA->stagger];
+        break;
+
+      case CCTK_VARIABLE_INT:
+        recv_dt = pughGH->recv_int_dt[GA->stagger];
+        break;
+
+      case CCTK_VARIABLE_REAL:
+        recv_dt = pughGH->recv_real_dt[GA->stagger];
+        break;
+
+      case CCTK_VARIABLE_COMPLEX:
+        recv_dt = pughGH->recv_complex_dt[GA->stagger];
+        break;
+
+      default:
+        CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
+        return;
+    }
+
+    for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
+    {
+      pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
+
+
+      CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
+                                   1, recv_dt[dir],
+                                   neighbour, (rtag + var) % 32768,
+                                   pughGH->PUGH_COMM_WORLD,
+                                   &GA->comm->rreq[dir]));
+    }
   }
 #endif
 }
+#endif   /* CCTK_MPI */