Erik's changes to PUGH's communication scheme.

Now send requests aren't simply freed anymore but rather explicitely finished
by calling MPI_Waitall() on all of them for a given communication.

This solves the problem of MPICH and LAM on some Linux SMP machines
where it needed a long time to complete a communication.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@268 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 12 insertions, 3 deletions

diff --git a/src/Comm.c b/src/Comm.c
index d4614b5..5b177e7 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -657,6 +657,8 @@ static int PUGH_Sync(pGH *pughGH,
   CCTK_TimerStartI(pughGH->comm_time);
 #endif
 
+  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
+
 #ifdef PUGH_WITH_DERIVED_DATATYPES
   if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
   {
@@ -665,8 +667,6 @@ static int PUGH_Sync(pGH *pughGH,
   }
 #endif
 
-  GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
-
 #ifdef DEBUG_PUGH
   printf (" PUGH_Sync: syncing group of %d vars with first var '%s'\n",
           comm->n_vars, GA->name);
@@ -756,8 +756,17 @@ static int PUGH_Sync(pGH *pughGH,
   }
 
 #ifdef PUGH_WITH_DERIVED_DATATYPES
-  free(sr);
+  if (pughGH->commmodel == PUGH_DERIVEDTYPES) 
+  {
+    free(sr);
+  }
+  else
 #endif
+  {
+    /* wait for MPI to finish all outstanding send requests */
+    CACTUS_MPI_ERROR (MPI_Waitall (2 * GA->extras->dim, comm->sreq,
+                                   comm->sstatus));
+  }
 
 #if 0
   /* get the time spent in communication */