Enabled PUGH to synchronize groups of arrays in one communication call

rather than doing it sequentially on individual arrays.
Nothing changes for the CCTK routines overloaded by PUGH.

For BAM and other thorns which don't care about groups there is now
a separate interface to synchronize individual arrays.

Also eliminated the restriction to synchronize 1D and 3D arrays only
- now it should do arbitrary dims (not yet tested).


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@222 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 75 insertions, 16 deletions

diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index 441ed59..3d4b1b0 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -13,6 +13,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
+#include <assert.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #include "cctk.h"
 #include "pugh.h"
@@ -46,7 +51,7 @@ pGH *PUGH_SetupPGH(void *callerid,
                    int *nsize, 
                    int *nghostzones,
                    int staggertype,
-		   int *perme) 
+                   int *perme) 
 {
   DECLARE_CCTK_PARAMETERS
 
@@ -77,28 +82,34 @@ pGH *PUGH_SetupPGH(void *callerid,
 
     GH->Connectivity[idim-1]=
       PUGH_SetupConnectivity(idim, 
-			     GH->nprocs, 
-			     nprocs,
-			     perme);
+                             GH->nprocs, 
+                             nprocs,
+                             perme);
     free(nprocs);
       
     GH->GFExtras[idim-1]=
       PUGH_SetupPGExtras(idim, 
-			 perme,
-			 staggertype,
-			 nsize,
-			 nghostzones,
-			 GH->nprocs,
-			 ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
-			 GH->myproc);
+                         perme,
+                         staggertype,
+                         nsize,
+                         nghostzones,
+                         GH->nprocs,
+                         ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
+                         GH->myproc);
 
   }
 
   GH->active = 1;
   GH->commmodel = PUGH_ALLOCATEDBUFFERS;
+
 #if 0
-  CactusResetTimer (&GH->comm_time);
+  /* create the timer for communication time */
+  if(comm_verbose)
+  {
+    GH->comm_time = CCTK_TimerCreateI ();
+  }
 #endif
+
   /* set staggering flag */
   /*GH->stagger = staggertype;                 */
   GH->level = 0;
@@ -201,7 +212,8 @@ void PUGH_DestroyPGH(pGH **GHin)
   {
 
 #ifdef DEBUG_PUGH
-    printf("Calling Destroying Group %s \n",CCTK_GroupName(group));
+    printf("Calling Destroying Group %s\n", CCTK_GroupName(group));
+    fflush(stdout);
 #endif 
 
     CCTK_GroupData(group,&pgroup);
@@ -214,7 +226,7 @@ void PUGH_DestroyPGH(pGH **GHin)
         {
           case GROUP_GF:
           case CCTK_ARRAY:
-            PUGH_DestroyPGA(&(((pGA ***)GH->variables)[variable][i]));
+            PUGH_DestroyGArray(&(((pGA ***)GH->variables)[variable][i]));
             break;
           case GROUP_SCALAR:
             free(GH->variables[variable][i]);
@@ -231,13 +243,12 @@ void PUGH_DestroyPGH(pGH **GHin)
     PUGH_DestroyPGExtras(&GH->GFExtras[i-1]);
   }
 
+  free(GH->identity_string);
   free(GH->Connectivity);
   free(GH->GFExtras);
   free(GH->variables);
   free(GH);
   *GHin = NULL;
-
-  fflush(stdout);
 }
 
 void pGH_DumpInfo(pGH *GH) 
@@ -323,3 +334,51 @@ int PUGH_SetupDefaultTopology(int dim, int *nprocs)
   return retval;
 }
 
+
+int PUGH_ParallelInit(cGH *GH)
+{
+  return 0;
+}
+
+int PUGH_Exit(int retval, cGH *GH)
+{
+#ifdef CCTK_MPI
+  CACTUS_MPI_ERROR(MPI_Finalize());
+#endif
+  exit(retval);
+  return(retval);
+}
+int PUGH_Abort(cGH *GH)
+{
+  assert(0);
+  exit(0);
+  return(0);
+}
+
+
+int PUGH_MyProc(cGH *GH)
+{
+  int myproc;
+
+#ifdef CCTK_MPI
+  CACTUS_MPI_ERROR(MPI_Comm_rank(PUGH_pGH(GH)->PUGH_COMM_WORLD, &myproc));
+#else
+  myproc = 0;
+#endif
+
+  return myproc;
+}
+
+
+int PUGH_nProcs(cGH *GH)
+{
+  int nprocs;
+
+#ifdef CCTK_MPI
+  CACTUS_MPI_ERROR(MPI_Comm_size(PUGH_pGH(GH)->PUGH_COMM_WORLD, &nprocs));
+#else
+  nprocs = 1;
+#endif
+
+  return nprocs;
+}