Enabled PUGH to synchronize groups of arrays in one communication call

rather than doing it sequentially on individual arrays.
Nothing changes for the CCTK routines overloaded by PUGH.

For BAM and other thorns which don't care about groups there is now
a separate interface to synchronize individual arrays.

Also eliminated the restriction to synchronize 1D and 3D arrays only
- now it should do arbitrary dims (not yet tested).


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@222 b61c5cb5-eaca-4651-9a7a-d64986f99364

4 files changed, 123 insertions, 77 deletions

diff --git a/src/include/pGH.h b/src/include/pGH.h
index 9fe17e7..d5139d9 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -13,10 +13,6 @@
 typedef struct PGH 
 {
 
-#ifdef CCTK_MPI
-  MPI_Comm PUGH_COMM_WORLD;       /* A MPIcommunicator */
-#endif
-
   /* pGH identifier */
   void     *callerid;
   int       dim;                /* The dimension of the GH */
@@ -55,12 +51,16 @@ typedef struct PGH
                                 /* 0 -> GH is skipped for conv.test */
                                 /* is used in NanCheckGH.c */
 
+#if 1
   /* FIXME */
   double    comm_time;          /* time spent in communication */
+#endif
 
 #ifdef CCTK_MPI
-  /* the complex datatype defined in SetupPGH.c */
-  MPI_Datatype PUGH_mpi_complex;
+  MPI_Datatype PUGH_mpi_complex;/* the MPI datatype for COMPLEX types */
+  MPI_Comm PUGH_COMM_WORLD;     /* the MPI communicator */
+
+#ifdef PUGH_WITH_DERIVED_DATATYPES
   /* Derived data types for communication */
   MPI_Datatype send_char_dt [PUGH_NSTAGGER][6];
   MPI_Datatype recv_char_dt [PUGH_NSTAGGER][6];
@@ -71,15 +71,14 @@ typedef struct PGH
   MPI_Datatype send_complex_dt [PUGH_NSTAGGER][6];
   MPI_Datatype recv_complex_dt [PUGH_NSTAGGER][6];
 #endif
+#endif
 
   /* Used for all grid functions */
   pGExtras **GFExtras;          /* [dim] stagger ? */
   pConnectivity **Connectivity; /* [dim] */
 
-  /* FIXME :  should be a dynamically allocated string.
-   * If you change this, please change the line in PUGH_SetupGH.
-   */
-  char identity_string[20];
+  char *identity_string;        /* identifier for this pGH */
+
 } pGH;
 
 #define XDP   1
diff --git a/src/include/pGV.h b/src/include/pGV.h
index 841ecb0..e86afb1 100644
--- a/src/include/pGV.h
+++ b/src/include/pGV.h
@@ -49,6 +49,10 @@ typedef struct PGExtras
   int      *rnpoints;           /* Number of points on each proc */
   int     **rnsize;             /* [#points on a proc][in each dir] */
 
+  /* Copying ghostzones to/from comm buffers */
+  int      *iterator;           /* Iterator for copying ghostzones [dim] */
+  int      *hyper_volume;       /* Points per subcube dimension [dim] */
+
   /* Ghosts and overlaps. */
   int      *nghostzones;        /* Width of ghost zone */
   int      *ownership[PUGH_NSTAGGER][2];        
@@ -65,19 +69,34 @@ typedef struct PGExtras
 
 } pGExtras;
 
+typedef struct PComm
+{
+  /*     buffer_sz[2*dim] */
+  int   *buffer_sz;             /* Size of the face ghost zones */
+  void  **send_buffer;          /* Storage for buffered Comm if */
+  void  **recv_buffer;          /* we don't use derived types */
+  int   commflag;               /* What is the comm flag set to? */
+  /*     do_comm[2*dim] */
+  int   *docomm;                /* Do we do comm or not? */
+
+  int first_var;                /* first variable to be synced */
+  int n_vars;                   /* number of variables to be synced */
+  int sync_timelevel;           /* the timelevel to be synced */
+
+#ifdef CCTK_MPI
+  MPI_Request *sreq, *rreq;     /* Comm requests and statuses. */
+  MPI_Status ms;
+  MPI_Datatype mpi_type;        /* MPI datatype to use for communication */
+#endif
+
+} pComm;
+
 typedef struct PGA
 {
   char  *name;               /* The name of the grid function */
   int   id;                  /* My ID number in my GH parent. */
   void  *padddata;           /* Storage for the data. */
   void  *data;               /* See the note above. */
-  /*     buffer_sz[2*dim] */
-  int   *buffer_sz;          /* Size of the face ghost zones */
-  void  **send_buffer;       /* Storage for buffered Comm if */
-  void  **recv_buffer;       /* we don't use derived types */
-  int   commflag;            /* What is the comm flag set to? */
-  /*     do_comm[2*dim] */
-  int   *docomm;             /* Do we do comm or not? */
   int   storage;             /* Do we have storage or not? */
   int   stagger;             /* Only Vertex Centered now... */
 
@@ -92,13 +111,12 @@ typedef struct PGA
   int   varsize;             /* The size of the data */
   int   vtype;               /* The type of the data */
 
-#ifdef CCTK_MPI
-  MPI_Request *sreq, *rreq;      /* Comm requests and statuses. */
-  MPI_Status ms;
-#endif
-
   pGExtras *extras;
   pConnectivity *connectivity;
+
+  pComm *comm;               /* comm buffer for single variable */
+  pComm *groupcomm;          /* comm buffer for a variable group */
+
 } pGA;
 
 typedef struct PGV
diff --git a/src/include/pugh.h b/src/include/pugh.h
index ce8c89b..7408c0f 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -72,18 +72,21 @@
 
 #ifdef CCTK_MPI
 
-#define CACTUS_MPI_ERROR(xf)  do {int errcode; \
-                                    if((errcode = xf) != MPI_SUCCESS)                     \
-                                    {                                                     \
-                                      char mpi_error_string[MPI_MAX_ERROR_STRING+1];      \
-                                      int resultlen;                                      \
-                                      MPI_Error_string(errcode, mpi_error_string, &resultlen);\
-                                      fprintf(stderr, "MPI Call %s returned error code %d (%s)\n", \
-                                      #xf, errcode, mpi_error_string);                    \
-                                      fprintf(stderr, "At line %d of file %s\n",                   \
-                                             __LINE__, __FILE__);                         \
-                                    }                                                     \
-                                  } while (0)
+#define CACTUS_MPI_ERROR(fn_call)                                             \
+          do {                                                                \
+            int errcode;                                                      \
+                                                                              \
+            if ((errcode = fn_call) != MPI_SUCCESS)                           \
+            {                                                                 \
+              char mpi_error_string[MPI_MAX_ERROR_STRING+1];                  \
+              int resultlen;                                                  \
+                                                                              \
+              MPI_Error_string (errcode, mpi_error_string, &resultlen);       \
+              fprintf (stderr, "MPI call '%s' returned error code %d (%s)\n", \
+                               #fn_call, errcode, mpi_error_string);          \
+              fprintf(stderr, "At line %d of file %s\n", __LINE__, __FILE__); \
+            }                                                                 \
+          } while (0)
 #endif
 
 #ifdef _cplusplus
@@ -93,21 +96,21 @@ extern "C"
 
 
 int PUGH_SetupGroup(pGH *newGH, 
-		    int *nsize,
-		    int *nghostsize,
-		    int  gtype, 
-		    int  vtype, 
-		    int  dim, 
-		    int  n_variables,
-		    int  staggertype,
-		    int  n_timelevels);
+                    int *nsize,
+                    int *nghostsize,
+                    int  gtype, 
+                    int  vtype, 
+                    int  dim, 
+                    int  n_variables,
+                    int  staggertype,
+                    int  n_timelevels);
 
 pGH *PUGH_SetupPGH(void *callerid, 
                    int dim, 
                    int *nsize, 
                    int *nghostzones,
-		   int staggertype,
-		   int *perme);
+                   int staggertype,
+                   int *perme);
 
 int PUGH_GFSize(int dim, 
                 int *nsize);
@@ -115,7 +118,7 @@ int PUGH_GFSize(int dim,
 int PUGH_GFGhostsize(int dim,
                      int *ghostsize);
 int PUGH_GFPeriodic(int dim,
-		    int *perme);
+                    int *perme);
 
 pGH *PUGH_pGH(cGH *GH);
 
diff --git a/src/include/pughi.h b/src/include/pughi.h
index 8defba7..f6fdf1c 100644
--- a/src/include/pughi.h
+++ b/src/include/pughi.h
@@ -30,22 +30,22 @@ pGExtras *PUGH_SetupPGExtras(int dim,
                              int *nprocs,
                              int this_proc);
 
-pGA *SetupPGA(void          *parent,
-	      pGExtras      *extras,
-	      pConnectivity *connectivity,
-	      const char    *name,
-	      int            id,
-	      int            varsize, 
-	      int            vtype, 
-	      int            stagger);      
-
-
-void pGH_DumpInfo(pGH *GH);
+pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
+                                 int dim,
+                                 int first_var,
+                                 int n_vars,
+                                 int sync_timelevel,
+                                 int vartype,
+                                 pGExtras *extras);
+
+void pGH_DumpInfo(pGH *pughGH);
 int  PUGH_SetupDefaultTopology(int dim, int *nprocs);
-int  pGH_SetupnProcs(pGH *GH,int dim);
-void PUGH_DestroyPGA(pGA **GA);
+int  pGH_SetupnProcs(pGH *pughGH,int dim);
+void PUGH_DestroyGArray(pGA **GA);
+void PUGH_DestroyComm(pComm **comm);
 void PUGH_DestroyConnectivity(pConnectivity **conn);
 void PUGH_DestroyPGExtras(pGExtras **PGExtras);
+void PUGH_DestroyPGH(pGH **pughGH);
 
 int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs);
 
@@ -104,25 +104,6 @@ int PUGH_SetupRemoteSizes(int dim,
                           int *nprocs,
                           pGExtras *this);
 
-pGA *PUGH_SetupPGA(void *parent,
-                   pGExtras *extras,
-                   pConnectivity *connectivity,
-                   const char *name,
-                   int id,
-                   int varsize, 
-                   int vtype, 
-                   int stagger);
-
-int PUGH_EnablePGAStorage(pGA *GA, 
-                          int this_proc,
-                          int zero_memory, 
-                          int padding_active,
-                          int padding_cacheline_bits,
-                          int padding_size,
-                          int padding_address_spacing);
-
-int PUGH_DisableGADataStorage(pGA *GA);
-
 int PUGH_SetupPGExtrasStaggering(int dim,
                                  int *perme,
                                  int stagger,
@@ -133,6 +114,51 @@ int PUGH_SetupPGExtrasStaggering(int dim,
                                  int this_proc,
                                  pGExtras *this);
 
+pGA *PUGH_SetupGArray(void *parent,
+                      pGExtras *extras,
+                      pConnectivity *connectivity,
+                      pComm *comm,
+                      const char *name,
+                      int id,
+                      int varsize, 
+                      int vtype, 
+                      int stagger);
+
+pComm *PUGH_SetupGArrayComm(pGH *pughGH,
+                            int dim,
+                            int var,
+                            int sync_timelevel,
+                            int vartype,
+                            pGExtras *extras);
+
+int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
+                                  int first_var,
+                                  int n_variables,
+                                  int n_timelevels);
+
+int PUGH_EnableGArrayDataStorage(pGA *GA,
+                                 int this_proc,
+                                 int zero_memory,
+                                 int padding_active,
+                                 int padding_cacheline_bits,
+                                 int padding_size,
+                                 int padding_address_spacing);
+
+int PUGH_DisableGArrayDataStorage(pGA *GA);
+
+int PUGH_EnableGArrayComm(pGA *GA,
+                          int commflag);
+
+int PUGH_DisableGArrayComm(pGA *GA);
+
+int PUGH_SyncGArray(pGA *GA);
+
+#ifdef CCTK_MPI
+void PostSendGA     (pGH *pughGH, int dir, pComm *comm);
+void PostReceiveGA  (pGH *pughGH, int dir, pComm *comm);
+void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm);
+#endif
+
 #ifdef _cplusplus
 }
 #endif