aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authortradke <tradke@b61c5cb5-eaca-4651-9a7a-d64986f99364>2000-06-14 16:46:53 +0000
committertradke <tradke@b61c5cb5-eaca-4651-9a7a-d64986f99364>2000-06-14 16:46:53 +0000
commit63815410954f7a8cc19bfaaf9ddc5ea3a8db2463 (patch)
treeb8b953cc09358a26d8995ca1191525d6f8ffed1b
parent6bbd2b0ad4481a7c4e9b2ecc32e2efc1d755cbab (diff)
Enabled PUGH to synchronize groups of arrays in one communication call
rather than doing it sequentially on individual arrays. Nothing changes for the CCTK routines overloaded by PUGH. For BAM and other thorns which don't care about groups there is now a separate interface to synchronize individual arrays. Also eliminated the restriction to synchronize 1D and 3D arrays only - now it should do arbitrary dims (not yet tested). git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@222 b61c5cb5-eaca-4651-9a7a-d64986f99364
Diffstat
-rw-r--r--src/Comm.c1266
-rw-r--r--src/Evolve.c2
-rw-r--r--src/FinishReceiveGA.c239
-rw-r--r--src/GHExtension.c15
-rw-r--r--src/LoadAware.c1
-rw-r--r--src/PostReceiveGA.c130
-rw-r--r--src/PostSendGA.c341
-rw-r--r--src/Reduction.c83
-rw-r--r--src/SetupGroup.c121
-rw-r--r--src/SetupPGH.c91
-rw-r--r--src/SetupPGV.c559
-rw-r--r--src/Startup.c10
-rw-r--r--src/Storage.c561
-rw-r--r--src/include/pGH.h19
-rw-r--r--src/include/pGV.h42
-rw-r--r--src/include/pugh.h49
-rw-r--r--src/include/pughi.h90
-rw-r--r--src/make.code.defn6
-rw-r--r--src/pugh_extension.h2
19 files changed, 1866 insertions, 1761 deletions
diff --git a/src/Comm.c b/src/Comm.c
index 528ec63..31e2350 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -14,15 +14,11 @@
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
-#include <assert.h>
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
#include "cctk.h"
#include "pugh.h"
+#include "pughi.h"
#include "pugh_Comm.h"
#include "cctk_Parameters.h"
@@ -30,41 +26,23 @@ static char *rcsid="$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_Comm_c)
-static int totalstorage=0; /* Storage for GAs in Bytes */
-static int totalnumber=0; /* Number of stored GAs */
-
-/* index into GH->extensions[] to get the PUGH extension handle */
-extern int pugh_GHExtension;
/* local function prototypes */
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+ int first_var,
+ int commflag);
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+ int first_var);
+static int PUGH_EnableComm(pGH *pughGH,
+ pComm *comm,
+ int commflag);
+static int PUGH_DisableComm(pGH *pughGH,
+ pComm *comm);
+static int PUGH_SyncGArrayGroup(pGH *pughGH,
+ int first_var);
+static int PUGH_Sync(pGH *pughGH,
+ pComm *comm);
-int PUGH_SyncGroupGA (cGH *GH,
- int group,
- int vtype,
- int n_vars,
- int timelevel);
-int PUGH_SyncGAs(pGH *pughGH,
- int dim,
- int first_var,
- int n_vars,
- int timelevel);
-int PUGH_EnableArrayGroupStorage(cGH *GH,
- int group,
- int vtype,
- int dim,
- int n_variables,
- int n_timelevels);
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing);
-void FinishReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostSendGA(pGH *GH, pGA *GA, int dir);
-int PUGH_GAComm(pGA *GA, int docomm);
/*@@
@routine PUGH_SyncGroup
@@ -75,7 +53,7 @@ int PUGH_GAComm(pGA *GA, int docomm);
Only groups of type GROUP_ARRAY and GROUP_GF can be synchronized.
@enddesc
@calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- PUGH_SyncGroupArray PUGH_SyncGroupGF
+ PUGH_SyncGArrayGroup
@history
@endhistory
@@ -91,66 +69,48 @@ int PUGH_GAComm(pGA *GA, int docomm);
@endvar
@@*/
-int PUGH_SyncGroup (cGH *GH, const char *groupname)
+int PUGH_SyncGroup(cGH *GH, const char *groupname)
{
- cGroup pgroup; /* pointer to group information */
+ cGroup pgroup; /* group information */
int group; /* group index */
- int timelevel; /* timelevel to be synchronized */
int rc; /* return code */
- group = CCTK_GroupIndex(groupname);
+#ifdef DEBUG_PUGH
+ printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
+ fflush (stdout);
+#endif
/* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
-#if 0
- /* Compute the timelevel to synchronize */
- if (pgroup.numtimelevels > 1)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- timelevel = pgroup.numtimelevels - 2;
+ rc = 0;
+ CCTK_WARN(4, "Synchronising a scalar in PUGH");
}
- else
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- timelevel = pgroup.numtimelevels - 1;
+ rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
}
-#endif
-
- timelevel = pgroup.numtimelevels - 1;
-
- /* branch to synchronization function according to group type */
- switch (pgroup.grouptype)
+ else
{
- case GROUP_SCALAR :
- rc = 0;
- CCTK_WARN(4,"Synchronising a scalar in PUGH");
- break;
-
- case GROUP_ARRAY :
- case GROUP_GF :
- rc = PUGH_SyncGroupGA (GH,
- group,
- pgroup.vartype,
- pgroup.numvars,
- timelevel);
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_SyncGroup");
- rc = 0;
- break;
+ CCTK_WARN(1, "Unknown group type in PUGH_SyncGroup");
+ rc = 0;
}
return (rc);
}
+
/*@@
- @routine PUGH_ArrayGroupSize
- @author Gabrielle Allen
- @date 11 Apr 1999
+ @routine PUGH_EnableGroupComm
+ @author Thomas Radke
+ @date 30 Mar 1999
@desc
- Returns size of arrays in a group, in a given direction
+ Enables communication for all variables in the group indicated by groupname.
@enddesc
- @calls
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
@history
@endhistory
@@ -163,109 +123,52 @@ int PUGH_SyncGroup (cGH *GH, const char *groupname)
@vdesc name of the group to be synchronized
@vtype const char *
@vio in
- @vcomment either index or groupname must be present
@endvar
- @var group
- @vdesc index of group
- @vtype int
- @vio in
- @vcomment either index or groupname must be given
- @endvar
- @var dir
- @vdesc direction
- @vtype int
- @vio in
@@*/
-const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
+int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
{
- int first;
- int *sizep=NULL;
- pGH *pughGH;
- pGA *pughGA;
-
- if (groupname)
- {
- group = CCTK_GroupIndex(groupname);
- }
+ int group; /* group index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
- /* get first variable in group */
- first = CCTK_FirstVarIndexI(group);
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
- if (CCTK_GroupTypeI(group)==GROUP_GF)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- /* get pointer to pGF for a timelevel of this variable */
- pughGA = (pGA *)(pughGH->variables[first][0]);
-
- if (pughGA->storage == PUGH_STORAGE)
- {
- switch (dir)
- {
- case 0:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[0];
- break;
- case 1:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[1];
- break;
- case 2:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[2];
- break;
- default :
- sizep = NULL;
- printf("Wrong value for dir in PUGH_ArrayGroupSize\n");
- break;
- }
- }
- else
- {
- sizep = &_cctk_one;
- }
+ rc = 1;
}
- else if (CCTK_GroupTypeI(group)==GROUP_ARRAY)
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- /* get pointer to pGA for a timelevel of this variable */
- pughGA = (pGA *)(pughGH->variables[first][0]);
-
- if (pughGA->storage == PUGH_STORAGE)
- {
- switch (dir)
- {
- case 0:
- sizep = &pughGA->extras->lnsize[0];
- break;
- case 1:
- sizep = &pughGA->extras->lnsize[1];
- break;
- case 2:
- sizep = &pughGA->extras->lnsize[2];
- break;
- default :
- sizep = NULL;
- CCTK_WARN(1,"Wrong value for dir in PUGH_ArrayGroupSize");
- break;
- }
- }
- else
- {
- sizep = &_cctk_one;
- }
+ rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
+ CCTK_FirstVarIndexI(group),
+ PUGH_ALLCOMM);
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
+ rc = 0;
}
- return (sizep);
+ return (rc);
}
/*@@
- @routine PUGH_QueryGroupStorage
- @author Gabrielle Allen
- @date 13 Apr 1999
+ @routine PUGH_DisableGroupComm
+ @author Thomas Radke
+ @date 30 Mar 1999
@desc
- Returns true if group has storage assigned, false otherwise
+ Disables communication for all variables in the group indicated by groupname.
@enddesc
- @calls CCTK_DecomposeName
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
@history
@endhistory
@@ -275,595 +178,523 @@ const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupna
@vio in
@endvar
@var groupname
- @vdesc name of the group to be queried
+ @vdesc name of the group to be synchronized
@vtype const char *
@vio in
- @vcomment either index or groupname must be present
@endvar
- @var group
- @vdesc index of group
- @vtype int
- @vio in
- @vcomment either index or groupname must be given
- @endvar
@@*/
-int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
+int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
{
+ int group; /* group index */
+ cGroup pgroup; /* pointer to group information */
+ int rc; /* return code */
- int first;
- int storage=PUGH_UNDEFINEDSTORAGE;
- int retval;
- int grouptype;
- pGH *pughGH;
-
- if (groupname)
- {
- group = CCTK_GroupIndex(groupname);
- }
-
- /* get first variable in group */
- first = CCTK_FirstVarIndexI(group);
- if (first < 0)
- {
- CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
- "Invalid group ID %d in PUGH_QueryGroupStorage", group);
- }
-
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-
- grouptype = CCTK_GroupTypeI(group);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
- if (grouptype == CCTK_SCALAR)
- {
- storage = PUGH_STORAGE;
- }
- else if (grouptype == CCTK_GF)
- {
- storage = ((pGA *)(pughGH->variables[first][0]))->storage;
- }
- else if (grouptype == CCTK_ARRAY)
- {
- storage = ((pGA *)(pughGH->variables[first][0]))->storage;
- }
- else
- {
- CCTK_WARN(0,"Unknown group type in PUGH_QueryGroupStorage");
- }
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
- if (storage == PUGH_STORAGE)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- retval = 1;
+ rc = 1;
}
- else if (storage == PUGH_NOSTORAGE)
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- retval = 0;
+ rc = PUGH_DisableGArrayGroupComm(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
}
else
{
- CCTK_WARN(0,"Inconsistency in PUGH_QueryGroupStorage");
- retval = PUGH_ERROR;
+ CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
+ rc = 0;
}
- return (retval);
+ return (rc);
}
-/*@@
- @routine PUGH_EnableGroupStorage
- @author Tom Goodale
- @date 30 Mar 1999
- @desc
- Enables storage for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- PUGH_EnableArrayGroupStorage
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
-
-int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_EnableGArrayComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Enables communication for a single array.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_EnableGArrayComm(pGA *GA,
+ int commflag)
{
- DECLARE_CCTK_PARAMETERS
-
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- int rc; /* return code */
- pGH *pughGH;
- int first_var;
-
#ifdef DEBUG_PUGH
- printf(" PUGH_EnableGroupStorage: request for group -%s-\n",groupname);
+ printf(" PUGH_EnableGArrayComm: request for var '%s' commflag %d\n",
+ GA->name, commflag);
fflush(stdout);
#endif
- group = CCTK_GroupIndex(groupname);
-
- /* get the group info from its index */
- CCTK_GroupData (group, &pgroup);
-
- /* branch to "enable storage" function according to group type */
- switch (pgroup.grouptype)
- {
- case CCTK_SCALAR :
- rc = 1;
- break;
-
- case CCTK_ARRAY :
- case CCTK_GF :
- rc = PUGH_EnableArrayGroupStorage (GH,
- group,
- pgroup.vartype,
- pgroup.dim,
- pgroup.numvars,
- pgroup.numtimelevels);
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupStorage");
- rc = 0;
- break;
- }
-
- /* Report on memory usage */
- if (storage_verbose)
- {
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ return (PUGH_EnableComm((pGH *) GA->parent, GA->comm, commflag));
+}
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
- if (rc == 0)
- {
- /* Memory toggled */
- totalnumber = totalnumber + pgroup.numvars;
- totalstorage = totalstorage +
- ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
- *CCTK_VarTypeSize(pgroup.vartype)
- *pgroup.numtimelevels
- *pgroup.numvars;
- printf("Switched memory on for %s \n [Num Arrays: %d Total Size: %d]\n",
- groupname,totalnumber,totalstorage);
- }
- else if (rc == 1)
- {
- /* Memory already on */
- printf("Memory already on for %s\n",groupname);
- }
- }
+ /*@@
+ @routine PUGH_DisableGArrayComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Disables communication for a single array.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_DisableGArrayComm(pGA *GA)
+{
- return (rc);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayComm: request for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
+ return (PUGH_DisableComm((pGH *) GA->parent, GA->comm));
}
-/*@@
- @routine PUGH_DisableGroupStorage
- @author Tom Goodale
- @date 30 Mar 1999
- @desc
- Disables storage for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
-
-int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_SyncGArrayGroup
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Synchronizes a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_SyncGArrayGroup(pGH *pughGH,
+ int first_var)
{
- DECLARE_CCTK_PARAMETERS
+ pGA *firstGA;
+
+
+ firstGA = (pGA *) pughGH->variables [first_var][0];
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- int rc; /* return code */
- pGH *pughGH; /* PUGH extension reference */
- int level;
- int first_var,var;
- int unchanged; /* count how many aren't toggled */
- int PUGH_DisableGADataStorage(pGA *GA);
-
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-
#ifdef DEBUG_PUGH
- printf(" PUGH_DisableGroupStorage: request for group -%s-\n",groupname);
+ printf(" PUGH_SyncGArrayGroup: request for group with first var '%s'\n",
+ firstGA->name);
fflush(stdout);
#endif
- group = CCTK_GroupIndex(groupname);
+ return (PUGH_Sync(pughGH, firstGA->groupcomm));
+}
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ /*@@
+ @routine PUGH_SyncGArray
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Synchronizes a single array variable given the GA structure
+ of this variable.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_SyncGArray(pGA *GA)
+{
- unchanged = 0;
- switch (pgroup.grouptype)
- {
- case GROUP_SCALAR :
- rc = 1;
- break;
+#ifdef DEBUG_PUGH
+ printf(" PUGH_SyncGArray: request for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- {
- for (level = 0; level < pgroup.numtimelevels; level++)
- {
- unchanged = unchanged +
- PUGH_DisableGADataStorage ((pGA *)(pughGH->variables[var][level]));
- }
- }
- rc = 1;
- break;
+ return (PUGH_Sync((pGH *) GA->parent, GA->comm));
+}
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
- rc = 0;
- break;
- }
- /* Report on memory usage */
- if (storage_verbose)
- {
- if (unchanged == 0)
- {
- /* Memory toggled */
- totalnumber = totalnumber - pgroup.numvars;
- totalstorage = totalstorage -
- ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
- *CCTK_VarTypeSize(pgroup.vartype)
- *pgroup.numtimelevels
- *pgroup.numvars;
- printf("Switched memory off for %s \n [Num Arrays: %d Total Size: %d]\n",
- groupname,totalnumber,totalstorage);
- }
- else if (unchanged == pgroup.numvars)
- {
- /* Memory already off */
- printf("Memory already off for %s\n",groupname);
- }
- else
- {
- CCTK_WARN(1,"Inconsistency in group memory assignment");
- }
- }
+int PUGH_Barrier(cGH *GH)
+{
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Barrier(PUGH_pGH(GH)->PUGH_COMM_WORLD));
+#endif
- return (rc);
+ return 0;
}
-/*@@
- @routine PUGH_EnableGroupComm
- @author Thomas Radke
- @date 30 Mar 1999
- @desc
- Enables communication for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
+/*****************************************************************************/
+/* local functions */
+/*****************************************************************************/
-int PUGH_EnableGroupComm (cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_EnableGArrayGroupComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Enables communication for a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+ int first_var,
+ int commflag)
{
- cGroup pgroup; /* pointer to group information */
- int group; /* group index */
- pGH *pughGH; /* PUGH extension reference */
- int var, level; /* loop variables */
- int first_var; /* global index of first variable in group */
- int rc; /* return code */
+ pGA *GA; /* first variable in group */
- group = CCTK_GroupIndex(groupname);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ GA = pughGH->variables [first_var][0];
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayGroupComm: request for group "
+ "with first var '%s', commflag %d\n", GA->name, commflag);
+ fflush(stdout);
+#endif
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ return (PUGH_EnableComm(pughGH, GA->groupcomm, commflag));
+}
- /* branch to "enable comm" function according to group type */
- switch (pgroup.grouptype)
- {
- case GROUP_SCALAR :
- rc = 1;
- break;
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- {
- for (level = 0; level < pgroup.numtimelevels; level++)
- {
- PUGH_GAComm ((pGA *)(pughGH->variables[var][level]),
- PUGH_ALLCOMM);
- }
- }
- rc = 1;
- break;
+ /*@@
+ @routine PUGH_DisableGArrayGroupComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Disables communication for a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+ int first_var)
+{
+ pGA *GA; /* first variable in group */
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupComm");
- rc = 0;
- break;
- }
- return (rc);
-}
+ GA = pughGH->variables [first_var][0];
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayGroupComm: request for group "
+ "with first var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
-/*@@
- @routine PUGH_DisableGroupComm
- @author Thomas Radke
- @date 30 Mar 1999
- @desc
- Disables communication for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
+ return (PUGH_DisableComm(pughGH, GA->groupcomm));
+}
-int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
+
+ /*@@
+ @routine PUGH_EnableComm
+ @date Sun Jan 23 12:46:23 2000
+ @author Gabrielle Allen
+ @desc
+ This sets the docomm[2*dim] array of the GA based
+ on the setting of the comm flag and allocates the comm buffers.
+ @enddesc
+ @history
+ @date Mon Jun 05 2000 @author Thomas Radke
+ Moved buffer allocation from PUGH_EnableGArrayDataStorage
+ @endhistory
+@@*/
+static int PUGH_EnableComm(pGH *pughGH,
+ pComm *comm,
+ int commflag)
{
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- pGH *pughGH; /* PUGH extension reference */
- int var, level; /* loop variables */
- int first_var; /* global index of first variable in group */
- int rc; /* return code */
+ int retval; /* return value */
+#ifdef CCTK_MPI
+ pGA *GA; /* GA structure the pComm belongs to */
+ int i, idir; /* loopers */
+ int dir; /* direction */
+ int sz; /* buffer size */
+#endif
- group = CCTK_GroupIndex(groupname);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ retval = 1;
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
+#ifdef CCTK_MPI
+ /* check whether this comm buffer is already set up */
+ if (comm->commflag == commflag)
+ {
+ return (retval);
+ }
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ /* get the GA structure the comm structure belongs to
+ For a group comm structure this GA is the first one within the group. */
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
- /* branch to "disable comm" function according to group type */
- switch (pgroup.grouptype)
+ if (comm->commflag == PUGH_NOCOMM)
{
- case GROUP_SCALAR :
- rc = 1;
- break;
-
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- for (level = 0; level < pgroup.numtimelevels; level++)
+
+#ifdef DEBUG_PUGH
+ printf (" PUGH_EnableComm: allocating comm buffer for %d vars starting "
+ "with %d '%s'\n", comm->n_vars, comm->first_var, GA->name);
+ fflush (stdout);
+#endif
+
+ /* allocate memory for communication buffers: 2 faces per direction */
+ for (i = 0; i < 2 * GA->extras->dim; i++)
+ {
+ if (GA->connectivity->neighbours[pughGH->myproc][i] >= 0)
+ {
+ dir = i/2;
+
+ /* no ghostzones -> no comm buffers */
+ sz = comm->n_vars * GA->extras->nghostzones[dir];
+ if (sz > 0)
{
- PUGH_GAComm ((pGA *)(pughGH->variables[var][level]), PUGH_NOCOMM);
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (idir != dir)
+ {
+ sz *= GA->extras->lnsize[idir];
+ }
+ }
+ comm->buffer_sz[i] = sz;
+ comm->send_buffer[i] = malloc(sz * GA->varsize);
+ comm->recv_buffer[i] = malloc(sz * GA->varsize);
+
+ if (! (comm->send_buffer[i] && comm->recv_buffer[i]))
+ {
+ for (; i >=0 ; i--)
+ {
+ if (comm->send_buffer[i])
+ {
+ free(comm->send_buffer[i]);
+ }
+ if (comm->recv_buffer[i])
+ {
+ free(comm->recv_buffer[i]);
+ }
+ comm->buffer_sz[i] = 0;
+ }
+
+ CCTK_WARN(1, "Out of memory !");
+ retval = -1;
+ break;
+ }
}
- rc = 1;
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupComm");
- rc = 0;
- break;
+ else
+ {
+ comm->buffer_sz[i] = 0;
+ comm->send_buffer[i] = NULL;
+ comm->recv_buffer[i] = NULL;
+ }
+ }
+ else
+ {
+ comm->buffer_sz[i] = 0;
+ comm->send_buffer[i] = NULL;
+ comm->recv_buffer[i] = NULL;
+ }
+ }
}
- return (rc);
-}
+ /* set up comm flags for each face */
+ if (retval >= 0)
+ {
+ /* Copy commflag */
+ comm->commflag = commflag;
+ /* First set all communcation off */
+ for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+ comm->docomm[idir] = 0;
-int PUGH_Barrier(cGH *GH)
-{
-#ifdef CCTK_MPI
- pGH *pughGH;
+ if (commflag == PUGH_ALLCOMM)
+ {
+ for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+ {
+ comm->docomm[idir] = 1;
+ }
+ }
+ else if (commflag == PUGH_PLUSFACESCOMM)
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ comm->docomm[2*idir+1] = 1;
+ }
+ }
+ else if (commflag == PUGH_MINUSFACESCOMM)
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ comm->docomm[2*idir] = 1;
+ }
+ }
+ else
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (commflag == PUGH_COMM(idir))
+ {
+ comm->docomm[2*idir] = 1;
+ comm->docomm[2*idir+1] = 1;
+ }
+ }
+ }
- pughGH = PUGH_pGH(GH);
+ /* FIXME Add back the check that you have a valid COMM model: Gab */
- CACTUS_MPI_ERROR (MPI_Barrier (pughGH->PUGH_COMM_WORLD));
-#endif
+ /* Handle nsize = 1 type cases. This is only important for one
+ processor MPI periodic boundaries */
- return 0;
-}
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (GA->extras->nsize[idir] == 1)
+ {
+ comm->docomm[2*idir] = 0;
+ comm->docomm[2*idir+1] = 0;
+ }
+ }
+ }
+#endif /* CCTK_MPI */
-/*
-int PUGH_Reduce(cGH *GH,
- const char *operation,
- int n_infields,
- int n_outfields,
- int out_type,
- void **outarray,
- ...)
-{
- return 0;
+ return retval;
}
-*/
-int PUGH_Interp(cGH *GH,
- const char *operation,
- int n_coords,
- int n_infields,
- int n_outfields,
- int n_points,
- int type,
- ...)
+ /*@@
+ @routine PUGH_DisableComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ This frees the communication buffers
+ of a given comm structure.
+ @enddesc
+ @history
+ Separated from routine PUGH_DisableGArrayDataStorage()
+ @endhistory
+@@*/
+static int PUGH_DisableComm(pGH *pughGH,
+ pComm *comm)
{
- return 0;
-}
+#ifdef CCTK_MPI
+ int i; /* looper */
+ pGA *GA; /* GA structure the comm structure belongs to */
-int PUGH_ParallelInit(cGH *GH)
-{
- return 0;
-}
+ /* get the GA to which the comm structure belongs to
+ For a comm structure this is the first variable within this group. */
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
-int PUGH_Exit(int retval, cGH *GH)
-{
-#ifdef CCTK_MPI
- CACTUS_MPI_ERROR(MPI_Finalize());
+#ifdef DEBUG_PUGH
+ printf (" PUGH_DisableComm: freeing comm buffer for group of %d vars and "
+ "first var '%s'\n", comm->n_vars, GA->name);
+ fflush (stdout);
#endif
- exit(retval);
- return(retval);
-}
-
-int PUGH_Abort(cGH *GH)
-{
- assert(0);
- exit(0);
- return(0);
-}
-
-
-/*****************************************************************************/
-/* local functions */
-/*****************************************************************************/
-
+ /* free memory for communication buffers: 2 faces per direction */
+ for (i = 0; i < 2 * GA->extras->dim; i++)
+ {
+ if (comm->send_buffer[i])
+ {
+ if (comm->sreq[i] != MPI_REQUEST_NULL)
+ {
+ CACTUS_MPI_ERROR(MPI_Request_free(&comm->sreq[i]));
+ }
+ free(comm->send_buffer[i]);
+ comm->send_buffer[i] = NULL;
+ }
-int PUGH_SyncGroupGA (cGH *GH,
- int group,
- int vtype,
- int n_vars,
- int timelevel)
-{
+ if (comm->recv_buffer[i])
+ {
+ free(comm->recv_buffer[i]);
+ comm->recv_buffer[i] = NULL;
+ }
- pGH *pughGH;
+ comm->buffer_sz[i] = 0;
+ }
- pughGH = PUGH_pGH(GH);
+ comm->commflag = PUGH_NOCOMM;
+#endif /* CCTK_MPI */
- return PUGH_SyncGAs(pughGH,
- CCTK_GroupDimI(group),
- CCTK_FirstVarIndexI(group),
- n_vars,
- timelevel);
+ return (1);
}
-int PUGH_SyncGAs(pGH *pughGH,
- int dim,
- int first_var,
- int n_vars,
- int timelevel)
+ /*@@
+ @routine PUGH_Sync
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Finally synchronizes a variable or group of variables
+ according to a given comm structure.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_Sync(pGH *pughGH,
+ pComm *comm)
{
#ifdef CCTK_MPI
- int Dir;
+ int dir;
pGA *GA;
-#ifdef COMM_TIMING
- double t1,t2;
-#endif
- MPI_Request *sr = NULL;
MPI_Status mss;
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ int i;
+ MPI_Request *sr;
+#endif
+#ifdef COMM_TIMING
+ double t1, t2;
#endif
- /* start the timer for communication time */
+
#if 0
- CactusStartTimer (&pughGH->comm_time);
+ /* start the timer for communication time */
+ CCTK_TimerStartI(pughGH->comm_time);
#endif
-#ifdef CCTK_MPI
+#ifdef PUGH_WITH_DERIVED_DATATYPES
if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
- int i;
/* 2 faces, send and receive is the 2 * 2 */
- sr = (MPI_Request *)malloc(n_vars * 2 * 2 * sizeof(MPI_Request));
- for (i=0;i<n_vars * 2 * 2; i++)
- sr[i] = MPI_REQUEST_NULL;
+ sr = (MPI_Request *) malloc(comm->n_vars * 2 * 2 * sizeof(MPI_Request));
}
+#endif
+
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
+
+#ifdef DEBUG_PUGH
+ printf (" PUGH_Sync: syncing group of %d vars with first var '%s'\n",
+ comm->n_vars, GA->name);
+ fflush (stdout);
+#endif
- for (Dir = 0; Dir < dim; Dir ++)
+ for (dir = 0; dir < GA->extras->dim; dir ++)
{
- int i;
#ifdef COMM_TIMING
t1 = MPI_Wtime();
#endif
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- PostReceiveGA(pughGH, GA, 2*Dir);
- PostReceiveGA(pughGH, GA, 2*Dir+1);
- }
+ PostReceiveGA(pughGH, 2*dir, comm);
+ PostReceiveGA(pughGH, 2*dir+1, comm);
#ifdef COMM_TIMING
t2 = MPI_Wtime();
- printf ("PR : %lf\n",t2-t1);
+ printf("PR : %f\n",t2-t1);
#endif
- if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
+
+ /* For allocated buffers: wait for the last send.
+ Since these will be null if they
+ are not used, this is always safe.
+ */
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- /* Wait for the last send. Since these will be null if they
- are not used, this is always safe.
- */
- MPI_Wait(&(GA->sreq[2*Dir]),&mss);
- PostSendGA(pughGH, GA, 2*Dir);
- MPI_Wait(&(GA->sreq[2*Dir+1]),&mss);
- PostSendGA(pughGH, GA, 2*Dir+1);
- }
- }
- else
+ MPI_Wait(&comm->sreq[2*dir], &mss);
+ }
+ PostSendGA(pughGH, 2*dir, comm);
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- PostSendGA(pughGH, GA, 2*Dir);
- PostSendGA(pughGH, GA, 2*Dir+1);
- }
+ MPI_Wait(&comm->sreq[2*dir+1], &mss);
}
+ PostSendGA(pughGH, 2*dir+1, comm);
#ifdef COMM_TIMING
t1 = MPI_Wtime();
- printf ("PS : %lf\n",t1-t2);
+ printf("PS : %f\n",t1-t2);
#endif
/* Now comes the big difference between derived types and
@@ -878,40 +709,37 @@ int PUGH_SyncGAs(pGH *pughGH,
if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
/* Do a wait any on the receives */
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- MPI_Wait(&(GA->rreq[2*Dir]),&mss);
- FinishReceiveGA(pughGH, GA, 2*Dir);
- MPI_Wait(&(GA->rreq[2*Dir+1]),&mss);
- FinishReceiveGA(pughGH, GA, 2*Dir+1);
- }
- }
-
- if (pughGH->commmodel == PUGH_DERIVEDTYPES)
+ MPI_Wait(&comm->rreq[2*dir], &mss);
+ FinishReceiveGA(pughGH, 2*dir, comm);
+ MPI_Wait(&comm->rreq[2*dir+1], &mss);
+ FinishReceiveGA(pughGH, 2*dir+1, comm);
+ }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
/* Load up the thing for the waitall */
- for (i=0;i<n_vars;i++)
+ for (i = 0; i < comm->n_vars; i++)
{
- int id = i*4;
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- if (GA->docomm[2*Dir] &&
+ int id = i * 2 * 2;
+ pGA *GA = (pGA *) pughGH->variables [i][comm->sync_timelevel];
+
+ if (GA->comm->docomm[2*dir] &&
GA->storage)
{
- sr[id] = GA->sreq[2*Dir];
- sr[id+1] = GA->rreq[2*Dir];
+ sr[id] = GA->comm->sreq[2*dir];
+ sr[id+1] = GA->comm->rreq[2*dir];
}
else
{
sr[id] = MPI_REQUEST_NULL;
sr[id+1] = MPI_REQUEST_NULL;
}
-
- if (GA->docomm[2*Dir+1] &&
+
+ if (GA->comm->docomm[2*dir+1] &&
GA->storage)
{
- sr[id+2] = GA->sreq[2*Dir+1];
- sr[id+3] = GA->rreq[2*Dir+1];
+ sr[id+2] = GA->comm->sreq[2*dir+1];
+ sr[id+3] = GA->comm->rreq[2*dir+1];
}
else
{
@@ -920,144 +748,26 @@ int PUGH_SyncGAs(pGH *pughGH,
}
}
/* Now do a waitall */
- MPI_Waitall(4*n_vars,sr,&mss);
-
+ MPI_Waitall(4*comm->n_vars, sr, &mss);
}
+#endif
- if (sr)
- free(sr);
#ifdef COMM_TIMING
t2 = MPI_Wtime();
- printf ("FR : %lf\n",t2-t1);
+ printf("FR : %f\n",t2-t1);
#endif
}
-#endif /* MPI */
-
- /* get the time spent in communication */
-
-#if 0
- CactusStopTimer (&pughGH->comm_time);
-#endif
-
- return (1);
-}
-
-
-int PUGH_EnableArrayGroupStorage(cGH *GH,
- int group,
- int vtype,
- int dim,
- int n_variables,
- int n_timelevels)
-{
- int nstorage; /* Number of Arrays for which storage was set */
- int nnostorage; /* Number of Arrays for which no storage was set */
- int retval;
- int first_var;
- int var;
- pGH *mypGH;
- pGA *GA;
- int level;
- /* FIXME: This should come from a parameter */
- int zero_memory=1;
- int padding_active=0;
- int padding_cacheline_bits=0;
- int padding_size=0;
- int padding_address_spacing=0;
-
- nstorage = 0;
- nnostorage = 0;
-
- mypGH = PUGH_pGH(GH);
- first_var = CCTK_FirstVarIndexI(group);
-
- for(var = first_var; var < first_var+n_variables; var++)
- {
- for(level = 0; level < n_timelevels; level++)
- {
- GA = (pGA *)(mypGH->variables[var][level]);
-
- if(!GA->storage)
- {
-#ifdef DEBUG_PUGH
- {
- const char *name;
- name = CCTK_FullName(var);
- printf("Storage on in PUGH_EnableArrayGroupStorage for %s\n",name);
- }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ free(sr);
#endif
- PUGH_EnablePGAStorage(GA,
- mypGH->myproc,
- zero_memory,
- padding_active,
- padding_cacheline_bits,
- padding_size,
- padding_address_spacing);
-
- GH->data[var][level] = GA->data;
-
- nnostorage++;
- }
- else
- {
- GH->data[var][level] = GA->data;
- nstorage++;
- }
- }
- }
- if (nstorage > 0 && nnostorage > 0)
- {
- CCTK_WARN(0,"Group storage violation in PUGH_EnableArrayGroupStorage");
- retval = -1;
- }
- else if (nstorage > 0)
- {
- retval = 1;
- }
- else if (nnostorage > 0)
- {
- retval = 0;
- }
- else
- {
- CCTK_WARN(0,"Problem in PUGH_EnableArrayGroupStorage");
- retval = -1;
- }
-
- return retval;
-
-}
-
-
-int PUGH_MyProc(cGH *GH)
-{
- int myproc;
-
-#ifdef CCTK_MPI
- pGH *pughGH;
- pughGH = PUGH_pGH(GH);
- CACTUS_MPI_ERROR(MPI_Comm_rank(pughGH->PUGH_COMM_WORLD, &myproc));
-#else
- myproc=0;
+#if 0
+ /* get the time spent in communication */
+ CCTK_TimerStopI(pughGH->comm_time);
#endif
- return myproc;
-}
-
-
-int PUGH_nProcs(cGH *GH)
-{
- int nprocs;
-
-#ifdef CCTK_MPI
- pGH *pughGH;
- pughGH = PUGH_pGH(GH);
- CACTUS_MPI_ERROR(MPI_Comm_size(pughGH->PUGH_COMM_WORLD, &nprocs));
-#else
- nprocs=1;
-#endif
+#endif /* CCTK_MPI */
- return nprocs;
+ return (0);
}
diff --git a/src/Evolve.c b/src/Evolve.c
index 3a1e9d9..e5b100b 100644
--- a/src/Evolve.c
+++ b/src/Evolve.c
@@ -49,8 +49,6 @@ static int DoneMainLoop (CCTK_REAL cctk_time, int iteration);
static int StepGH(cGH *GH);
static void RotateTimeLevelsGH(cGH *cgh);
-extern int pugh_GHExtension;
-
/* the iteration counter used in the evolution loop */
static int iteration = 0;
diff --git a/src/FinishReceiveGA.c b/src/FinishReceiveGA.c
index d1689da..df4154c 100644
--- a/src/FinishReceiveGA.c
+++ b/src/FinishReceiveGA.c
@@ -3,9 +3,8 @@
@date Wed Feb 2 11:37:28 1997
@author Gabrielle Allen
@desc
- The routine which finalize the MPI recieves for a grid
- function. Critically linked with @seefile pGA_PostRecv.c
- and @seefile pGA_PostSend.c
+ The routine which finalize the MPI recieves for a communication.
+ Critically linked with @seefile PostRecvGA.c and @seefile PostSendGA.c
@enddesc
@version $Header$
@@*/
@@ -17,221 +16,129 @@
#include "cctk.h"
#include "pugh.h"
-void FinishReceiveGA3(pGH *GH, pGA *GA, int dir);
-void FinishReceiveGA1(pGH *GH, pGA *GA, int dir);
-
static char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_FinishReceiveGA_c)
+
+#ifdef CCTK_MPI
+
/*@@
@routine FinishReceiveGA
@date Thu Apr 3 11:38:07 1997
@author Paul Walker
@desc
This routine finalizes the MPI communication through a face.
- It is crucially linked with @seeroutine pGA_PostRecv and
- @seeroutine pGA_PostSend.
+ It is crucially linked with @seeroutine PostRecvGA and
+ @seeroutine PostSendGA.
<p>
<b>Important!</b>
This routine does <b>not</b> wait on the recieves.
Before it is called, you must do the MPI_Wait or
else you will not get the right answer. for an
- example of this, see @seeroutine SyncGroupGA or
- @seeroutine SyncGroupGA
+ example of this, see @seeroutine PUGH_DisableArrayGroupComm or
+ @seeroutine PUGH_Sync
@enddesc
- @calledby SyncGroupGA
+ @calledby PUGH_Sync
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GAs with storage
@endhistory
@@*/
-void FinishReceiveGA(pGH *GH, pGA *GA, int dir)
+void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
-#ifdef CCTK_MPI
+ pGA *GA;
+ int i, dim, copy_bytes, offset;
+ char *copy_from, *copy_to;
+ int *istart, *iend, *iterator;
+
+
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
/* Return if GA has no storage */
- if (!(GA->storage))
+ if (! GA->storage)
{
return;
}
/* Return if communication not required and no forced synchronisation */
- if (!(GH->forceSync || GA->docomm[dir]))
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] >= 0)
+ if (GA->connectivity->neighbours[pughGH->myproc][dir] < 0)
{
+ return;
+ }
- /* Here wait one at a time, so the others can arrive while
- we copy the data back in... */
+ /* Here wait one at a time, so the others can arrive while
+ we copy the data back in... */
#ifdef DEBUG_PUGH
- printf ("FINISHRECV: GA %s Side %d Proc %d request %d\n",
- GA->name,dir,GH->myproc,GA->rreq[dir]);
+ printf ("FinishReceiveGA: into direction %d from proc %d for %d vars "
+ "starting with '%s'\n",
+ dir, GA->connectivity->neighbours[pughGH->myproc][dir],
+ comm->n_vars, GA->name);
#endif
- if (GA->extras->dim == 3)
- {
- FinishReceiveGA3(GH,GA,dir);
- }
- else if (GA->extras->dim == 1)
- {
- FinishReceiveGA1(GH,GA,dir);
- }
- else
- {
- CCTK_WARN(0,"PUGH does not support this dimension grid");
- }
+ istart = GA->extras->ghosts[GA->stagger][0][dir];
+ iend = GA->extras->ghosts[GA->stagger][1][dir];
+ iterator = GA->extras->iterator;
+ /* set iterator to the start vector */
+ for(dim = 0; dim < GA->extras->dim; dim++)
+ {
+ iterator[dim] = istart[dim];
}
-#endif
-}
+ /* get the source and the number of bytes to copy in the lowest dimension */
+ copy_from = comm->recv_buffer[dir];
+ copy_bytes = (iend[0] - istart[0]) * GA->varsize;
-void FinishReceiveGA3(pGH *GH,pGA *GA,int dir)
-{
- int istart,iend,jstart,jend,kstart,kend;
- int ii,jj,kk,xx;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy buffer onto GA Storage */
- istart = GA->extras->ghosts[GA->stagger][0][dir][0];
- iend = GA->extras->ghosts[GA->stagger][1][dir][0];
- jstart = GA->extras->ghosts[GA->stagger][0][dir][1];
- jend = GA->extras->ghosts[GA->stagger][1][dir][1];
- kstart = GA->extras->ghosts[GA->stagger][0][dir][2];
- kend = GA->extras->ghosts[GA->stagger][1][dir][2];
-
- /* Great now copy. Note ordering is just link in PostSend */
- xx=0;
- switch (GA->vtype)
+ /* now do the nested loops starting with the innermost */
+ dim = 1;
+ while (1)
{
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- char_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_CHAR *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->data;
- for (kk=kstart; kk<kend; kk++)
+ /* check for end of current loop */
+ if (GA->extras->dim > 1 && iterator[dim] >= iend[dim])
+ {
+ /* increment outermost loopers */
+ for (dim++; dim < GA->extras->dim; dim++)
{
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- int_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_INT *) GA->recv_buffer[dir]) [xx++];
- }
- }
+ iterator[dim]++;
+ if (iterator[dim] < iend[dim])
+ break;
}
- break;
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- real_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_REAL *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_COMPLEX *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
+ /* done if beyond outermost loop */
+ if (dim >= GA->extras->dim)
+ break;
- default:
- CCTK_WARN (1, "Unsupported variable type in FinishReceiveGA3");
- return;
- }
-}
+ /* reset innermost loopers */
+ for (dim--; dim > 0; dim--)
+ iterator[dim] = istart[dim];
+ dim = 1;
+ }
+ /* get the linear offset */
+ offset = istart[0] * GA->varsize;
+ for (i = 1; i < GA->extras->dim; i++)
+ offset += iterator[i] * GA->extras->hyper_volume[i];
+ /* copy the data in dimension 0 */
+ for (i = comm->first_var; i < comm->first_var + comm->n_vars; i++)
+ {
+ copy_to = ((pGA *) pughGH->variables[i][comm->sync_timelevel])->data;
+ memcpy (copy_to + offset, copy_from, copy_bytes);
+ copy_from += copy_bytes;
+ }
-void FinishReceiveGA1(pGH *GH,pGA *GA,int dir)
-{
- int istart,iend;
- int ii,xx;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy buffer onto GA Storage */
- istart = GA->extras->ghosts[GA->stagger][0][dir][0];
- iend = GA->extras->ghosts[GA->stagger][1][dir][0];
-
- /* Great now copy. Note ordering is just link in PostSend */
- xx=0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- char_data [ii] = ((CCTK_CHAR *)GA->recv_buffer[dir])[xx++];
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- int_data [ii] = ((CCTK_INT *)GA->recv_buffer[dir])[xx++];
- }
- break;
+ /* increment current looper */
+ iterator[dim]++;
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- real_data [ii] = ((CCTK_REAL *)GA->recv_buffer[dir])[xx++];
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [ii] = ((CCTK_COMPLEX *)GA->recv_buffer[dir])[xx++];
- }
- break;
+ } /* end of nested loops over all dimensions */
- default:
- CCTK_WARN (1, "Unsupported variable type in FinishReceiveGA3");
- return;
- }
}
+#endif /* CCTK_MPI */
diff --git a/src/GHExtension.c b/src/GHExtension.c
index 7d9bfcc..23627ff 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -80,13 +80,8 @@ void *PUGH_SetupGH(tFleshConfig *config,
CCTK_WARN(0,"Failed to allocate memory for a pGH");
}
- /* Set the identity for this pGH. */
-
- /* FIXME : should be a dynamically allocated string.
- * If you change this, please change the line in pugh.h.
- */
-
- strcpy(newGH->identity_string, "");
+ /* Set the identity for this pGH. For now this is an empty string. */
+ newGH->identity_string = (char *) calloc (1, sizeof (char *));
/* Set up groups on the GH */
@@ -186,7 +181,7 @@ static void PUGH_InitGHBasics (cGH *GH)
pGH *mypGH;
pGExtras *GFExtras;
- mypGH = (pGH *)GH->extensions[pugh_GHExtension];
+ mypGH = PUGH_pGH(GH);
GFExtras = mypGH->GFExtras[GH->cctk_dim-1];
GH->cctk_convlevel = 0;
@@ -207,7 +202,7 @@ static void PUGH_InitGHBasics (cGH *GH)
{
GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)] = GH->cctk_lsh[idir];
if ((GH->cctk_bbox[2*idir+1]==1) && (istag>0))
- GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)]--;
+ GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)]--;
}
}
}
@@ -236,7 +231,7 @@ static void PUGH_InitGHVariables (cGH *GH)
pGH *mypGH;
- mypGH = (pGH *)GH->extensions[pugh_GHExtension];
+ mypGH = PUGH_pGH(GH);
for(var = 0; var < mypGH->nvariables; var++)
{
diff --git a/src/LoadAware.c b/src/LoadAware.c
index 94c2d17..4a5ce80 100644
--- a/src/LoadAware.c
+++ b/src/LoadAware.c
@@ -9,6 +9,7 @@
@@*/
#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <ctype.h>
#include "pugh.h"
diff --git a/src/PostReceiveGA.c b/src/PostReceiveGA.c
index 3a7864b..caacff0 100644
--- a/src/PostReceiveGA.c
+++ b/src/PostReceiveGA.c
@@ -3,9 +3,9 @@
@date Wed Feb 2 11:28:06 1997
@author Gabrielle Allen
@desc
- Routines which post all the IRecv commands for a GA. These
- allow the asyncronous model proposed in, for example,
- @seeroutine SyncGA to go!
+ Routines which post all the IRecv commands for a single sync.
+ These allow the asyncronous model proposed in, for example,
+ @seeroutine PUGH_Sync to go!
@enddesc
@version $Header$
@@*/
@@ -21,6 +21,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
/*#define DEBUG_PUGH*/
+#ifdef CCTK_MPI
+
/*@@
@routine PostReceiveGA
@date Thu Apr 3 12:10:46 1997
@@ -28,8 +30,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
@desc
This routine posts a recieve (MPI_Irecv) of the buffer
we want to get the send from another processor, which
- will be sent by @seeroutine pGA_PostSend and then
- finalized by @seeroutoine pGA_FinishRecv.
+ will be sent by @seeroutine PostSendGA and then
+ finalized by @seeroutoine FinishRecvGA.
<p>
Aside from a silly calculation to get a unique tag
based on neigbors and processors, this is a very
@@ -41,83 +43,95 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
@endhistory
@@*/
-void PostReceiveGA(pGH *GH, pGA *GA, int dir)
+void PostReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
-#ifdef CCTK_MPI
+ pGA *GA;
int rtag;
- MPI_Datatype mpi_type;
- MPI_Datatype *recv_dt;
+ int neighbour;
- if (!(GA->storage))
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
+
+ /* return if no storage assigned */
+ if (! GA->storage)
{
CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
"Trying to synchronize variable '%s' with no storage", GA->name);
return;
}
- /* Return if communication not required and no forced synchronisation */
- if (!(GH->forceSync || GA->docomm[dir]))
+ /* return if communication not required and no forced synchronisation */
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] < 0)
+ /* return if there is no neighbour in the given direction */
+ neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
+ if (neighbour < 0)
{
return;
}
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- mpi_type = PUGH_MPI_CHAR;
- recv_dt = GH->recv_char_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_INT:
- mpi_type = PUGH_MPI_INT;
- recv_dt = GH->recv_int_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_REAL:
- mpi_type = PUGH_MPI_REAL;
- recv_dt = GH->recv_real_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- mpi_type = GH->PUGH_mpi_complex;
- recv_dt = GH->recv_complex_dt [GA->stagger];
- break;
-
- default:
- CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
- return;
- }
-
- rtag = 1000 + dir + 2 * (GH->myproc + GH->nprocs * GA->id);
+ /* note this is the complement of the stag set in PostSendGA */
+ rtag = 1000 + dir + 2 * (pughGH->myproc + pughGH->nprocs * GA->id);
/* mod the tag to force MPI compliance */
rtag = rtag % 32768;
+
#ifdef DEBUG_PUGH
- printf ("RECV GA %s Side %d Proc %d rtag %d size %d from proc %d\n",
- GA->name,
- dir,
- GH->myproc,
- rtag,
- GA->buffer_sz[dir],
- GA->connectivity->neighbours[GH->myproc][dir]);
+ printf ("PostReceiveGA: into direction %d from proc %d with rtag %d size %d "
+ "for %d vars starting from '%s'\n",
+ dir, neighbour, rtag, comm->buffer_sz[dir], comm->n_vars, GA->name);
#endif
- if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
+
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- CACTUS_MPI_ERROR (MPI_Irecv (GA->recv_buffer [dir],
- GA->buffer_sz [dir], mpi_type,
- GA->connectivity->neighbours [GH->myproc][dir], rtag,
- GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+ CACTUS_MPI_ERROR (MPI_Irecv (comm->recv_buffer[dir],
+ comm->buffer_sz[dir], comm->mpi_type,
+ neighbour, rtag,
+ pughGH->PUGH_COMM_WORLD, &comm->rreq[dir]));
}
- if (GH->commmodel == PUGH_DERIVEDTYPES)
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
- CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
- 1, recv_dt [dir],
- GA->connectivity->neighbours [GH->myproc][dir], rtag,
- GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+ int var;
+ MPI_Datatype *recv_dt;
+
+
+ switch (GA->vtype)
+ {
+ case CCTK_VARIABLE_CHAR:
+ recv_dt = pughGH->recv_char_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_INT:
+ recv_dt = pughGH->recv_int_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_REAL:
+ recv_dt = pughGH->recv_real_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_COMPLEX:
+ recv_dt = pughGH->recv_complex_dt[GA->stagger];
+ break;
+
+ default:
+ CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
+ return;
+ }
+
+ for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
+ {
+ pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
+
+
+ CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
+ 1, recv_dt[dir],
+ neighbour, (rtag + var) % 32768,
+ pughGH->PUGH_COMM_WORLD,
+ &GA->comm->rreq[dir]));
+ }
}
#endif
}
+#endif /* CCTK_MPI */
diff --git a/src/PostSendGA.c b/src/PostSendGA.c
index 1ef6640..ccaff91 100644
--- a/src/PostSendGA.c
+++ b/src/PostSendGA.c
@@ -20,11 +20,12 @@ static char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_PostSendGA_c)
-void PostSendGA1(pGH *GH, pGA *GA, int dir);
-void PostSendGA3(pGH *GH, pGA *GA, int dir);
-
/*#define DEBUG_PUGH*/
+
+#ifdef CCTK_MPI
+
+
/*@@
@routine PostSendGA
@date Thu Apr 3 11:58:59 1997
@@ -32,277 +33,183 @@ void PostSendGA3(pGH *GH, pGA *GA, int dir);
@desc
This routine posts an asyncronous MPI send of a buffer
for a face (MPI_Isend). It does this by first packing up
- a send buffer (allocated in @seeroutine SetupPGF) then
+ a send buffer (allocated in @seeroutine PUGH_SetupGroupComm) then
sending it out on the pipe.
<p>
Since this is an asynchronous communications model we
assume that a recieve has been posted for the send.
thus the correct calling order for this is as in
- @seeroutine SyncGF, eg, after a Recv.
+ @seeroutine PUGH_Sync, eg, after a Recv.
<p>
Note this does <b>not</b> wait on the send buffer from previous
communications. It is the users responsibility to wait on
that buffer.
@enddesc
- @calledby SyncGroupGF
+ @calledby PUGH_Sync
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GFs with storage
@endhistory
@@*/
-void PostSendGA(pGH *GH, pGA *GA, int dir)
+void PostSendGA(pGH *pughGH, int dir, pComm *comm)
{
-
-#ifdef CCTK_MPI
+ pGA *GA;
int stag, dircomp;
- double ts, tw, tp;
- MPI_Datatype mpi_type;
- MPI_Datatype *send_dt;
+ int neighbour;
+ int i, dim, copy_bytes, offset;
+ char *copy_from, *copy_to;
+ int *istart, *iend, *iterator;
- if (!(GA->storage))
- {
- return;
- }
-
- if (!(GH->forceSync || GA->docomm[dir]))
+
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
+
+ /* return if no storage assigned */
+ if (! GA->storage)
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] < 0)
+ /* return if communication not required and no forced synchronisation */
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- switch (GA->vtype)
+ /* return if there is no neighbour in the given direction */
+ neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
+ if (neighbour < 0)
{
-
- case CCTK_VARIABLE_CHAR:
- mpi_type = PUGH_MPI_CHAR;
- send_dt = GH->send_char_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_INT:
- mpi_type = PUGH_MPI_INT;
- send_dt = GH->send_int_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_REAL:
- mpi_type = PUGH_MPI_REAL;
- send_dt = GH->send_real_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- mpi_type = GH->PUGH_mpi_complex;
- send_dt = GH->send_complex_dt [GA->stagger];
- break;
-
- default:
- CCTK_WARN (1, "Unknown variable type in PostSendGA");
- return;
-
+ return;
}
- ts = MPI_Wtime();
dircomp = dir+1; /* Complementary direction */
- if (dircomp %2 == 0)
+ if (dircomp % 2 == 0)
{
dircomp = dir-1;
}
- /* Note this is the complement of the rtag set in PostReceive */
- stag = 1000 + dircomp + 2 *
- (GA->connectivity->neighbours[GH->myproc][dir] +
- GH->nprocs * GA->id);
-
+ /* note this is the complement of the rtag set in PostReceiveGA */
+ stag = 1000 + dircomp + 2 * (neighbour + pughGH->nprocs * GA->id);
/* mod the tag to force MPI compliance */
stag = stag % 32768;
#ifdef DEBUG_PUGH
- printf ("SEND: GA %s Side %d Proc %d stag %d Size %d to %d\n",
- GA->name,
- dir,
- GH->myproc,
- stag,
- GA->buffer_sz[dir],
- GA->connectivity->neighbours[GH->myproc][dir]);
+ printf ("PostSendGA: into direction %d to proc %d with stag %d size %d for "
+ "%d vars staring with '%s'\n",
+ dir, neighbour, stag, comm->buffer_sz[dir], comm->n_vars, GA->name);
#endif
- if (GH->commmodel == PUGH_DERIVEDTYPES)
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- CACTUS_MPI_ERROR (MPI_Isend (GA->data,
- 1, send_dt [dir],
- GA->connectivity->neighbours [GH->myproc][dir], stag,
- GH->PUGH_COMM_WORLD, &(GA->sreq [dir])));
- }
- if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
- {
- if (GA->connectivity->dim == 3)
- {
- PostSendGA3(GH,GA,dir);
- }
- else if (GA->connectivity->dim == 1)
- {
- PostSendGA1(GH,GA,dir);
- }
- else
+ istart = GA->extras->overlap[GA->stagger][0][dir];
+ iend = GA->extras->overlap[GA->stagger][1][dir];
+ iterator = GA->extras->iterator;
+
+ /* set iterator to the start vector */
+ for(dim = 0; dim < GA->extras->dim; dim++)
{
- CCTK_WARN(0,"PUGH does not yet support this dimension");
+ iterator[dim] = istart[dim];
}
- tp = MPI_Wtime();
-
- /* post send */
- CACTUS_MPI_ERROR (MPI_Isend (GA->send_buffer [dir],
- GA->buffer_sz [dir], mpi_type,
- GA->connectivity->neighbours [GH->myproc][dir], stag,
- GH->PUGH_COMM_WORLD, &(GA->sreq [dir])));
- tw = MPI_Wtime();
-#ifdef COMM_TIMING
- printf ("GA %s Dir %d Time %f\n", GA->name, dir, tw-ts);
-#endif
- }
-#endif
-}
+ /* get the source and the number of bytes to copy in the lowest dimension */
+ copy_to = comm->send_buffer[dir];
+ copy_bytes = (iend[0] - istart[0]) * GA->varsize;
-void PostSendGA3(pGH *GH, pGA *GA, int dir)
-{
-
- int ii,jj,kk,xx;
- int istart, iend;
- int jstart, jend;
- int kstart, kend;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy my information into the send buffer */
- istart = GA->extras->overlap[GA->stagger][0][dir][0];
- iend = GA->extras->overlap[GA->stagger][1][dir][0];
- jstart = GA->extras->overlap[GA->stagger][0][dir][1];
- jend = GA->extras->overlap[GA->stagger][1][dir][1];
- kstart = GA->extras->overlap[GA->stagger][0][dir][2];
- kend = GA->extras->overlap[GA->stagger][1][dir][2];
-
- /* Great now copy */
- xx = 0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- char_data [xx++] =
- ((CCTK_CHAR *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
+ /* now do the nested loops starting with the innermost */
+ dim = 1;
+ while (1)
+ {
+ /* check for end of current loop */
+ if (GA->extras->dim > 1 && iterator[dim] >= iend[dim])
{
- for (jj=jstart; jj<jend; jj++)
+ /* increment outermost loopers */
+ for (dim++; dim < GA->extras->dim; dim++)
{
- for (ii=istart; ii<iend; ii++)
- {
- int_data [xx++] =
- ((CCTK_INT *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- real_data [xx++] =
- ((CCTK_REAL *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
+ iterator[dim]++;
+ if (iterator[dim] < iend[dim])
+ break;
}
+
+ /* done if beyond outermost loop */
+ if (dim >= GA->extras->dim)
+ break;
+
+ /* reset innermost loopers */
+ for (dim--; dim > 0; dim--)
+ iterator[dim] = istart[dim];
+ dim = 1;
}
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
+
+ /* get the linear offset */
+ offset = istart[0] * GA->varsize;
+ for (i = 1; i < GA->extras->dim; i++)
+ offset += iterator[i] * GA->extras->hyper_volume[i];
+
+ /* copy the data in dimension 0 */
+ for (i = comm->first_var; i < comm->first_var + comm->n_vars; i++)
{
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [xx++] =
- ((CCTK_COMPLEX *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
+ copy_from = ((pGA *) pughGH->variables[i][comm->sync_timelevel])->data;
+ memcpy (copy_to, copy_from + offset, copy_bytes);
+ copy_to += copy_bytes;
}
- break;
+
+ /* increment current looper */
+ iterator[dim]++;
+
+ } /* end of nested loops over all dimensions */
+
+ /* now post send */
+ CACTUS_MPI_ERROR (MPI_Isend (comm->send_buffer[dir],
+ comm->buffer_sz[dir], comm->mpi_type,
+ neighbour, stag,
+ pughGH->PUGH_COMM_WORLD, &comm->sreq[dir]));
}
-}
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
+ {
+ int var;
+ MPI_Datatype *send_dt;
+ switch (GA->vtype)
+ {
-void PostSendGA1(pGH *GH, pGA *GA, int dir)
-{
+ case CCTK_VARIABLE_CHAR:
+ send_dt = pughGH->send_char_dt[GA->stagger];
+ break;
- int ii,xx;
- int istart, iend;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy my information into the send buffer */
- istart = GA->extras->overlap[GA->stagger][0][dir][0];
- iend = GA->extras->overlap[GA->stagger][1][dir][0];
-
- /* Great now copy */
- xx = 0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- char_data [xx++] = ((CCTK_CHAR *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- int_data [xx++] = ((CCTK_INT *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- real_data [xx++] = ((CCTK_REAL *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [xx++] = ((CCTK_COMPLEX *) GA->data) [ii];
- }
- break;
+ case CCTK_VARIABLE_INT:
+ send_dt = pughGH->send_int_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_REAL:
+ send_dt = pughGH->send_real_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_COMPLEX:
+ send_dt = pughGH->send_complex_dt[GA->stagger];
+ break;
+
+ default:
+ CCTK_WARN (1, "Unknown variable type in PostSendGA");
+ return;
+
+ }
+
+ for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
+ {
+ pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
+
+
+ CACTUS_MPI_ERROR (MPI_Isend (GA->data,
+ 1, send_dt[dir],
+ neighbour, (stag + var) % 32768,
+ pughGH->PUGH_COMM_WORLD,
+ &GA->comm->sreq[dir]));
+ }
}
+#endif
}
+#endif /* CCTK_MPI */
diff --git a/src/Reduction.c b/src/Reduction.c
index a9243ab..a91d469 100644
--- a/src/Reduction.c
+++ b/src/Reduction.c
@@ -37,9 +37,6 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_Reduction_c)
#define SQR(x) ((x) * (x))
-/* index into GH->extensions[] to get the PUGH extension handle */
-extern int pugh_GHExtension;
-
/* local function prototypes */
int PUGH_Norm1_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal);
int PUGH_Norm2_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal);
@@ -142,12 +139,10 @@ void PUGH_ReduceArrayMinVal (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayMinVal: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -265,12 +260,10 @@ void PUGH_ReduceArrayMaxVal (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayMaxVal: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array "
+ "with %d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -386,12 +379,10 @@ void PUGH_ReduceArraySum (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArraySum: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -511,12 +502,10 @@ void PUGH_ReduceArrayNorm1 (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayNorm1: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -636,12 +625,10 @@ void PUGH_ReduceArrayNorm2 (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayNorm2: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -725,12 +712,12 @@ void PUGH_ReduceArrayNorm2 (cGH *GH, int proc, int nDims, int dims [],
@endvar
@@*/
void PUGH_ReduceMinVal (cGH *GH,
- int proc,
- int nOutVals,
+ int proc,
+ int nOutVals,
int outType,
- void *outVals,
- int numinvars,
- int *varlist)
+ void *outVals,
+ int numinvars,
+ int *varlist)
{
int i;
@@ -750,11 +737,9 @@ void PUGH_ReduceMinVal (cGH *GH,
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceMinVal: don't know how to reduce a GF "
- "to an output array of %d elements", nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a GF to an output array "
+ "of %d elements", nOutVals);
return;
}
nPoints = 1;
@@ -884,12 +869,9 @@ void PUGH_ReduceMaxVal (cGH *GH, int proc, int nOutVals,
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceMaxVal: don't know how to reduce a GF "
- "to an output array of %d elements", nOutVals);
- CCTK_WARN (1, msg);
- return;
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a GF to an output array "
+ "of %d elements", nOutVals);
}
nPoints = 1;
}
@@ -2118,9 +2100,6 @@ int PUGH_Norm2_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal)
CCTK_INT np;
CCTK_REAL local_result;
#endif
- int dim;
-
- dim = GH->cctk_dim;
timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
if (timelevel > 0)
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index 812aae4..7bba580 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -36,13 +36,10 @@ CCTK_FILEVERSION (CactusPUGH_PUGH_SetupGroup_c)
@@*/
-int PUGH_SetupScalarGroup
- (
- pGH *newGH,
- int vtype,
- int n_variables,
- int n_timelevels
- )
+int PUGH_SetupScalarGroup(pGH *newGH,
+ int vtype,
+ int n_variables,
+ int n_timelevels)
{
int retval=PUGH_SUCCESS;
int variable;
@@ -67,7 +64,7 @@ int PUGH_SetupScalarGroup
for(level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
- (void *)malloc(var_size);
+ (void *)malloc(var_size);
}
newGH->nvariables++;
}
@@ -105,16 +102,14 @@ int PUGH_SetupScalarGroup
@@*/
int PUGH_SetupArrayGroup(pGH *newGH,
- int *nsize,
- int *ghostsize,
- int vtype,
- int dim,
- int n_variables,
- int staggercode,
- int n_timelevels)
+ int *nsize,
+ int *ghostsize,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggercode,
+ int n_timelevels)
{
-
- int i;
int variable;
int var_size = CCTK_VarTypeSize(vtype);
int retval = PUGH_SUCCESS;
@@ -122,34 +117,16 @@ int PUGH_SetupArrayGroup(pGH *newGH,
void ***temp;
pConnectivity *connectivity;
pGExtras *extras;
+ pComm *groupcomm;
int *perme;
int *nprocs;
- /* Arrays can't (yet) have periodicity */
- perme = (int *) malloc(dim*sizeof(int));
- if (perme)
- {
- for (i=0;i<dim;i++)
- {
- perme[i]=0;
- }
- }
- else
- {
- CCTK_WARN(0,"Memory error in PUGH");
- }
-
- /* Arrays can't (yet) have manual set up */
- nprocs = (int *) malloc(dim*sizeof(int));
- if (nprocs)
- {
- for (i=0;i<dim;i++)
- {
- nprocs[i]=0;
- }
- }
- else
+ /* Arrays can't (yet) have periodicity and manual setup,
+ so initialize them to zero */
+ perme = (int *) calloc(dim, sizeof(int));
+ nprocs = (int *) calloc(dim, sizeof(int));
+ if (! (perme && nprocs))
{
CCTK_WARN(0,"Memory error in PUGH");
}
@@ -168,6 +145,23 @@ int PUGH_SetupArrayGroup(pGH *newGH,
free(nprocs);
free(perme);
+ /* Set up the communication buffer used for all variables within this group.
+ Note: only with allocated buffers we can have group communication. */
+ if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
+ {
+ groupcomm = PUGH_SetupGArrayGroupComm(newGH,
+ dim,
+ newGH->nvariables,
+ n_variables,
+ n_timelevels - 1,
+ vtype,
+ extras);
+ }
+ else
+ {
+ groupcomm = NULL;
+ }
+
temp = (void ***)realloc
(newGH->variables,(newGH->nvariables+n_variables)*sizeof(void **));
@@ -185,14 +179,15 @@ int PUGH_SetupArrayGroup(pGH *newGH,
for(level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
- PUGH_SetupPGA(newGH,
- extras,
- connectivity,
- CCTK_VarName(newGH->nvariables),
- newGH->nvariables,
- var_size,
- vtype,
- staggercode);
+ PUGH_SetupGArray(newGH,
+ extras,
+ connectivity,
+ groupcomm,
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode);
}
newGH->nvariables++;
}
@@ -241,6 +236,15 @@ int PUGH_SetupGFGroup(pGH *newGH,
int retval=PUGH_SUCCESS;
int level;
void ***temp;
+ pComm *groupcomm;
+
+ groupcomm = PUGH_SetupGArrayGroupComm(newGH,
+ dim,
+ newGH->nvariables,
+ n_variables,
+ n_timelevels - 1,
+ vtype,
+ newGH->GFExtras[dim-1]);
temp = (void ***)realloc(newGH->variables,
(newGH->nvariables+n_variables)*sizeof(void **));
@@ -258,15 +262,16 @@ int PUGH_SetupGFGroup(pGH *newGH,
{
for(level = 0; level < n_timelevels; level++)
{
- newGH->variables[newGH->nvariables][level] =
- PUGH_SetupPGA(newGH,
- newGH->GFExtras[dim-1],
- newGH->Connectivity[dim-1],
- CCTK_VarName(newGH->nvariables),
- newGH->nvariables,
- var_size,
- vtype,
- staggercode);
+ newGH->variables[newGH->nvariables][level] =
+ PUGH_SetupGArray(newGH,
+ newGH->GFExtras[dim-1],
+ newGH->Connectivity[dim-1],
+ groupcomm,
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode);
}
newGH->nvariables++;
}
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index 441ed59..3d4b1b0 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -13,6 +13,11 @@
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
+#include <assert.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
#include "cctk.h"
#include "pugh.h"
@@ -46,7 +51,7 @@ pGH *PUGH_SetupPGH(void *callerid,
int *nsize,
int *nghostzones,
int staggertype,
- int *perme)
+ int *perme)
{
DECLARE_CCTK_PARAMETERS
@@ -77,28 +82,34 @@ pGH *PUGH_SetupPGH(void *callerid,
GH->Connectivity[idim-1]=
PUGH_SetupConnectivity(idim,
- GH->nprocs,
- nprocs,
- perme);
+ GH->nprocs,
+ nprocs,
+ perme);
free(nprocs);
GH->GFExtras[idim-1]=
PUGH_SetupPGExtras(idim,
- perme,
- staggertype,
- nsize,
- nghostzones,
- GH->nprocs,
- ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
- GH->myproc);
+ perme,
+ staggertype,
+ nsize,
+ nghostzones,
+ GH->nprocs,
+ ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
+ GH->myproc);
}
GH->active = 1;
GH->commmodel = PUGH_ALLOCATEDBUFFERS;
+
#if 0
- CactusResetTimer (&GH->comm_time);
+ /* create the timer for communication time */
+ if(comm_verbose)
+ {
+ GH->comm_time = CCTK_TimerCreateI ();
+ }
#endif
+
/* set staggering flag */
/*GH->stagger = staggertype; */
GH->level = 0;
@@ -201,7 +212,8 @@ void PUGH_DestroyPGH(pGH **GHin)
{
#ifdef DEBUG_PUGH
- printf("Calling Destroying Group %s \n",CCTK_GroupName(group));
+ printf("Calling Destroying Group %s\n", CCTK_GroupName(group));
+ fflush(stdout);
#endif
CCTK_GroupData(group,&pgroup);
@@ -214,7 +226,7 @@ void PUGH_DestroyPGH(pGH **GHin)
{
case GROUP_GF:
case CCTK_ARRAY:
- PUGH_DestroyPGA(&(((pGA ***)GH->variables)[variable][i]));
+ PUGH_DestroyGArray(&(((pGA ***)GH->variables)[variable][i]));
break;
case GROUP_SCALAR:
free(GH->variables[variable][i]);
@@ -231,13 +243,12 @@ void PUGH_DestroyPGH(pGH **GHin)
PUGH_DestroyPGExtras(&GH->GFExtras[i-1]);
}
+ free(GH->identity_string);
free(GH->Connectivity);
free(GH->GFExtras);
free(GH->variables);
free(GH);
*GHin = NULL;
-
- fflush(stdout);
}
void pGH_DumpInfo(pGH *GH)
@@ -323,3 +334,51 @@ int PUGH_SetupDefaultTopology(int dim, int *nprocs)
return retval;
}
+
+int PUGH_ParallelInit(cGH *GH)
+{
+ return 0;
+}
+
+int PUGH_Exit(int retval, cGH *GH)
+{
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Finalize());
+#endif
+ exit(retval);
+ return(retval);
+}
+int PUGH_Abort(cGH *GH)
+{
+ assert(0);
+ exit(0);
+ return(0);
+}
+
+
+int PUGH_MyProc(cGH *GH)
+{
+ int myproc;
+
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Comm_rank(PUGH_pGH(GH)->PUGH_COMM_WORLD, &myproc));
+#else
+ myproc = 0;
+#endif
+
+ return myproc;
+}
+
+
+int PUGH_nProcs(cGH *GH)
+{
+ int nprocs;
+
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Comm_size(PUGH_pGH(GH)->PUGH_COMM_WORLD, &nprocs));
+#else
+ nprocs = 1;
+#endif
+
+ return nprocs;
+}
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 171aa7c..d27ace0 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -69,12 +69,12 @@ pGExtras *PUGH_SetupPGExtras(int dim,
this->dim = dim;
- PUGH_SetupPGExtrasSizes (dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc,this);
+ PUGH_SetupPGExtrasSizes(dim, perme, stagger, sh, nghosts,
+ total_procs, nprocs, this_proc,this);
PUGH_SetupPGExtrasOwnership(dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc, this);
+ total_procs, nprocs, this_proc, this);
PUGH_SetupPGExtrasStaggering(dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc, this);
+ total_procs, nprocs, this_proc, this);
}
}
@@ -124,6 +124,8 @@ void PUGH_DestroyPGExtras(pGExtras **PGExtras)
free((*PGExtras)->nghostzones);
free((*PGExtras)->nsize);
free((*PGExtras)->lnsize);
+ free((*PGExtras)->iterator);
+ free((*PGExtras)->hyper_volume);
free(*PGExtras);
*PGExtras = NULL;
@@ -552,9 +554,11 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints = (int *) malloc(total_procs*sizeof(int));
/* Things just depending on dimension */
- this->nghostzones = (int *)malloc(dim*sizeof(int));
- this->nsize = (int *)malloc(dim*sizeof(int));
- this->lnsize = (int *)malloc(dim*sizeof(int));
+ this->nghostzones = (int *)malloc(dim*sizeof(int));
+ this->nsize = (int *)malloc(dim*sizeof(int));
+ this->lnsize = (int *)malloc(dim*sizeof(int));
+ this->iterator = (int *)malloc(dim*sizeof(int));
+ this->hyper_volume = (int *)malloc(dim*sizeof(int));
/* Check all the above succeeded and then get memory for
* arrays hanging off the above.
@@ -565,7 +569,9 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints &&
this->nghostzones &&
this->nsize &&
- this->lnsize)
+ this->lnsize &&
+ this->iterator &&
+ this->hyper_volume)
{
this->lb[0] = (int *)malloc(total_procs *dim*sizeof(int));
this->ub[0] = (int *)malloc(total_procs *dim*sizeof(int));
@@ -607,6 +613,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
}
else
@@ -626,6 +636,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
@@ -635,7 +649,9 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints &&
this->nghostzones &&
this->nsize &&
- this->lnsize)
+ this->lnsize &&
+ this->iterator &&
+ this->hyper_volume)
{
retcode = 0;
for (i = 0 ; i < PUGH_NSTAGGER; i++)
@@ -735,6 +751,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
}
}
@@ -853,14 +873,14 @@ int PUGH_SetupPGExtrasSizes(int dim,
@@*/
int PUGH_SetupPGExtrasOwnership(int dim,
- int *perme,
- int stagger,
- int *sh,
- int *nghosts,
- int total_procs,
- int *nprocs,
- int this_proc,
- pGExtras *this)
+ int *perme,
+ int stagger,
+ int *sh,
+ int *nghosts,
+ int total_procs,
+ int *nprocs,
+ int this_proc,
+ pGExtras *this)
{
int tmp;
int dir, idir;
@@ -1180,394 +1200,307 @@ int PUGH_SetupRemoteSizes(int dim,
return 0;
}
+
/*@@
- @routine PUGH_SetupPGA
- @date Mon Nov 8 16:29:34 1999
- @author Tom Goodale
+ @routine PUGH_SetupGArrayGroupComm
+ @date Tue 06 Jun 2000
+ @author Thomas Radke
@desc
- Sets up a new pGA.
+ Sets up a communication buffer for a group of GAs.
@enddesc
@calls
@calledby
@history
@endhistory
-
@@*/
-pGA *PUGH_SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger)
+pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
+ int dim,
+ int first_var,
+ int n_vars,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras)
{
- pGA *this;
int i;
- this = (pGA *)malloc(sizeof(pGA));
+ pComm *this;
- if(this)
- {
- this->extras = extras;
- this->connectivity = connectivity;
- this->parent = parent;
- this->varsize = varsize;
- this->vtype = vtype;
- this->stagger = stagger;
- this->id = id;
- this->storage = PUGH_NOSTORAGE;
- this->commflag = PUGH_NOCOMM;
+ this = (pComm *) malloc (sizeof (pComm));
- this->name = (char *) malloc((strlen(name)+1)*sizeof(char));
- this->buffer_sz = (int *) malloc(2*extras->dim*sizeof(int));
- this->send_buffer = (void **)malloc(2*extras->dim*sizeof(void *));
- this->recv_buffer = (void **)malloc(2*extras->dim*sizeof(void *));
- this->docomm = (int *) malloc(2*extras->dim*sizeof(int));
- this->padddata = (void *) calloc(1, varsize);
- this->data = this->padddata;
-
+ if (this)
+ {
+ this->buffer_sz = (int *) malloc (2 * dim * sizeof (int));
+ this->send_buffer = (void **) malloc (2 * dim * sizeof (void *));
+ this->recv_buffer = (void **) malloc (2 * dim * sizeof (void *));
#ifdef CCTK_MPI
- this->sreq = (MPI_Request *)malloc(2*extras->dim*sizeof(MPI_Request));
- this->rreq = (MPI_Request *)malloc(2*extras->dim*sizeof(MPI_Request));
+ this->sreq = (MPI_Request *) malloc (2 * dim * sizeof (MPI_Request));
+ this->rreq = (MPI_Request *) malloc (2 * dim * sizeof (MPI_Request));
#endif
+ this->docomm = (int *) malloc (2 * dim * sizeof (int));
- if(this->name &&
- this->buffer_sz &&
+ if(this->buffer_sz &&
this->send_buffer &&
this->recv_buffer &&
- this->docomm &&
#ifdef CCTK_MPI
this->sreq &&
this->rreq &&
#endif
- this->padddata)
+ this->docomm)
{
- strcpy(this->name, name);
+ this->commflag = PUGH_NOCOMM;
+ this->first_var = first_var;
+ this->n_vars = n_vars;
+ this->sync_timelevel = sync_timelevel;
- for(i = 0; i < 2*extras->dim; i++)
+#ifdef CCTK_MPI
+ switch (vartype)
+ {
+ case CCTK_VARIABLE_CHAR:
+ this->mpi_type = PUGH_MPI_CHAR; break;
+ case CCTK_VARIABLE_INT4:
+ this->mpi_type = PUGH_MPI_INT4; break;
+ case CCTK_VARIABLE_INT:
+ this->mpi_type = PUGH_MPI_INT; break;
+ case CCTK_VARIABLE_REAL4:
+ this->mpi_type = PUGH_MPI_REAL4; break;
+ case CCTK_VARIABLE_REAL:
+ this->mpi_type = PUGH_MPI_REAL; break;
+ case CCTK_VARIABLE_COMPLEX:
+ this->mpi_type = pughGH->PUGH_mpi_complex; break;
+
+ default:
+ CCTK_WARN (1, "Unsupported variable type for PUGH communication");
+ break;
+ }
+#endif
+
+ for (i = 0; i < 2 * dim; i++)
{
this->buffer_sz[i] = 0;
this->send_buffer[i] = NULL;
this->recv_buffer[i] = NULL;
- this->docomm[i] = 0;
#ifdef CCTK_MPI
/* Null my send and receive requests */
- this->sreq[i] = MPI_REQUEST_NULL;
- this->rreq[i] = MPI_REQUEST_NULL;
+ this->sreq[i] = MPI_REQUEST_NULL;
+ this->rreq[i] = MPI_REQUEST_NULL;
#endif
+ this->docomm[i] = PUGH_NOCOMM;
}
}
else
{
- free(this->name);
- free(this->buffer_sz);
- free(this->send_buffer);
- free(this->recv_buffer);
- free(this->docomm);
- free(this->padddata);
-
+ if (this->buffer_sz)
+ {
+ free (this->buffer_sz);
+ }
+ if (this->send_buffer)
+ {
+ free (this->send_buffer);
+ }
+ if (this->recv_buffer)
+ {
+ free (this->recv_buffer);
+ }
#ifdef CCTK_MPI
- free(this->sreq);
- free(this->rreq);
+ if (this->sreq)
+ {
+ free (this->sreq);
+ }
+ if (this->rreq)
+ {
+ free (this->rreq);
+ }
#endif
- free(this);
+ if (this->docomm)
+ {
+ free (this->docomm);
+ }
+
+ free (this);
this = NULL;
}
}
+ /* Now set the hyper_volume vector in the extras structure
+ which is used to copy the ghostzones from/to the comm buffers.
+ NOTE: Sizes are already in bytes here. hyper_volume[0] is unused.
+ */
+ if(this)
+ {
+ extras->hyper_volume[0] = CCTK_VarTypeSize(vartype);
+ for (i = 1; i < extras->dim; i++)
+ {
+ extras->hyper_volume[i] = extras->hyper_volume[i-1] * extras->lnsize[i-1];
+ }
+ }
+
return this;
}
+
/*@@
- @routine PUGH_DestroyPGA
- @date Mar 12 2000
+ @routine PUGH_SetupGArrayComm
+ @date Tue 06 Jun 2000
@author Thomas Radke
@desc
- Destroys a pGA object.
+ Sets up a communication buffer for a GA.
@enddesc
@calls
@calledby
@history
@endhistory
-
@@*/
-void PUGH_DestroyPGA(pGA **GA)
+pComm *PUGH_SetupGArrayComm(pGH *pughGH,
+ int dim,
+ int var,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras)
{
- if(GA && *GA)
- {
- if((*GA)->storage != PUGH_NOSTORAGE)
- {
- PUGH_DisableGADataStorage(*GA);
- }
- free((*GA)->name);
- free((*GA)->buffer_sz);
- free((*GA)->send_buffer);
- free((*GA)->recv_buffer);
- free((*GA)->docomm);
- free((*GA)->padddata);
-
-#ifdef CCTK_MPI
- free((*GA)->sreq);
- free((*GA)->rreq);
-#endif
- free(*GA);
- *GA = NULL;
- }
+ return PUGH_SetupGArrayGroupComm(pughGH,
+ dim,
+ var,
+ 1,
+ sync_timelevel,
+ vartype,
+ extras);
}
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing)
+
+ /*@@
+ @routine PUGH_DestroyComm
+ @date Tue 06 Jun 2000
+ @author Thomas Radke
+ @desc
+ Destroys a communication buffer.
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
+@@*/
+void PUGH_DestroyComm(pComm **comm)
{
- int retval;
- int i;
- int dir;
- int sz;
- int ldir;
- char *temp_byte;
-#if 0
- int special_pad, cache_size;
- long start;
+ free ((*comm)->buffer_sz);
+ free ((*comm)->send_buffer);
+ free ((*comm)->recv_buffer);
+#ifdef CCTK_MPI
+ free ((*comm)->sreq);
+ free ((*comm)->rreq);
#endif
+ free ((*comm)->docomm);
+ free (*comm);
- retval = 0;
- if(GA->storage == PUGH_NOSTORAGE)
- {
- /* Allocate memory for communication buffers */
- for(i = 0; i < 2*GA->extras->dim; i++)
- {
- dir = i/2;
-
- sz = GA->extras->nghostzones[dir];
- for(ldir = 0; ldir < GA->extras->dim; ldir++)
- {
- sz *= GA->extras->lnsize[ldir];
- }
-
- sz /= GA->extras->lnsize[dir];
-
- if(GA->connectivity->neighbours[this_proc][i] >= 0)
- {
- GA->buffer_sz[i] = sz;
- GA->send_buffer[i] = malloc(sz * GA->varsize);
- GA->recv_buffer[i] = malloc(sz * GA->varsize);
-
- if(!(GA->send_buffer[i]
- && GA->recv_buffer[i]))
- {
- retval = -1;
- break;
- }
- }
- else
- {
- GA->buffer_sz[i] = 0;
- GA->send_buffer[i] = NULL;
- GA->recv_buffer[i] = NULL;
- }
- }
-
- if(retval)
- {
- for(; i >=0 ; i--)
- {
- free(GA->send_buffer[i]);
- free(GA->recv_buffer[i]);
- GA->buffer_sz[i] = 0;
- }
- }
-
- /* Now assign memory for the variable itself */
- if(!retval)
- {
- if(GA->padddata)
- {
- free(GA->padddata);
- }
+ *comm = NULL;
+}
- if(!padding_active)
- {
- /* Easy case. */
- if(zero_memory)
- {
- GA->padddata = calloc(GA->extras->npoints, GA->varsize);
- }
- else
- {
- GA->padddata = malloc(GA->extras->npoints * GA->varsize);
- }
- GA->data = GA->padddata;
- }
- else
- {
+ /*@@
+ @routine PUGH_SetupGArray
+ @date Mon Nov 8 16:29:34 1999
+ @author Tom Goodale
+ @desc
+ Sets up a new pGA.
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
- /* Use the Cactus Cache alignment function */
- GA->data = Util_CacheMalloc(GA->id,
- GA->extras->npoints * GA->varsize,
- &(GA->padddata));
-
- /* Zero the memory if desired. */
- if(GA->data && zero_memory)
- {
- for(temp_byte = (char *)GA->data;
- temp_byte < ((char *)GA->data)+GA->extras->npoints * GA->varsize;
- temp_byte++)
- {
- *temp_byte = 0;
- }
- }
+@@*/
+pGA *PUGH_SetupGArray(void *parent,
+ pGExtras *extras,
+ pConnectivity *connectivity,
+ pComm *groupcomm,
+ const char *name,
+ int id,
+ int varsize,
+ int vtype,
+ int stagger)
+{
+ pGA *this;
- }
- }
- }
+ this = (pGA *)malloc(sizeof(pGA));
- if (!GA->padddata)
+ if(this)
{
- CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
- "FATAL ERROR: Cannot allocate data for %s [%d]\n",
- GA->name, GA->id);
- }
-
- GA->storage = PUGH_STORAGE;
-
- return retval;
-
-}
+ this->extras = extras;
+ this->connectivity = connectivity;
+ this->comm = NULL;
+ this->groupcomm = groupcomm;
+ this->parent = parent;
+ this->varsize = varsize;
+ this->vtype = vtype;
+ this->stagger = stagger;
+ this->id = id;
-int PUGH_DisableGADataStorage(pGA *GA)
-{
- int retval;
- int i;
+ this->storage = PUGH_NOSTORAGE;
- if (GA->storage == PUGH_STORAGE)
- {
- if (GA->padddata)
+ this->name = (char *) malloc((strlen(name)+1)*sizeof(char));
+ this->padddata = (void *) calloc(1, varsize);
+ this->data = this->padddata;
+
+ if(this->name && this->padddata)
{
- free(GA->padddata);
- GA->padddata = NULL;
- GA->data = NULL;
+ strcpy(this->name, name);
}
-
- for (i = 0; i < 2*GA->extras->dim; i++)
+ else
{
- if (GA->send_buffer[i])
+ if(this->name)
{
-#ifdef CCTK_MPI
- if(GA->sreq[i] != MPI_REQUEST_NULL)
- {
- CACTUS_MPI_ERROR(MPI_Request_free(&(GA->sreq[i])));
- }
-#endif
- free(GA->send_buffer[i]);
- GA->send_buffer[i] = NULL;
+ free(this->name);
}
- }
-
- for (i = 0; i < 2*GA->extras->dim; i++)
- {
- if (GA->recv_buffer[i])
+ if(this->padddata)
{
- free(GA->recv_buffer[i]);
- GA->recv_buffer[i] = NULL;
+ free(this->padddata);
}
+ free(this);
+ this = NULL;
}
-
- GA->padddata = calloc(1, GA->varsize);
- GA->data = GA->padddata;
-
- GA->storage = PUGH_NOSTORAGE;
- retval = 0;
- }
- else
- {
- retval = 1;
}
- return retval;
+ return this;
}
-
/*@@
- @routine PUGH_GAComm
- @date Sun Jan 23 12:46:23 2000
- @author Gabrielle Allen
+ @routine PUGH_DestroyGArray
+ @date Mar 12 2000
+ @author Thomas Radke
@desc
- This sets the docomm[2*dim] array of the GA based
- on the setting of the comm flag.
+ Destroys a pGA object.
@enddesc
-@@*/
-
+ @calls
+ @calledby
+ @history
+
+ @endhistory
-void PUGH_GAComm(pGA *GA, int docomm)
+@@*/
+void PUGH_DestroyGArray(pGA **GA)
{
- int idir;
- int dim = GA->extras->dim;
-
- /* Copy docomm */
- GA->commflag = docomm;
-
- /* First set all communcation off */
- for (idir=0;idir<2*dim;idir++)
- GA->docomm[idir] = 0;
-
- if (docomm == PUGH_NOCOMM)
- {
- for (idir=0;idir<2*dim;idir++)
- GA->docomm[idir] = 0;
- }
- else if (docomm == PUGH_ALLCOMM)
- {
- for (idir=0;idir<2*dim;idir++)
- {
- GA->docomm[idir] = 1;
- }
- }
- else if (docomm == PUGH_PLUSFACESCOMM)
- {
- for (idir=0;idir<dim;idir++)
- {
- GA->docomm[2*idir+1] = 1;
- }
- }
- else if (docomm == PUGH_MINUSFACESCOMM)
+ if(GA && *GA)
{
- for (idir=0;idir<dim;idir++)
+ if((*GA)->storage != PUGH_NOSTORAGE)
{
- GA->docomm[2*idir] = 1;
+ PUGH_DisableGArrayDataStorage(*GA);
}
- }
- else
- {
- for (idir=0;idir<dim;idir++)
+ /* destroy group comm buffers only once per group */
+ if((*GA)->groupcomm && (*GA)->groupcomm->first_var == (*GA)->id)
{
- if (docomm == PUGH_COMM(idir))
- {
- GA->docomm[2*idir] = 1;
- GA->docomm[2*idir+1] = 1;
- }
+ PUGH_DestroyComm(&(*GA)->groupcomm);
}
- }
-
- /* FIXME Add back the check that you have a valid COMM model: Gab */
-
- /* Handle nsize = 1 type cases. This is only important for one
- processor MPI periodic boundaries */
-
- for (idir=0;idir<dim;idir++)
- {
- if (GA->extras->nsize[idir] == 1)
+ if((*GA)->comm)
{
- GA->docomm[2*idir] = 0;
- GA->docomm[2*idir+1] = 0;
+ PUGH_DestroyComm(&(*GA)->comm);
}
+ free((*GA)->name);
+ free((*GA)->padddata);
+
+ free(*GA);
+ *GA = NULL;
}
-
}
diff --git a/src/Startup.c b/src/Startup.c
index fb5a51a..10a0740 100644
--- a/src/Startup.c
+++ b/src/Startup.c
@@ -17,15 +17,10 @@
#include "pugh_Comm.h"
#include "pugh_reductions.h"
-static char *WhatIsPugh(void);
-
static char *rcsid="$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_Startup_c)
-/* Store the handle in a global variable for the moment. */
-int pugh_GHExtension;
-
/*@@
@routine PUGH_Startup
@@ -43,6 +38,7 @@ int pugh_GHExtension;
@@*/
int PUGH_Startup(void)
{
+ int pugh_GHExtension;
char *banner;
/* Register the stuff in the GH extension. */
@@ -94,7 +90,7 @@ int PUGH_Startup(void)
}
-
+#if 0
/*@@
@routine WhatIsPugh
@date Mon Apr 14 22:13:12 1997
@@ -140,4 +136,4 @@ static char *WhatIsPugh(void)
return (pughnames[which]);
}
-
+#endif
diff --git a/src/Storage.c b/src/Storage.c
new file mode 100644
index 0000000..801f1a4
--- /dev/null
+++ b/src/Storage.c
@@ -0,0 +1,561 @@
+ /*@@
+ @file Storage.c
+ @date Tue Jun 13 2000
+ @author Thomas Radke
+ @desc
+ Pugh storage functions
+ @enddesc
+ @version $Header$
+ @@*/
+
+/*#define DEBUG_PUGH*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+#include "pugh.h"
+#include "pughi.h"
+
+static char *rcsid="$Header$";
+
+CCTK_FILEVERSION(CactusPUGH_PUGH_Storage_c)
+
+static int totalstorage = 0; /* Storage for GAs in Bytes */
+static int totalnumber = 0; /* Number of stored GAs */
+
+
+/*@@
+ @routine PUGH_ArrayGroupSize
+ @author Gabrielle Allen
+ @date 11 Apr 1999
+ @desc
+ Returns size of arrays in a group, in a given direction
+ @enddesc
+ @calls
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @vcomment either index or groupname must be present
+ @endvar
+ @var group
+ @vdesc index of group
+ @vtype int
+ @vio in
+ @vcomment either index or groupname must be given
+ @endvar
+ @var dir
+ @vdesc direction
+ @vtype int
+ @vio in
+ @@*/
+
+const int *PUGH_ArrayGroupSize(cGH *GH, int dir, int group, const char *groupname)
+{
+ int *sizep;
+ pGA *GA;
+
+ if (groupname)
+ {
+ group = CCTK_GroupIndex(groupname);
+ }
+
+ /* get pointer to pGA for a timelevel of first variable in this group */
+ GA = (pGA *) PUGH_pGH(GH)->variables[CCTK_FirstVarIndexI(group)][0];
+
+ if (GA->storage == PUGH_STORAGE)
+ {
+ if (dir >= 0 && dir < GA->extras->dim)
+ {
+ sizep = &GA->extras->lnsize[dir];
+ }
+ else
+ {
+ sizep = NULL;
+ CCTK_WARN(1, "Wrong value for dir");
+ }
+ }
+ else
+ {
+ sizep = &_cctk_one;
+ }
+
+ return (sizep);
+}
+
+
+/*@@
+ @routine PUGH_QueryGroupStorage
+ @author Gabrielle Allen
+ @date 13 Apr 1999
+ @desc
+ Returns true if group has storage assigned, false otherwise
+ @enddesc
+ @calls CCTK_DecomposeName
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be queried
+ @vtype const char *
+ @vio in
+ @vcomment either index or groupname must be present
+ @endvar
+ @var group
+ @vdesc index of group
+ @vtype int
+ @vio in
+ @vcomment either index or groupname must be given
+ @endvar
+ @@*/
+
+int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
+{
+
+ int first;
+ int storage=PUGH_UNDEFINEDSTORAGE;
+ int retval;
+ int grouptype;
+
+ if (groupname)
+ {
+ group = CCTK_GroupIndex(groupname);
+ }
+
+ /* get first variable in group */
+ first = CCTK_FirstVarIndexI(group);
+ if (first < 0)
+ {
+ CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Invalid group ID %d in PUGH_QueryGroupStorage", group);
+ }
+
+ grouptype = CCTK_GroupTypeI(group);
+
+ if (grouptype == CCTK_SCALAR)
+ {
+ storage = PUGH_STORAGE;
+ }
+ else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
+ {
+ storage = ((pGA *) PUGH_pGH(GH)->variables[first][0])->storage;
+ }
+ else
+ {
+ CCTK_WARN(0,"Unknown group type in PUGH_QueryGroupStorage");
+ }
+
+ if (storage == PUGH_STORAGE)
+ {
+ retval = 1;
+ }
+ else if (storage == PUGH_NOSTORAGE)
+ {
+ retval = 0;
+ }
+ else
+ {
+ CCTK_WARN(0,"Inconsistency in PUGH_QueryGroupStorage");
+ retval = PUGH_ERROR;
+ }
+
+ return (retval);
+}
+
+
+/*@@
+ @routine PUGH_EnableGroupStorage
+ @author Tom Goodale
+ @date 30 Mar 1999
+ @desc
+ Enables storage for all variables in the group indicated by groupname.
+ @enddesc
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
+ PUGH_EnableGArrayGroupStorage
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @endvar
+ @@*/
+
+int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
+{
+
+ DECLARE_CCTK_PARAMETERS
+
+ int group; /* group index */
+ int first_var; /* first variable's index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
+ pGA *GA;
+ pGH *pughGH;
+
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGroupStorage: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
+
+ pughGH = PUGH_pGH(GH);
+ group = CCTK_GroupIndex(groupname);
+ first_var = CCTK_FirstVarIndexI(group);
+
+ /* get the group info from its index */
+ CCTK_GroupData(group, &pgroup);
+
+ /* SCALAR types have always switched on memory */
+ if (pgroup.grouptype == CCTK_SCALAR)
+ {
+ rc = 1;
+ }
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
+ {
+ rc = PUGH_EnableGArrayGroupStorage(pughGH,
+ first_var,
+ pgroup.numvars,
+ pgroup.numtimelevels);
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupStorage");
+ rc = -1;
+ }
+
+ /* Report on memory usage */
+ if (storage_verbose)
+ {
+ if (rc == 0)
+ {
+ /* get GA pointer of first var in group */
+ GA = (pGA *) pughGH->variables[first_var][0];
+
+ /* Memory toggled */
+ totalnumber += pgroup.numvars;
+ totalstorage += GA->extras->npoints * GA->varsize *
+ pgroup.numtimelevels * pgroup.numvars;
+ printf("Switched memory on for %s \n [Num Arrays: %d Total Size: %d]\n",
+ groupname,totalnumber,totalstorage);
+ }
+ else if (rc == 1)
+ {
+ /* Memory already on */
+ printf("Memory already on for %s\n",groupname);
+ }
+ }
+
+ return (rc);
+
+}
+
+
+/*@@
+ @routine PUGH_DisableGroupStorage
+ @author Tom Goodale
+ @date 30 Mar 1999
+ @desc
+ Disables storage for all variables in the group indicated by groupname.
+ @enddesc
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @endvar
+ @@*/
+
+int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
+{
+ DECLARE_CCTK_PARAMETERS
+
+ int group; /* group index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
+ pGH *pughGH; /* PUGH extension reference */
+ pGA *GA;
+ int level;
+ int first_var,var;
+ int unchanged; /* count how many aren't toggled */
+
+ /* get PUGH extension handle */
+ pughGH = PUGH_pGH(GH);
+
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
+
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
+
+ /* get global index of first variable in group */
+ first_var = CCTK_FirstVarIndexI(group);
+
+ unchanged = 0;
+
+ if (pgroup.grouptype == CCTK_SCALAR)
+ {
+ rc = 1;
+ }
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
+ {
+ for (var = first_var; var < first_var+pgroup.numvars; var++)
+ {
+ for (level = 0; level < pgroup.numtimelevels; level++)
+ {
+ unchanged = unchanged +
+ PUGH_DisableGArrayDataStorage((pGA *)(pughGH->variables[var][level]));
+ }
+ }
+
+ rc = 1;
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupStorage");
+ rc = 0;
+ }
+
+ /* Report on memory usage */
+ if (storage_verbose)
+ {
+ if (unchanged == 0)
+ {
+ GA = (pGA *) pughGH->variables[first_var][0];
+
+ /* Memory toggled */
+ totalnumber -= pgroup.numvars;
+ totalstorage -= GA->extras->npoints * GA->varsize *
+ pgroup.numtimelevels * pgroup.numvars;
+ printf("Switched memory off for %s \n [Num Arrays: %d Total Size: %d]\n",
+ groupname,totalnumber,totalstorage);
+ }
+ else if (unchanged == pgroup.numvars)
+ {
+ /* Memory already off */
+ printf("Memory already off for %s\n",groupname);
+ }
+ else
+ {
+ CCTK_WARN(1,"Inconsistency in group memory assignment");
+ }
+ }
+
+ return (rc);
+}
+
+
+/*****************************************************************************/
+/* local functions */
+/*****************************************************************************/
+
+int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
+ int first_var,
+ int n_variables,
+ int n_timelevels)
+{
+ DECLARE_CCTK_PARAMETERS
+ int nstorage; /* Number of Arrays for which storage was set */
+ int nnostorage; /* Number of Arrays for which no storage was set */
+ int retval;
+ int var;
+ pGA *GA;
+ int level;
+
+ nstorage = 0;
+ nnostorage = 0;
+
+ for (var = first_var; var < first_var + n_variables; var++)
+ {
+ for (level = 0; level < n_timelevels; level++)
+ {
+ GA = (pGA *) pughGH->variables[var][level];
+
+ if (! GA->storage)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayGroupStorage: request for var '%s' "
+ "timelevel %d\n", GA->name, level);
+ fflush(stdout);
+#endif
+ PUGH_EnableGArrayDataStorage(GA,
+ pughGH->myproc,
+ zero_memory,
+ padding_active,
+ padding_cacheline_bits,
+ padding_size,
+ padding_address_spacing);
+
+ ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
+
+ nnostorage++;
+ }
+ else
+ {
+ ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
+ nstorage++;
+ }
+ }
+ }
+
+ if (nstorage > 0 && nnostorage > 0)
+ {
+ CCTK_WARN(0, "Group storage violation in PUGH_EnableGArrayGroupStorage");
+ retval = -1;
+ }
+ else if (nstorage > 0)
+ {
+ retval = 1;
+ }
+ else if (nnostorage > 0)
+ {
+ retval = 0;
+ }
+ else
+ {
+ CCTK_WARN(0, "Problem in PUGH_EnableGArrayGroupStorage");
+ retval = -1;
+ }
+
+ return retval;
+
+}
+
+
+int PUGH_EnableGArrayDataStorage(pGA *GA,
+ int this_proc,
+ int zero_memory,
+ int padding_active,
+ int padding_cacheline_bits,
+ int padding_size,
+ int padding_address_spacing)
+{
+
+ if(GA->storage == PUGH_NOSTORAGE)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayDataStorage: allocating storage "
+ "for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
+
+ /* Now assign memory for the variable itself */
+ if(GA->padddata)
+ {
+ free(GA->padddata);
+ }
+
+ if(!padding_active)
+ {
+ /* Easy case. */
+ if(zero_memory)
+ {
+ GA->padddata = calloc(GA->extras->npoints, GA->varsize);
+ }
+ else
+ {
+ GA->padddata = malloc(GA->extras->npoints * GA->varsize);
+ }
+
+ GA->data = GA->padddata;
+ }
+ else
+ {
+
+ /* Use the Cactus Cache alignment function */
+ GA->data = Util_CacheMalloc(GA->id,
+ GA->extras->npoints * GA->varsize,
+ &(GA->padddata));
+
+ /* Zero the memory if desired. */
+ if(GA->data && zero_memory)
+ {
+ memset(GA->data, 0, GA->extras->npoints * GA->varsize);
+ }
+
+ }
+ }
+
+ if (!GA->padddata)
+ {
+ CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "FATAL ERROR: Cannot allocate data for %s [%d]\n",
+ GA->name, GA->id);
+ }
+
+ GA->storage = PUGH_STORAGE;
+
+ return 0;
+
+}
+
+
+int PUGH_DisableGArrayDataStorage(pGA *GA)
+{
+ int retval;
+
+ if (GA->storage == PUGH_STORAGE)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayDataStorage: freeing storage for var '%s'\n",
+ GA->name);
+ fflush(stdout);
+#endif
+ if (GA->padddata)
+ {
+ free(GA->padddata);
+ GA->padddata = NULL;
+ GA->data = NULL;
+ }
+
+ GA->padddata = calloc(1, GA->varsize);
+ GA->data = GA->padddata;
+
+ GA->storage = PUGH_NOSTORAGE;
+ retval = 0;
+ }
+ else
+ {
+ retval = 1;
+ }
+
+ return retval;
+}
diff --git a/src/include/pGH.h b/src/include/pGH.h
index 9fe17e7..d5139d9 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -13,10 +13,6 @@
typedef struct PGH
{
-#ifdef CCTK_MPI
- MPI_Comm PUGH_COMM_WORLD; /* A MPIcommunicator */
-#endif
-
/* pGH identifier */
void *callerid;
int dim; /* The dimension of the GH */
@@ -55,12 +51,16 @@ typedef struct PGH
/* 0 -> GH is skipped for conv.test */
/* is used in NanCheckGH.c */
+#if 1
/* FIXME */
double comm_time; /* time spent in communication */
+#endif
#ifdef CCTK_MPI
- /* the complex datatype defined in SetupPGH.c */
- MPI_Datatype PUGH_mpi_complex;
+ MPI_Datatype PUGH_mpi_complex;/* the MPI datatype for COMPLEX types */
+ MPI_Comm PUGH_COMM_WORLD; /* the MPI communicator */
+
+#ifdef PUGH_WITH_DERIVED_DATATYPES
/* Derived data types for communication */
MPI_Datatype send_char_dt [PUGH_NSTAGGER][6];
MPI_Datatype recv_char_dt [PUGH_NSTAGGER][6];
@@ -71,15 +71,14 @@ typedef struct PGH
MPI_Datatype send_complex_dt [PUGH_NSTAGGER][6];
MPI_Datatype recv_complex_dt [PUGH_NSTAGGER][6];
#endif
+#endif
/* Used for all grid functions */
pGExtras **GFExtras; /* [dim] stagger ? */
pConnectivity **Connectivity; /* [dim] */
- /* FIXME : should be a dynamically allocated string.
- * If you change this, please change the line in PUGH_SetupGH.
- */
- char identity_string[20];
+ char *identity_string; /* identifier for this pGH */
+
} pGH;
#define XDP 1
diff --git a/src/include/pGV.h b/src/include/pGV.h
index 841ecb0..e86afb1 100644
--- a/src/include/pGV.h
+++ b/src/include/pGV.h
@@ -49,6 +49,10 @@ typedef struct PGExtras
int *rnpoints; /* Number of points on each proc */
int **rnsize; /* [#points on a proc][in each dir] */
+ /* Copying ghostzones to/from comm buffers */
+ int *iterator; /* Iterator for copying ghostzones [dim] */
+ int *hyper_volume; /* Points per subcube dimension [dim] */
+
/* Ghosts and overlaps. */
int *nghostzones; /* Width of ghost zone */
int *ownership[PUGH_NSTAGGER][2];
@@ -65,19 +69,34 @@ typedef struct PGExtras
} pGExtras;
+typedef struct PComm
+{
+ /* buffer_sz[2*dim] */
+ int *buffer_sz; /* Size of the face ghost zones */
+ void **send_buffer; /* Storage for buffered Comm if */
+ void **recv_buffer; /* we don't use derived types */
+ int commflag; /* What is the comm flag set to? */
+ /* do_comm[2*dim] */
+ int *docomm; /* Do we do comm or not? */
+
+ int first_var; /* first variable to be synced */
+ int n_vars; /* number of variables to be synced */
+ int sync_timelevel; /* the timelevel to be synced */
+
+#ifdef CCTK_MPI
+ MPI_Request *sreq, *rreq; /* Comm requests and statuses. */
+ MPI_Status ms;
+ MPI_Datatype mpi_type; /* MPI datatype to use for communication */
+#endif
+
+} pComm;
+
typedef struct PGA
{
char *name; /* The name of the grid function */
int id; /* My ID number in my GH parent. */
void *padddata; /* Storage for the data. */
void *data; /* See the note above. */
- /* buffer_sz[2*dim] */
- int *buffer_sz; /* Size of the face ghost zones */
- void **send_buffer; /* Storage for buffered Comm if */
- void **recv_buffer; /* we don't use derived types */
- int commflag; /* What is the comm flag set to? */
- /* do_comm[2*dim] */
- int *docomm; /* Do we do comm or not? */
int storage; /* Do we have storage or not? */
int stagger; /* Only Vertex Centered now... */
@@ -92,13 +111,12 @@ typedef struct PGA
int varsize; /* The size of the data */
int vtype; /* The type of the data */
-#ifdef CCTK_MPI
- MPI_Request *sreq, *rreq; /* Comm requests and statuses. */
- MPI_Status ms;
-#endif
-
pGExtras *extras;
pConnectivity *connectivity;
+
+ pComm *comm; /* comm buffer for single variable */
+ pComm *groupcomm; /* comm buffer for a variable group */
+
} pGA;
typedef struct PGV
diff --git a/src/include/pugh.h b/src/include/pugh.h
index ce8c89b..7408c0f 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -72,18 +72,21 @@
#ifdef CCTK_MPI
-#define CACTUS_MPI_ERROR(xf) do {int errcode; \
- if((errcode = xf) != MPI_SUCCESS) \
- { \
- char mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
- int resultlen; \
- MPI_Error_string(errcode, mpi_error_string, &resultlen);\
- fprintf(stderr, "MPI Call %s returned error code %d (%s)\n", \
- #xf, errcode, mpi_error_string); \
- fprintf(stderr, "At line %d of file %s\n", \
- __LINE__, __FILE__); \
- } \
- } while (0)
+#define CACTUS_MPI_ERROR(fn_call) \
+ do { \
+ int errcode; \
+ \
+ if ((errcode = fn_call) != MPI_SUCCESS) \
+ { \
+ char mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
+ int resultlen; \
+ \
+ MPI_Error_string (errcode, mpi_error_string, &resultlen); \
+ fprintf (stderr, "MPI call '%s' returned error code %d (%s)\n", \
+ #fn_call, errcode, mpi_error_string); \
+ fprintf(stderr, "At line %d of file %s\n", __LINE__, __FILE__); \
+ } \
+ } while (0)
#endif
#ifdef _cplusplus
@@ -93,21 +96,21 @@ extern "C"
int PUGH_SetupGroup(pGH *newGH,
- int *nsize,
- int *nghostsize,
- int gtype,
- int vtype,
- int dim,
- int n_variables,
- int staggertype,
- int n_timelevels);
+ int *nsize,
+ int *nghostsize,
+ int gtype,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggertype,
+ int n_timelevels);
pGH *PUGH_SetupPGH(void *callerid,
int dim,
int *nsize,
int *nghostzones,
- int staggertype,
- int *perme);
+ int staggertype,
+ int *perme);
int PUGH_GFSize(int dim,
int *nsize);
@@ -115,7 +118,7 @@ int PUGH_GFSize(int dim,
int PUGH_GFGhostsize(int dim,
int *ghostsize);
int PUGH_GFPeriodic(int dim,
- int *perme);
+ int *perme);
pGH *PUGH_pGH(cGH *GH);
diff --git a/src/include/pughi.h b/src/include/pughi.h
index 8defba7..f6fdf1c 100644
--- a/src/include/pughi.h
+++ b/src/include/pughi.h
@@ -30,22 +30,22 @@ pGExtras *PUGH_SetupPGExtras(int dim,
int *nprocs,
int this_proc);
-pGA *SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger);
-
-
-void pGH_DumpInfo(pGH *GH);
+pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
+ int dim,
+ int first_var,
+ int n_vars,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras);
+
+void pGH_DumpInfo(pGH *pughGH);
int PUGH_SetupDefaultTopology(int dim, int *nprocs);
-int pGH_SetupnProcs(pGH *GH,int dim);
-void PUGH_DestroyPGA(pGA **GA);
+int pGH_SetupnProcs(pGH *pughGH,int dim);
+void PUGH_DestroyGArray(pGA **GA);
+void PUGH_DestroyComm(pComm **comm);
void PUGH_DestroyConnectivity(pConnectivity **conn);
void PUGH_DestroyPGExtras(pGExtras **PGExtras);
+void PUGH_DestroyPGH(pGH **pughGH);
int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs);
@@ -104,25 +104,6 @@ int PUGH_SetupRemoteSizes(int dim,
int *nprocs,
pGExtras *this);
-pGA *PUGH_SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger);
-
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing);
-
-int PUGH_DisableGADataStorage(pGA *GA);
-
int PUGH_SetupPGExtrasStaggering(int dim,
int *perme,
int stagger,
@@ -133,6 +114,51 @@ int PUGH_SetupPGExtrasStaggering(int dim,
int this_proc,
pGExtras *this);
+pGA *PUGH_SetupGArray(void *parent,
+ pGExtras *extras,
+ pConnectivity *connectivity,
+ pComm *comm,
+ const char *name,
+ int id,
+ int varsize,
+ int vtype,
+ int stagger);
+
+pComm *PUGH_SetupGArrayComm(pGH *pughGH,
+ int dim,
+ int var,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras);
+
+int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
+ int first_var,
+ int n_variables,
+ int n_timelevels);
+
+int PUGH_EnableGArrayDataStorage(pGA *GA,
+ int this_proc,
+ int zero_memory,
+ int padding_active,
+ int padding_cacheline_bits,
+ int padding_size,
+ int padding_address_spacing);
+
+int PUGH_DisableGArrayDataStorage(pGA *GA);
+
+int PUGH_EnableGArrayComm(pGA *GA,
+ int commflag);
+
+int PUGH_DisableGArrayComm(pGA *GA);
+
+int PUGH_SyncGArray(pGA *GA);
+
+#ifdef CCTK_MPI
+void PostSendGA (pGH *pughGH, int dir, pComm *comm);
+void PostReceiveGA (pGH *pughGH, int dir, pComm *comm);
+void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm);
+#endif
+
#ifdef _cplusplus
}
#endif
diff --git a/src/make.code.defn b/src/make.code.defn
index d28f187..f5cda42 100644
--- a/src/make.code.defn
+++ b/src/make.code.defn
@@ -1,8 +1,4 @@
# Main make.code.defn file for thorn pugh
# : /usr/users/cactus/CCTK/lib/make/new_thorn.pl,v 1.1 1999/02/03 17:00:50 goodale Exp n
# Source files in this directory
-SRCS = Evolve.c Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
-
-# Subdirectories containing source files
-SUBDIRS =
-
+SRCS = Evolve.c Startup.c GHExtension.c Comm.c Storage.c SetupPGH.c SetupGroup.c Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
diff --git a/src/pugh_extension.h b/src/pugh_extension.h
index b2b2b1b..62cf29b 100644
--- a/src/pugh_extension.h
+++ b/src/pugh_extension.h
@@ -11,8 +11,6 @@
#ifndef _PUGH_EXTENSION_H_
#define _PUGH_EXTENSION_H_ 1
-extern int pugh_GHExtension;
-
#ifdef _cplusplus
extern "C"
{
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-20 15:25:45 +0000