aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/Comm.c1266
-rw-r--r--src/Evolve.c2
-rw-r--r--src/FinishReceiveGA.c239
-rw-r--r--src/GHExtension.c15
-rw-r--r--src/LoadAware.c1
-rw-r--r--src/PostReceiveGA.c130
-rw-r--r--src/PostSendGA.c341
-rw-r--r--src/Reduction.c83
-rw-r--r--src/SetupGroup.c121
-rw-r--r--src/SetupPGH.c91
-rw-r--r--src/SetupPGV.c559
-rw-r--r--src/Startup.c10
-rw-r--r--src/Storage.c561
-rw-r--r--src/include/pGH.h19
-rw-r--r--src/include/pGV.h42
-rw-r--r--src/include/pugh.h49
-rw-r--r--src/include/pughi.h90
-rw-r--r--src/make.code.defn6
-rw-r--r--src/pugh_extension.h2
19 files changed, 1866 insertions, 1761 deletions
diff --git a/src/Comm.c b/src/Comm.c
index 528ec63..31e2350 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -14,15 +14,11 @@
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
-#include <assert.h>
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
#include "cctk.h"
#include "pugh.h"
+#include "pughi.h"
#include "pugh_Comm.h"
#include "cctk_Parameters.h"
@@ -30,41 +26,23 @@ static char *rcsid="$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_Comm_c)
-static int totalstorage=0; /* Storage for GAs in Bytes */
-static int totalnumber=0; /* Number of stored GAs */
-
-/* index into GH->extensions[] to get the PUGH extension handle */
-extern int pugh_GHExtension;
/* local function prototypes */
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+ int first_var,
+ int commflag);
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+ int first_var);
+static int PUGH_EnableComm(pGH *pughGH,
+ pComm *comm,
+ int commflag);
+static int PUGH_DisableComm(pGH *pughGH,
+ pComm *comm);
+static int PUGH_SyncGArrayGroup(pGH *pughGH,
+ int first_var);
+static int PUGH_Sync(pGH *pughGH,
+ pComm *comm);
-int PUGH_SyncGroupGA (cGH *GH,
- int group,
- int vtype,
- int n_vars,
- int timelevel);
-int PUGH_SyncGAs(pGH *pughGH,
- int dim,
- int first_var,
- int n_vars,
- int timelevel);
-int PUGH_EnableArrayGroupStorage(cGH *GH,
- int group,
- int vtype,
- int dim,
- int n_variables,
- int n_timelevels);
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing);
-void FinishReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostReceiveGA(pGH *GH, pGA *GA, int dir);
-void PostSendGA(pGH *GH, pGA *GA, int dir);
-int PUGH_GAComm(pGA *GA, int docomm);
/*@@
@routine PUGH_SyncGroup
@@ -75,7 +53,7 @@ int PUGH_GAComm(pGA *GA, int docomm);
Only groups of type GROUP_ARRAY and GROUP_GF can be synchronized.
@enddesc
@calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- PUGH_SyncGroupArray PUGH_SyncGroupGF
+ PUGH_SyncGArrayGroup
@history
@endhistory
@@ -91,66 +69,48 @@ int PUGH_GAComm(pGA *GA, int docomm);
@endvar
@@*/
-int PUGH_SyncGroup (cGH *GH, const char *groupname)
+int PUGH_SyncGroup(cGH *GH, const char *groupname)
{
- cGroup pgroup; /* pointer to group information */
+ cGroup pgroup; /* group information */
int group; /* group index */
- int timelevel; /* timelevel to be synchronized */
int rc; /* return code */
- group = CCTK_GroupIndex(groupname);
+#ifdef DEBUG_PUGH
+ printf (" PUGH_SyncGroup: request for group '%s'\n", groupname);
+ fflush (stdout);
+#endif
/* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
-#if 0
- /* Compute the timelevel to synchronize */
- if (pgroup.numtimelevels > 1)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- timelevel = pgroup.numtimelevels - 2;
+ rc = 0;
+ CCTK_WARN(4, "Synchronising a scalar in PUGH");
}
- else
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- timelevel = pgroup.numtimelevels - 1;
+ rc = PUGH_SyncGArrayGroup(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
}
-#endif
-
- timelevel = pgroup.numtimelevels - 1;
-
- /* branch to synchronization function according to group type */
- switch (pgroup.grouptype)
+ else
{
- case GROUP_SCALAR :
- rc = 0;
- CCTK_WARN(4,"Synchronising a scalar in PUGH");
- break;
-
- case GROUP_ARRAY :
- case GROUP_GF :
- rc = PUGH_SyncGroupGA (GH,
- group,
- pgroup.vartype,
- pgroup.numvars,
- timelevel);
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_SyncGroup");
- rc = 0;
- break;
+ CCTK_WARN(1, "Unknown group type in PUGH_SyncGroup");
+ rc = 0;
}
return (rc);
}
+
/*@@
- @routine PUGH_ArrayGroupSize
- @author Gabrielle Allen
- @date 11 Apr 1999
+ @routine PUGH_EnableGroupComm
+ @author Thomas Radke
+ @date 30 Mar 1999
@desc
- Returns size of arrays in a group, in a given direction
+ Enables communication for all variables in the group indicated by groupname.
@enddesc
- @calls
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
@history
@endhistory
@@ -163,109 +123,52 @@ int PUGH_SyncGroup (cGH *GH, const char *groupname)
@vdesc name of the group to be synchronized
@vtype const char *
@vio in
- @vcomment either index or groupname must be present
@endvar
- @var group
- @vdesc index of group
- @vtype int
- @vio in
- @vcomment either index or groupname must be given
- @endvar
- @var dir
- @vdesc direction
- @vtype int
- @vio in
@@*/
-const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
+int PUGH_EnableGroupComm(cGH *GH, const char *groupname)
{
- int first;
- int *sizep=NULL;
- pGH *pughGH;
- pGA *pughGA;
-
- if (groupname)
- {
- group = CCTK_GroupIndex(groupname);
- }
+ int group; /* group index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGroupComm: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
- /* get first variable in group */
- first = CCTK_FirstVarIndexI(group);
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
- if (CCTK_GroupTypeI(group)==GROUP_GF)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- /* get pointer to pGF for a timelevel of this variable */
- pughGA = (pGA *)(pughGH->variables[first][0]);
-
- if (pughGA->storage == PUGH_STORAGE)
- {
- switch (dir)
- {
- case 0:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[0];
- break;
- case 1:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[1];
- break;
- case 2:
- sizep = &pughGH->GFExtras[pughGH->dim-1]->lnsize[2];
- break;
- default :
- sizep = NULL;
- printf("Wrong value for dir in PUGH_ArrayGroupSize\n");
- break;
- }
- }
- else
- {
- sizep = &_cctk_one;
- }
+ rc = 1;
}
- else if (CCTK_GroupTypeI(group)==GROUP_ARRAY)
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- /* get pointer to pGA for a timelevel of this variable */
- pughGA = (pGA *)(pughGH->variables[first][0]);
-
- if (pughGA->storage == PUGH_STORAGE)
- {
- switch (dir)
- {
- case 0:
- sizep = &pughGA->extras->lnsize[0];
- break;
- case 1:
- sizep = &pughGA->extras->lnsize[1];
- break;
- case 2:
- sizep = &pughGA->extras->lnsize[2];
- break;
- default :
- sizep = NULL;
- CCTK_WARN(1,"Wrong value for dir in PUGH_ArrayGroupSize");
- break;
- }
- }
- else
- {
- sizep = &_cctk_one;
- }
+ rc = PUGH_EnableGArrayGroupComm(PUGH_pGH(GH),
+ CCTK_FirstVarIndexI(group),
+ PUGH_ALLCOMM);
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupComm");
+ rc = 0;
}
- return (sizep);
+ return (rc);
}
/*@@
- @routine PUGH_QueryGroupStorage
- @author Gabrielle Allen
- @date 13 Apr 1999
+ @routine PUGH_DisableGroupComm
+ @author Thomas Radke
+ @date 30 Mar 1999
@desc
- Returns true if group has storage assigned, false otherwise
+ Disables communication for all variables in the group indicated by groupname.
@enddesc
- @calls CCTK_DecomposeName
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
@history
@endhistory
@@ -275,595 +178,523 @@ const int *PUGH_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupna
@vio in
@endvar
@var groupname
- @vdesc name of the group to be queried
+ @vdesc name of the group to be synchronized
@vtype const char *
@vio in
- @vcomment either index or groupname must be present
@endvar
- @var group
- @vdesc index of group
- @vtype int
- @vio in
- @vcomment either index or groupname must be given
- @endvar
@@*/
-int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
+int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
{
+ int group; /* group index */
+ cGroup pgroup; /* pointer to group information */
+ int rc; /* return code */
- int first;
- int storage=PUGH_UNDEFINEDSTORAGE;
- int retval;
- int grouptype;
- pGH *pughGH;
-
- if (groupname)
- {
- group = CCTK_GroupIndex(groupname);
- }
-
- /* get first variable in group */
- first = CCTK_FirstVarIndexI(group);
- if (first < 0)
- {
- CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
- "Invalid group ID %d in PUGH_QueryGroupStorage", group);
- }
-
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-
- grouptype = CCTK_GroupTypeI(group);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGroupComm: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
- if (grouptype == CCTK_SCALAR)
- {
- storage = PUGH_STORAGE;
- }
- else if (grouptype == CCTK_GF)
- {
- storage = ((pGA *)(pughGH->variables[first][0]))->storage;
- }
- else if (grouptype == CCTK_ARRAY)
- {
- storage = ((pGA *)(pughGH->variables[first][0]))->storage;
- }
- else
- {
- CCTK_WARN(0,"Unknown group type in PUGH_QueryGroupStorage");
- }
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
- if (storage == PUGH_STORAGE)
+ if (pgroup.grouptype == CCTK_SCALAR)
{
- retval = 1;
+ rc = 1;
}
- else if (storage == PUGH_NOSTORAGE)
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
{
- retval = 0;
+ rc = PUGH_DisableGArrayGroupComm(PUGH_pGH(GH), CCTK_FirstVarIndexI(group));
}
else
{
- CCTK_WARN(0,"Inconsistency in PUGH_QueryGroupStorage");
- retval = PUGH_ERROR;
+ CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupComm");
+ rc = 0;
}
- return (retval);
+ return (rc);
}
-/*@@
- @routine PUGH_EnableGroupStorage
- @author Tom Goodale
- @date 30 Mar 1999
- @desc
- Enables storage for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- PUGH_EnableArrayGroupStorage
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
-
-int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_EnableGArrayComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Enables communication for a single array.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_EnableGArrayComm(pGA *GA,
+ int commflag)
{
- DECLARE_CCTK_PARAMETERS
-
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- int rc; /* return code */
- pGH *pughGH;
- int first_var;
-
#ifdef DEBUG_PUGH
- printf(" PUGH_EnableGroupStorage: request for group -%s-\n",groupname);
+ printf(" PUGH_EnableGArrayComm: request for var '%s' commflag %d\n",
+ GA->name, commflag);
fflush(stdout);
#endif
- group = CCTK_GroupIndex(groupname);
-
- /* get the group info from its index */
- CCTK_GroupData (group, &pgroup);
-
- /* branch to "enable storage" function according to group type */
- switch (pgroup.grouptype)
- {
- case CCTK_SCALAR :
- rc = 1;
- break;
-
- case CCTK_ARRAY :
- case CCTK_GF :
- rc = PUGH_EnableArrayGroupStorage (GH,
- group,
- pgroup.vartype,
- pgroup.dim,
- pgroup.numvars,
- pgroup.numtimelevels);
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupStorage");
- rc = 0;
- break;
- }
-
- /* Report on memory usage */
- if (storage_verbose)
- {
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ return (PUGH_EnableComm((pGH *) GA->parent, GA->comm, commflag));
+}
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
- if (rc == 0)
- {
- /* Memory toggled */
- totalnumber = totalnumber + pgroup.numvars;
- totalstorage = totalstorage +
- ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
- *CCTK_VarTypeSize(pgroup.vartype)
- *pgroup.numtimelevels
- *pgroup.numvars;
- printf("Switched memory on for %s \n [Num Arrays: %d Total Size: %d]\n",
- groupname,totalnumber,totalstorage);
- }
- else if (rc == 1)
- {
- /* Memory already on */
- printf("Memory already on for %s\n",groupname);
- }
- }
+ /*@@
+ @routine PUGH_DisableGArrayComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Disables communication for a single array.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_DisableGArrayComm(pGA *GA)
+{
- return (rc);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayComm: request for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
+ return (PUGH_DisableComm((pGH *) GA->parent, GA->comm));
}
-/*@@
- @routine PUGH_DisableGroupStorage
- @author Tom Goodale
- @date 30 Mar 1999
- @desc
- Disables storage for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
-
-int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_SyncGArrayGroup
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Synchronizes a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_SyncGArrayGroup(pGH *pughGH,
+ int first_var)
{
- DECLARE_CCTK_PARAMETERS
+ pGA *firstGA;
+
+
+ firstGA = (pGA *) pughGH->variables [first_var][0];
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- int rc; /* return code */
- pGH *pughGH; /* PUGH extension reference */
- int level;
- int first_var,var;
- int unchanged; /* count how many aren't toggled */
- int PUGH_DisableGADataStorage(pGA *GA);
-
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
-
#ifdef DEBUG_PUGH
- printf(" PUGH_DisableGroupStorage: request for group -%s-\n",groupname);
+ printf(" PUGH_SyncGArrayGroup: request for group with first var '%s'\n",
+ firstGA->name);
fflush(stdout);
#endif
- group = CCTK_GroupIndex(groupname);
+ return (PUGH_Sync(pughGH, firstGA->groupcomm));
+}
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ /*@@
+ @routine PUGH_SyncGArray
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Synchronizes a single array variable given the GA structure
+ of this variable.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+int PUGH_SyncGArray(pGA *GA)
+{
- unchanged = 0;
- switch (pgroup.grouptype)
- {
- case GROUP_SCALAR :
- rc = 1;
- break;
+#ifdef DEBUG_PUGH
+ printf(" PUGH_SyncGArray: request for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- {
- for (level = 0; level < pgroup.numtimelevels; level++)
- {
- unchanged = unchanged +
- PUGH_DisableGADataStorage ((pGA *)(pughGH->variables[var][level]));
- }
- }
- rc = 1;
- break;
+ return (PUGH_Sync((pGH *) GA->parent, GA->comm));
+}
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupStorage");
- rc = 0;
- break;
- }
- /* Report on memory usage */
- if (storage_verbose)
- {
- if (unchanged == 0)
- {
- /* Memory toggled */
- totalnumber = totalnumber - pgroup.numvars;
- totalstorage = totalstorage -
- ((pGA ***)pughGH->variables)[first_var][0]->extras->npoints
- *CCTK_VarTypeSize(pgroup.vartype)
- *pgroup.numtimelevels
- *pgroup.numvars;
- printf("Switched memory off for %s \n [Num Arrays: %d Total Size: %d]\n",
- groupname,totalnumber,totalstorage);
- }
- else if (unchanged == pgroup.numvars)
- {
- /* Memory already off */
- printf("Memory already off for %s\n",groupname);
- }
- else
- {
- CCTK_WARN(1,"Inconsistency in group memory assignment");
- }
- }
+int PUGH_Barrier(cGH *GH)
+{
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Barrier(PUGH_pGH(GH)->PUGH_COMM_WORLD));
+#endif
- return (rc);
+ return 0;
}
-/*@@
- @routine PUGH_EnableGroupComm
- @author Thomas Radke
- @date 30 Mar 1999
- @desc
- Enables communication for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
+/*****************************************************************************/
+/* local functions */
+/*****************************************************************************/
-int PUGH_EnableGroupComm (cGH *GH, const char *groupname)
+ /*@@
+ @routine PUGH_EnableGArrayGroupComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Enables communication for a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_EnableGArrayGroupComm(pGH *pughGH,
+ int first_var,
+ int commflag)
{
- cGroup pgroup; /* pointer to group information */
- int group; /* group index */
- pGH *pughGH; /* PUGH extension reference */
- int var, level; /* loop variables */
- int first_var; /* global index of first variable in group */
- int rc; /* return code */
+ pGA *GA; /* first variable in group */
- group = CCTK_GroupIndex(groupname);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ GA = pughGH->variables [first_var][0];
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayGroupComm: request for group "
+ "with first var '%s', commflag %d\n", GA->name, commflag);
+ fflush(stdout);
+#endif
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ return (PUGH_EnableComm(pughGH, GA->groupcomm, commflag));
+}
- /* branch to "enable comm" function according to group type */
- switch (pgroup.grouptype)
- {
- case GROUP_SCALAR :
- rc = 1;
- break;
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- {
- for (level = 0; level < pgroup.numtimelevels; level++)
- {
- PUGH_GAComm ((pGA *)(pughGH->variables[var][level]),
- PUGH_ALLCOMM);
- }
- }
- rc = 1;
- break;
+ /*@@
+ @routine PUGH_DisableGArrayGroupComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Disables communication for a group of arrays
+ given the first variable within this group.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_DisableGArrayGroupComm(pGH *pughGH,
+ int first_var)
+{
+ pGA *GA; /* first variable in group */
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_EnableGroupComm");
- rc = 0;
- break;
- }
- return (rc);
-}
+ GA = pughGH->variables [first_var][0];
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayGroupComm: request for group "
+ "with first var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
-/*@@
- @routine PUGH_DisableGroupComm
- @author Thomas Radke
- @date 30 Mar 1999
- @desc
- Disables communication for all variables in the group indicated by groupname.
- @enddesc
- @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
- @history
-
- @endhistory
- @var GH
- @vdesc Pointer to CCTK grid hierarchy
- @vtype cGH
- @vio in
- @endvar
- @var groupname
- @vdesc name of the group to be synchronized
- @vtype const char *
- @vio in
- @endvar
- @@*/
+ return (PUGH_DisableComm(pughGH, GA->groupcomm));
+}
-int PUGH_DisableGroupComm(cGH *GH, const char *groupname)
+
+ /*@@
+ @routine PUGH_EnableComm
+ @date Sun Jan 23 12:46:23 2000
+ @author Gabrielle Allen
+ @desc
+ This sets the docomm[2*dim] array of the GA based
+ on the setting of the comm flag and allocates the comm buffers.
+ @enddesc
+ @history
+ @date Mon Jun 05 2000 @author Thomas Radke
+ Moved buffer allocation from PUGH_EnableGArrayDataStorage
+ @endhistory
+@@*/
+static int PUGH_EnableComm(pGH *pughGH,
+ pComm *comm,
+ int commflag)
{
- int group; /* group index */
- cGroup pgroup; /* pointer to group information */
- pGH *pughGH; /* PUGH extension reference */
- int var, level; /* loop variables */
- int first_var; /* global index of first variable in group */
- int rc; /* return code */
+ int retval; /* return value */
+#ifdef CCTK_MPI
+ pGA *GA; /* GA structure the pComm belongs to */
+ int i, idir; /* loopers */
+ int dir; /* direction */
+ int sz; /* buffer size */
+#endif
- group = CCTK_GroupIndex(groupname);
- /* get the group info from its index */
- CCTK_GroupData (group,&pgroup);
+ retval = 1;
- /* get global index of first variable in group */
- first_var = CCTK_FirstVarIndexI (group);
+#ifdef CCTK_MPI
+ /* check whether this comm buffer is already set up */
+ if (comm->commflag == commflag)
+ {
+ return (retval);
+ }
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [pugh_GHExtension];
+ /* get the GA structure the comm structure belongs to
+ For a group comm structure this GA is the first one within the group. */
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
- /* branch to "disable comm" function according to group type */
- switch (pgroup.grouptype)
+ if (comm->commflag == PUGH_NOCOMM)
{
- case GROUP_SCALAR :
- rc = 1;
- break;
-
- case GROUP_ARRAY :
- case GROUP_GF :
- for (var = first_var; var < first_var+pgroup.numvars; var++)
- for (level = 0; level < pgroup.numtimelevels; level++)
+
+#ifdef DEBUG_PUGH
+ printf (" PUGH_EnableComm: allocating comm buffer for %d vars starting "
+ "with %d '%s'\n", comm->n_vars, comm->first_var, GA->name);
+ fflush (stdout);
+#endif
+
+ /* allocate memory for communication buffers: 2 faces per direction */
+ for (i = 0; i < 2 * GA->extras->dim; i++)
+ {
+ if (GA->connectivity->neighbours[pughGH->myproc][i] >= 0)
+ {
+ dir = i/2;
+
+ /* no ghostzones -> no comm buffers */
+ sz = comm->n_vars * GA->extras->nghostzones[dir];
+ if (sz > 0)
{
- PUGH_GAComm ((pGA *)(pughGH->variables[var][level]), PUGH_NOCOMM);
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (idir != dir)
+ {
+ sz *= GA->extras->lnsize[idir];
+ }
+ }
+ comm->buffer_sz[i] = sz;
+ comm->send_buffer[i] = malloc(sz * GA->varsize);
+ comm->recv_buffer[i] = malloc(sz * GA->varsize);
+
+ if (! (comm->send_buffer[i] && comm->recv_buffer[i]))
+ {
+ for (; i >=0 ; i--)
+ {
+ if (comm->send_buffer[i])
+ {
+ free(comm->send_buffer[i]);
+ }
+ if (comm->recv_buffer[i])
+ {
+ free(comm->recv_buffer[i]);
+ }
+ comm->buffer_sz[i] = 0;
+ }
+
+ CCTK_WARN(1, "Out of memory !");
+ retval = -1;
+ break;
+ }
}
- rc = 1;
- break;
-
- default :
- CCTK_WARN (1, "Unknown group type in PUGH_DisableGroupComm");
- rc = 0;
- break;
+ else
+ {
+ comm->buffer_sz[i] = 0;
+ comm->send_buffer[i] = NULL;
+ comm->recv_buffer[i] = NULL;
+ }
+ }
+ else
+ {
+ comm->buffer_sz[i] = 0;
+ comm->send_buffer[i] = NULL;
+ comm->recv_buffer[i] = NULL;
+ }
+ }
}
- return (rc);
-}
+ /* set up comm flags for each face */
+ if (retval >= 0)
+ {
+ /* Copy commflag */
+ comm->commflag = commflag;
+ /* First set all communcation off */
+ for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+ comm->docomm[idir] = 0;
-int PUGH_Barrier(cGH *GH)
-{
-#ifdef CCTK_MPI
- pGH *pughGH;
+ if (commflag == PUGH_ALLCOMM)
+ {
+ for (idir = 0; idir < 2 * GA->extras->dim; idir++)
+ {
+ comm->docomm[idir] = 1;
+ }
+ }
+ else if (commflag == PUGH_PLUSFACESCOMM)
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ comm->docomm[2*idir+1] = 1;
+ }
+ }
+ else if (commflag == PUGH_MINUSFACESCOMM)
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ comm->docomm[2*idir] = 1;
+ }
+ }
+ else
+ {
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (commflag == PUGH_COMM(idir))
+ {
+ comm->docomm[2*idir] = 1;
+ comm->docomm[2*idir+1] = 1;
+ }
+ }
+ }
- pughGH = PUGH_pGH(GH);
+ /* FIXME Add back the check that you have a valid COMM model: Gab */
- CACTUS_MPI_ERROR (MPI_Barrier (pughGH->PUGH_COMM_WORLD));
-#endif
+ /* Handle nsize = 1 type cases. This is only important for one
+ processor MPI periodic boundaries */
- return 0;
-}
+ for (idir = 0; idir < GA->extras->dim; idir++)
+ {
+ if (GA->extras->nsize[idir] == 1)
+ {
+ comm->docomm[2*idir] = 0;
+ comm->docomm[2*idir+1] = 0;
+ }
+ }
+ }
+#endif /* CCTK_MPI */
-/*
-int PUGH_Reduce(cGH *GH,
- const char *operation,
- int n_infields,
- int n_outfields,
- int out_type,
- void **outarray,
- ...)
-{
- return 0;
+ return retval;
}
-*/
-int PUGH_Interp(cGH *GH,
- const char *operation,
- int n_coords,
- int n_infields,
- int n_outfields,
- int n_points,
- int type,
- ...)
+ /*@@
+ @routine PUGH_DisableComm
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ This frees the communication buffers
+ of a given comm structure.
+ @enddesc
+ @history
+ Separated from routine PUGH_DisableGArrayDataStorage()
+ @endhistory
+@@*/
+static int PUGH_DisableComm(pGH *pughGH,
+ pComm *comm)
{
- return 0;
-}
+#ifdef CCTK_MPI
+ int i; /* looper */
+ pGA *GA; /* GA structure the comm structure belongs to */
-int PUGH_ParallelInit(cGH *GH)
-{
- return 0;
-}
+ /* get the GA to which the comm structure belongs to
+ For a comm structure this is the first variable within this group. */
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
-int PUGH_Exit(int retval, cGH *GH)
-{
-#ifdef CCTK_MPI
- CACTUS_MPI_ERROR(MPI_Finalize());
+#ifdef DEBUG_PUGH
+ printf (" PUGH_DisableComm: freeing comm buffer for group of %d vars and "
+ "first var '%s'\n", comm->n_vars, GA->name);
+ fflush (stdout);
#endif
- exit(retval);
- return(retval);
-}
-
-int PUGH_Abort(cGH *GH)
-{
- assert(0);
- exit(0);
- return(0);
-}
-
-
-/*****************************************************************************/
-/* local functions */
-/*****************************************************************************/
-
+ /* free memory for communication buffers: 2 faces per direction */
+ for (i = 0; i < 2 * GA->extras->dim; i++)
+ {
+ if (comm->send_buffer[i])
+ {
+ if (comm->sreq[i] != MPI_REQUEST_NULL)
+ {
+ CACTUS_MPI_ERROR(MPI_Request_free(&comm->sreq[i]));
+ }
+ free(comm->send_buffer[i]);
+ comm->send_buffer[i] = NULL;
+ }
-int PUGH_SyncGroupGA (cGH *GH,
- int group,
- int vtype,
- int n_vars,
- int timelevel)
-{
+ if (comm->recv_buffer[i])
+ {
+ free(comm->recv_buffer[i]);
+ comm->recv_buffer[i] = NULL;
+ }
- pGH *pughGH;
+ comm->buffer_sz[i] = 0;
+ }
- pughGH = PUGH_pGH(GH);
+ comm->commflag = PUGH_NOCOMM;
+#endif /* CCTK_MPI */
- return PUGH_SyncGAs(pughGH,
- CCTK_GroupDimI(group),
- CCTK_FirstVarIndexI(group),
- n_vars,
- timelevel);
+ return (1);
}
-int PUGH_SyncGAs(pGH *pughGH,
- int dim,
- int first_var,
- int n_vars,
- int timelevel)
+ /*@@
+ @routine PUGH_Sync
+ @date Mon Jun 05 2000
+ @author Thomas Radke
+ @desc
+ Finally synchronizes a variable or group of variables
+ according to a given comm structure.
+ @enddesc
+ @history
+ @endhistory
+@@*/
+static int PUGH_Sync(pGH *pughGH,
+ pComm *comm)
{
#ifdef CCTK_MPI
- int Dir;
+ int dir;
pGA *GA;
-#ifdef COMM_TIMING
- double t1,t2;
-#endif
- MPI_Request *sr = NULL;
MPI_Status mss;
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ int i;
+ MPI_Request *sr;
+#endif
+#ifdef COMM_TIMING
+ double t1, t2;
#endif
- /* start the timer for communication time */
+
#if 0
- CactusStartTimer (&pughGH->comm_time);
+ /* start the timer for communication time */
+ CCTK_TimerStartI(pughGH->comm_time);
#endif
-#ifdef CCTK_MPI
+#ifdef PUGH_WITH_DERIVED_DATATYPES
if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
- int i;
/* 2 faces, send and receive is the 2 * 2 */
- sr = (MPI_Request *)malloc(n_vars * 2 * 2 * sizeof(MPI_Request));
- for (i=0;i<n_vars * 2 * 2; i++)
- sr[i] = MPI_REQUEST_NULL;
+ sr = (MPI_Request *) malloc(comm->n_vars * 2 * 2 * sizeof(MPI_Request));
}
+#endif
+
+ GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
+
+#ifdef DEBUG_PUGH
+ printf (" PUGH_Sync: syncing group of %d vars with first var '%s'\n",
+ comm->n_vars, GA->name);
+ fflush (stdout);
+#endif
- for (Dir = 0; Dir < dim; Dir ++)
+ for (dir = 0; dir < GA->extras->dim; dir ++)
{
- int i;
#ifdef COMM_TIMING
t1 = MPI_Wtime();
#endif
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- PostReceiveGA(pughGH, GA, 2*Dir);
- PostReceiveGA(pughGH, GA, 2*Dir+1);
- }
+ PostReceiveGA(pughGH, 2*dir, comm);
+ PostReceiveGA(pughGH, 2*dir+1, comm);
#ifdef COMM_TIMING
t2 = MPI_Wtime();
- printf ("PR : %lf\n",t2-t1);
+ printf("PR : %f\n",t2-t1);
#endif
- if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
+
+ /* For allocated buffers: wait for the last send.
+ Since these will be null if they
+ are not used, this is always safe.
+ */
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- /* Wait for the last send. Since these will be null if they
- are not used, this is always safe.
- */
- MPI_Wait(&(GA->sreq[2*Dir]),&mss);
- PostSendGA(pughGH, GA, 2*Dir);
- MPI_Wait(&(GA->sreq[2*Dir+1]),&mss);
- PostSendGA(pughGH, GA, 2*Dir+1);
- }
- }
- else
+ MPI_Wait(&comm->sreq[2*dir], &mss);
+ }
+ PostSendGA(pughGH, 2*dir, comm);
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- PostSendGA(pughGH, GA, 2*Dir);
- PostSendGA(pughGH, GA, 2*Dir+1);
- }
+ MPI_Wait(&comm->sreq[2*dir+1], &mss);
}
+ PostSendGA(pughGH, 2*dir+1, comm);
#ifdef COMM_TIMING
t1 = MPI_Wtime();
- printf ("PS : %lf\n",t1-t2);
+ printf("PS : %f\n",t1-t2);
#endif
/* Now comes the big difference between derived types and
@@ -878,40 +709,37 @@ int PUGH_SyncGAs(pGH *pughGH,
if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
/* Do a wait any on the receives */
- for (i = first_var; i < first_var + n_vars; i++)
- {
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- MPI_Wait(&(GA->rreq[2*Dir]),&mss);
- FinishReceiveGA(pughGH, GA, 2*Dir);
- MPI_Wait(&(GA->rreq[2*Dir+1]),&mss);
- FinishReceiveGA(pughGH, GA, 2*Dir+1);
- }
- }
-
- if (pughGH->commmodel == PUGH_DERIVEDTYPES)
+ MPI_Wait(&comm->rreq[2*dir], &mss);
+ FinishReceiveGA(pughGH, 2*dir, comm);
+ MPI_Wait(&comm->rreq[2*dir+1], &mss);
+ FinishReceiveGA(pughGH, 2*dir+1, comm);
+ }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
/* Load up the thing for the waitall */
- for (i=0;i<n_vars;i++)
+ for (i = 0; i < comm->n_vars; i++)
{
- int id = i*4;
- GA = (pGA *) (pughGH->variables [i][timelevel]);
- if (GA->docomm[2*Dir] &&
+ int id = i * 2 * 2;
+ pGA *GA = (pGA *) pughGH->variables [i][comm->sync_timelevel];
+
+ if (GA->comm->docomm[2*dir] &&
GA->storage)
{
- sr[id] = GA->sreq[2*Dir];
- sr[id+1] = GA->rreq[2*Dir];
+ sr[id] = GA->comm->sreq[2*dir];
+ sr[id+1] = GA->comm->rreq[2*dir];
}
else
{
sr[id] = MPI_REQUEST_NULL;
sr[id+1] = MPI_REQUEST_NULL;
}
-
- if (GA->docomm[2*Dir+1] &&
+
+ if (GA->comm->docomm[2*dir+1] &&
GA->storage)
{
- sr[id+2] = GA->sreq[2*Dir+1];
- sr[id+3] = GA->rreq[2*Dir+1];
+ sr[id+2] = GA->comm->sreq[2*dir+1];
+ sr[id+3] = GA->comm->rreq[2*dir+1];
}
else
{
@@ -920,144 +748,26 @@ int PUGH_SyncGAs(pGH *pughGH,
}
}
/* Now do a waitall */
- MPI_Waitall(4*n_vars,sr,&mss);
-
+ MPI_Waitall(4*comm->n_vars, sr, &mss);
}
+#endif
- if (sr)
- free(sr);
#ifdef COMM_TIMING
t2 = MPI_Wtime();
- printf ("FR : %lf\n",t2-t1);
+ printf("FR : %f\n",t2-t1);
#endif
}
-#endif /* MPI */
-
- /* get the time spent in communication */
-
-#if 0
- CactusStopTimer (&pughGH->comm_time);
-#endif
-
- return (1);
-}
-
-
-int PUGH_EnableArrayGroupStorage(cGH *GH,
- int group,
- int vtype,
- int dim,
- int n_variables,
- int n_timelevels)
-{
- int nstorage; /* Number of Arrays for which storage was set */
- int nnostorage; /* Number of Arrays for which no storage was set */
- int retval;
- int first_var;
- int var;
- pGH *mypGH;
- pGA *GA;
- int level;
- /* FIXME: This should come from a parameter */
- int zero_memory=1;
- int padding_active=0;
- int padding_cacheline_bits=0;
- int padding_size=0;
- int padding_address_spacing=0;
-
- nstorage = 0;
- nnostorage = 0;
-
- mypGH = PUGH_pGH(GH);
- first_var = CCTK_FirstVarIndexI(group);
-
- for(var = first_var; var < first_var+n_variables; var++)
- {
- for(level = 0; level < n_timelevels; level++)
- {
- GA = (pGA *)(mypGH->variables[var][level]);
-
- if(!GA->storage)
- {
-#ifdef DEBUG_PUGH
- {
- const char *name;
- name = CCTK_FullName(var);
- printf("Storage on in PUGH_EnableArrayGroupStorage for %s\n",name);
- }
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ free(sr);
#endif
- PUGH_EnablePGAStorage(GA,
- mypGH->myproc,
- zero_memory,
- padding_active,
- padding_cacheline_bits,
- padding_size,
- padding_address_spacing);
-
- GH->data[var][level] = GA->data;
-
- nnostorage++;
- }
- else
- {
- GH->data[var][level] = GA->data;
- nstorage++;
- }
- }
- }
- if (nstorage > 0 && nnostorage > 0)
- {
- CCTK_WARN(0,"Group storage violation in PUGH_EnableArrayGroupStorage");
- retval = -1;
- }
- else if (nstorage > 0)
- {
- retval = 1;
- }
- else if (nnostorage > 0)
- {
- retval = 0;
- }
- else
- {
- CCTK_WARN(0,"Problem in PUGH_EnableArrayGroupStorage");
- retval = -1;
- }
-
- return retval;
-
-}
-
-
-int PUGH_MyProc(cGH *GH)
-{
- int myproc;
-
-#ifdef CCTK_MPI
- pGH *pughGH;
- pughGH = PUGH_pGH(GH);
- CACTUS_MPI_ERROR(MPI_Comm_rank(pughGH->PUGH_COMM_WORLD, &myproc));
-#else
- myproc=0;
+#if 0
+ /* get the time spent in communication */
+ CCTK_TimerStopI(pughGH->comm_time);
#endif
- return myproc;
-}
-
-
-int PUGH_nProcs(cGH *GH)
-{
- int nprocs;
-
-#ifdef CCTK_MPI
- pGH *pughGH;
- pughGH = PUGH_pGH(GH);
- CACTUS_MPI_ERROR(MPI_Comm_size(pughGH->PUGH_COMM_WORLD, &nprocs));
-#else
- nprocs=1;
-#endif
+#endif /* CCTK_MPI */
- return nprocs;
+ return (0);
}
diff --git a/src/Evolve.c b/src/Evolve.c
index 3a1e9d9..e5b100b 100644
--- a/src/Evolve.c
+++ b/src/Evolve.c
@@ -49,8 +49,6 @@ static int DoneMainLoop (CCTK_REAL cctk_time, int iteration);
static int StepGH(cGH *GH);
static void RotateTimeLevelsGH(cGH *cgh);
-extern int pugh_GHExtension;
-
/* the iteration counter used in the evolution loop */
static int iteration = 0;
diff --git a/src/FinishReceiveGA.c b/src/FinishReceiveGA.c
index d1689da..df4154c 100644
--- a/src/FinishReceiveGA.c
+++ b/src/FinishReceiveGA.c
@@ -3,9 +3,8 @@
@date Wed Feb 2 11:37:28 1997
@author Gabrielle Allen
@desc
- The routine which finalize the MPI recieves for a grid
- function. Critically linked with @seefile pGA_PostRecv.c
- and @seefile pGA_PostSend.c
+ The routine which finalize the MPI recieves for a communication.
+ Critically linked with @seefile PostRecvGA.c and @seefile PostSendGA.c
@enddesc
@version $Header$
@@*/
@@ -17,221 +16,129 @@
#include "cctk.h"
#include "pugh.h"
-void FinishReceiveGA3(pGH *GH, pGA *GA, int dir);
-void FinishReceiveGA1(pGH *GH, pGA *GA, int dir);
-
static char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_FinishReceiveGA_c)
+
+#ifdef CCTK_MPI
+
/*@@
@routine FinishReceiveGA
@date Thu Apr 3 11:38:07 1997
@author Paul Walker
@desc
This routine finalizes the MPI communication through a face.
- It is crucially linked with @seeroutine pGA_PostRecv and
- @seeroutine pGA_PostSend.
+ It is crucially linked with @seeroutine PostRecvGA and
+ @seeroutine PostSendGA.
<p>
<b>Important!</b>
This routine does <b>not</b> wait on the recieves.
Before it is called, you must do the MPI_Wait or
else you will not get the right answer. for an
- example of this, see @seeroutine SyncGroupGA or
- @seeroutine SyncGroupGA
+ example of this, see @seeroutine PUGH_DisableArrayGroupComm or
+ @seeroutine PUGH_Sync
@enddesc
- @calledby SyncGroupGA
+ @calledby PUGH_Sync
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GAs with storage
@endhistory
@@*/
-void FinishReceiveGA(pGH *GH, pGA *GA, int dir)
+void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
-#ifdef CCTK_MPI
+ pGA *GA;
+ int i, dim, copy_bytes, offset;
+ char *copy_from, *copy_to;
+ int *istart, *iend, *iterator;
+
+
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
/* Return if GA has no storage */
- if (!(GA->storage))
+ if (! GA->storage)
{
return;
}
/* Return if communication not required and no forced synchronisation */
- if (!(GH->forceSync || GA->docomm[dir]))
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] >= 0)
+ if (GA->connectivity->neighbours[pughGH->myproc][dir] < 0)
{
+ return;
+ }
- /* Here wait one at a time, so the others can arrive while
- we copy the data back in... */
+ /* Here wait one at a time, so the others can arrive while
+ we copy the data back in... */
#ifdef DEBUG_PUGH
- printf ("FINISHRECV: GA %s Side %d Proc %d request %d\n",
- GA->name,dir,GH->myproc,GA->rreq[dir]);
+ printf ("FinishReceiveGA: into direction %d from proc %d for %d vars "
+ "starting with '%s'\n",
+ dir, GA->connectivity->neighbours[pughGH->myproc][dir],
+ comm->n_vars, GA->name);
#endif
- if (GA->extras->dim == 3)
- {
- FinishReceiveGA3(GH,GA,dir);
- }
- else if (GA->extras->dim == 1)
- {
- FinishReceiveGA1(GH,GA,dir);
- }
- else
- {
- CCTK_WARN(0,"PUGH does not support this dimension grid");
- }
+ istart = GA->extras->ghosts[GA->stagger][0][dir];
+ iend = GA->extras->ghosts[GA->stagger][1][dir];
+ iterator = GA->extras->iterator;
+ /* set iterator to the start vector */
+ for(dim = 0; dim < GA->extras->dim; dim++)
+ {
+ iterator[dim] = istart[dim];
}
-#endif
-}
+ /* get the source and the number of bytes to copy in the lowest dimension */
+ copy_from = comm->recv_buffer[dir];
+ copy_bytes = (iend[0] - istart[0]) * GA->varsize;
-void FinishReceiveGA3(pGH *GH,pGA *GA,int dir)
-{
- int istart,iend,jstart,jend,kstart,kend;
- int ii,jj,kk,xx;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy buffer onto GA Storage */
- istart = GA->extras->ghosts[GA->stagger][0][dir][0];
- iend = GA->extras->ghosts[GA->stagger][1][dir][0];
- jstart = GA->extras->ghosts[GA->stagger][0][dir][1];
- jend = GA->extras->ghosts[GA->stagger][1][dir][1];
- kstart = GA->extras->ghosts[GA->stagger][0][dir][2];
- kend = GA->extras->ghosts[GA->stagger][1][dir][2];
-
- /* Great now copy. Note ordering is just link in PostSend */
- xx=0;
- switch (GA->vtype)
+ /* now do the nested loops starting with the innermost */
+ dim = 1;
+ while (1)
{
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- char_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_CHAR *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->data;
- for (kk=kstart; kk<kend; kk++)
+ /* check for end of current loop */
+ if (GA->extras->dim > 1 && iterator[dim] >= iend[dim])
+ {
+ /* increment outermost loopers */
+ for (dim++; dim < GA->extras->dim; dim++)
{
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- int_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_INT *) GA->recv_buffer[dir]) [xx++];
- }
- }
+ iterator[dim]++;
+ if (iterator[dim] < iend[dim])
+ break;
}
- break;
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- real_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_REAL *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->data;
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [DATINDEX (GA->extras,ii,jj,kk)] =
- ((CCTK_COMPLEX *) GA->recv_buffer[dir]) [xx++];
- }
- }
- }
- break;
+ /* done if beyond outermost loop */
+ if (dim >= GA->extras->dim)
+ break;
- default:
- CCTK_WARN (1, "Unsupported variable type in FinishReceiveGA3");
- return;
- }
-}
+ /* reset innermost loopers */
+ for (dim--; dim > 0; dim--)
+ iterator[dim] = istart[dim];
+ dim = 1;
+ }
+ /* get the linear offset */
+ offset = istart[0] * GA->varsize;
+ for (i = 1; i < GA->extras->dim; i++)
+ offset += iterator[i] * GA->extras->hyper_volume[i];
+ /* copy the data in dimension 0 */
+ for (i = comm->first_var; i < comm->first_var + comm->n_vars; i++)
+ {
+ copy_to = ((pGA *) pughGH->variables[i][comm->sync_timelevel])->data;
+ memcpy (copy_to + offset, copy_from, copy_bytes);
+ copy_from += copy_bytes;
+ }
-void FinishReceiveGA1(pGH *GH,pGA *GA,int dir)
-{
- int istart,iend;
- int ii,xx;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy buffer onto GA Storage */
- istart = GA->extras->ghosts[GA->stagger][0][dir][0];
- iend = GA->extras->ghosts[GA->stagger][1][dir][0];
-
- /* Great now copy. Note ordering is just link in PostSend */
- xx=0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- char_data [ii] = ((CCTK_CHAR *)GA->recv_buffer[dir])[xx++];
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- int_data [ii] = ((CCTK_INT *)GA->recv_buffer[dir])[xx++];
- }
- break;
+ /* increment current looper */
+ iterator[dim]++;
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- real_data [ii] = ((CCTK_REAL *)GA->recv_buffer[dir])[xx++];
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->data;
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [ii] = ((CCTK_COMPLEX *)GA->recv_buffer[dir])[xx++];
- }
- break;
+ } /* end of nested loops over all dimensions */
- default:
- CCTK_WARN (1, "Unsupported variable type in FinishReceiveGA3");
- return;
- }
}
+#endif /* CCTK_MPI */
diff --git a/src/GHExtension.c b/src/GHExtension.c
index 7d9bfcc..23627ff 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -80,13 +80,8 @@ void *PUGH_SetupGH(tFleshConfig *config,
CCTK_WARN(0,"Failed to allocate memory for a pGH");
}
- /* Set the identity for this pGH. */
-
- /* FIXME : should be a dynamically allocated string.
- * If you change this, please change the line in pugh.h.
- */
-
- strcpy(newGH->identity_string, "");
+ /* Set the identity for this pGH. For now this is an empty string. */
+ newGH->identity_string = (char *) calloc (1, sizeof (char *));
/* Set up groups on the GH */
@@ -186,7 +181,7 @@ static void PUGH_InitGHBasics (cGH *GH)
pGH *mypGH;
pGExtras *GFExtras;
- mypGH = (pGH *)GH->extensions[pugh_GHExtension];
+ mypGH = PUGH_pGH(GH);
GFExtras = mypGH->GFExtras[GH->cctk_dim-1];
GH->cctk_convlevel = 0;
@@ -207,7 +202,7 @@ static void PUGH_InitGHBasics (cGH *GH)
{
GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)] = GH->cctk_lsh[idir];
if ((GH->cctk_bbox[2*idir+1]==1) && (istag>0))
- GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)]--;
+ GH->cctk_lssh[CCTK_LSSH_IDX(istag,idir)]--;
}
}
}
@@ -236,7 +231,7 @@ static void PUGH_InitGHVariables (cGH *GH)
pGH *mypGH;
- mypGH = (pGH *)GH->extensions[pugh_GHExtension];
+ mypGH = PUGH_pGH(GH);
for(var = 0; var < mypGH->nvariables; var++)
{
diff --git a/src/LoadAware.c b/src/LoadAware.c
index 94c2d17..4a5ce80 100644
--- a/src/LoadAware.c
+++ b/src/LoadAware.c
@@ -9,6 +9,7 @@
@@*/
#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <ctype.h>
#include "pugh.h"
diff --git a/src/PostReceiveGA.c b/src/PostReceiveGA.c
index 3a7864b..caacff0 100644
--- a/src/PostReceiveGA.c
+++ b/src/PostReceiveGA.c
@@ -3,9 +3,9 @@
@date Wed Feb 2 11:28:06 1997
@author Gabrielle Allen
@desc
- Routines which post all the IRecv commands for a GA. These
- allow the asyncronous model proposed in, for example,
- @seeroutine SyncGA to go!
+ Routines which post all the IRecv commands for a single sync.
+ These allow the asyncronous model proposed in, for example,
+ @seeroutine PUGH_Sync to go!
@enddesc
@version $Header$
@@*/
@@ -21,6 +21,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
/*#define DEBUG_PUGH*/
+#ifdef CCTK_MPI
+
/*@@
@routine PostReceiveGA
@date Thu Apr 3 12:10:46 1997
@@ -28,8 +30,8 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
@desc
This routine posts a recieve (MPI_Irecv) of the buffer
we want to get the send from another processor, which
- will be sent by @seeroutine pGA_PostSend and then
- finalized by @seeroutoine pGA_FinishRecv.
+ will be sent by @seeroutine PostSendGA and then
+ finalized by @seeroutoine FinishRecvGA.
<p>
Aside from a silly calculation to get a unique tag
based on neigbors and processors, this is a very
@@ -41,83 +43,95 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c)
@endhistory
@@*/
-void PostReceiveGA(pGH *GH, pGA *GA, int dir)
+void PostReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
-#ifdef CCTK_MPI
+ pGA *GA;
int rtag;
- MPI_Datatype mpi_type;
- MPI_Datatype *recv_dt;
+ int neighbour;
- if (!(GA->storage))
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
+
+ /* return if no storage assigned */
+ if (! GA->storage)
{
CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
"Trying to synchronize variable '%s' with no storage", GA->name);
return;
}
- /* Return if communication not required and no forced synchronisation */
- if (!(GH->forceSync || GA->docomm[dir]))
+ /* return if communication not required and no forced synchronisation */
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] < 0)
+ /* return if there is no neighbour in the given direction */
+ neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
+ if (neighbour < 0)
{
return;
}
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- mpi_type = PUGH_MPI_CHAR;
- recv_dt = GH->recv_char_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_INT:
- mpi_type = PUGH_MPI_INT;
- recv_dt = GH->recv_int_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_REAL:
- mpi_type = PUGH_MPI_REAL;
- recv_dt = GH->recv_real_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- mpi_type = GH->PUGH_mpi_complex;
- recv_dt = GH->recv_complex_dt [GA->stagger];
- break;
-
- default:
- CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
- return;
- }
-
- rtag = 1000 + dir + 2 * (GH->myproc + GH->nprocs * GA->id);
+ /* note this is the complement of the stag set in PostSendGA */
+ rtag = 1000 + dir + 2 * (pughGH->myproc + pughGH->nprocs * GA->id);
/* mod the tag to force MPI compliance */
rtag = rtag % 32768;
+
#ifdef DEBUG_PUGH
- printf ("RECV GA %s Side %d Proc %d rtag %d size %d from proc %d\n",
- GA->name,
- dir,
- GH->myproc,
- rtag,
- GA->buffer_sz[dir],
- GA->connectivity->neighbours[GH->myproc][dir]);
+ printf ("PostReceiveGA: into direction %d from proc %d with rtag %d size %d "
+ "for %d vars starting from '%s'\n",
+ dir, neighbour, rtag, comm->buffer_sz[dir], comm->n_vars, GA->name);
#endif
- if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
+
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- CACTUS_MPI_ERROR (MPI_Irecv (GA->recv_buffer [dir],
- GA->buffer_sz [dir], mpi_type,
- GA->connectivity->neighbours [GH->myproc][dir], rtag,
- GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+ CACTUS_MPI_ERROR (MPI_Irecv (comm->recv_buffer[dir],
+ comm->buffer_sz[dir], comm->mpi_type,
+ neighbour, rtag,
+ pughGH->PUGH_COMM_WORLD, &comm->rreq[dir]));
}
- if (GH->commmodel == PUGH_DERIVEDTYPES)
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
- CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
- 1, recv_dt [dir],
- GA->connectivity->neighbours [GH->myproc][dir], rtag,
- GH->PUGH_COMM_WORLD, &(GA->rreq [dir])));
+ int var;
+ MPI_Datatype *recv_dt;
+
+
+ switch (GA->vtype)
+ {
+ case CCTK_VARIABLE_CHAR:
+ recv_dt = pughGH->recv_char_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_INT:
+ recv_dt = pughGH->recv_int_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_REAL:
+ recv_dt = pughGH->recv_real_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_COMPLEX:
+ recv_dt = pughGH->recv_complex_dt[GA->stagger];
+ break;
+
+ default:
+ CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
+ return;
+ }
+
+ for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
+ {
+ pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
+
+
+ CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
+ 1, recv_dt[dir],
+ neighbour, (rtag + var) % 32768,
+ pughGH->PUGH_COMM_WORLD,
+ &GA->comm->rreq[dir]));
+ }
}
#endif
}
+#endif /* CCTK_MPI */
diff --git a/src/PostSendGA.c b/src/PostSendGA.c
index 1ef6640..ccaff91 100644
--- a/src/PostSendGA.c
+++ b/src/PostSendGA.c
@@ -20,11 +20,12 @@ static char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_PostSendGA_c)
-void PostSendGA1(pGH *GH, pGA *GA, int dir);
-void PostSendGA3(pGH *GH, pGA *GA, int dir);
-
/*#define DEBUG_PUGH*/
+
+#ifdef CCTK_MPI
+
+
/*@@
@routine PostSendGA
@date Thu Apr 3 11:58:59 1997
@@ -32,277 +33,183 @@ void PostSendGA3(pGH *GH, pGA *GA, int dir);
@desc
This routine posts an asyncronous MPI send of a buffer
for a face (MPI_Isend). It does this by first packing up
- a send buffer (allocated in @seeroutine SetupPGF) then
+ a send buffer (allocated in @seeroutine PUGH_SetupGroupComm) then
sending it out on the pipe.
<p>
Since this is an asynchronous communications model we
assume that a recieve has been posted for the send.
thus the correct calling order for this is as in
- @seeroutine SyncGF, eg, after a Recv.
+ @seeroutine PUGH_Sync, eg, after a Recv.
<p>
Note this does <b>not</b> wait on the send buffer from previous
communications. It is the users responsibility to wait on
that buffer.
@enddesc
- @calledby SyncGroupGF
+ @calledby PUGH_Sync
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GFs with storage
@endhistory
@@*/
-void PostSendGA(pGH *GH, pGA *GA, int dir)
+void PostSendGA(pGH *pughGH, int dir, pComm *comm)
{
-
-#ifdef CCTK_MPI
+ pGA *GA;
int stag, dircomp;
- double ts, tw, tp;
- MPI_Datatype mpi_type;
- MPI_Datatype *send_dt;
+ int neighbour;
+ int i, dim, copy_bytes, offset;
+ char *copy_from, *copy_to;
+ int *istart, *iend, *iterator;
- if (!(GA->storage))
- {
- return;
- }
-
- if (!(GH->forceSync || GA->docomm[dir]))
+
+ GA = (pGA *) pughGH->variables[comm->first_var][0];
+
+ /* return if no storage assigned */
+ if (! GA->storage)
{
return;
}
- if (GA->connectivity->neighbours[GH->myproc][dir] < 0)
+ /* return if communication not required and no forced synchronisation */
+ if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
- switch (GA->vtype)
+ /* return if there is no neighbour in the given direction */
+ neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
+ if (neighbour < 0)
{
-
- case CCTK_VARIABLE_CHAR:
- mpi_type = PUGH_MPI_CHAR;
- send_dt = GH->send_char_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_INT:
- mpi_type = PUGH_MPI_INT;
- send_dt = GH->send_int_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_REAL:
- mpi_type = PUGH_MPI_REAL;
- send_dt = GH->send_real_dt [GA->stagger];
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- mpi_type = GH->PUGH_mpi_complex;
- send_dt = GH->send_complex_dt [GA->stagger];
- break;
-
- default:
- CCTK_WARN (1, "Unknown variable type in PostSendGA");
- return;
-
+ return;
}
- ts = MPI_Wtime();
dircomp = dir+1; /* Complementary direction */
- if (dircomp %2 == 0)
+ if (dircomp % 2 == 0)
{
dircomp = dir-1;
}
- /* Note this is the complement of the rtag set in PostReceive */
- stag = 1000 + dircomp + 2 *
- (GA->connectivity->neighbours[GH->myproc][dir] +
- GH->nprocs * GA->id);
-
+ /* note this is the complement of the rtag set in PostReceiveGA */
+ stag = 1000 + dircomp + 2 * (neighbour + pughGH->nprocs * GA->id);
/* mod the tag to force MPI compliance */
stag = stag % 32768;
#ifdef DEBUG_PUGH
- printf ("SEND: GA %s Side %d Proc %d stag %d Size %d to %d\n",
- GA->name,
- dir,
- GH->myproc,
- stag,
- GA->buffer_sz[dir],
- GA->connectivity->neighbours[GH->myproc][dir]);
+ printf ("PostSendGA: into direction %d to proc %d with stag %d size %d for "
+ "%d vars staring with '%s'\n",
+ dir, neighbour, stag, comm->buffer_sz[dir], comm->n_vars, GA->name);
#endif
- if (GH->commmodel == PUGH_DERIVEDTYPES)
+ if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
- CACTUS_MPI_ERROR (MPI_Isend (GA->data,
- 1, send_dt [dir],
- GA->connectivity->neighbours [GH->myproc][dir], stag,
- GH->PUGH_COMM_WORLD, &(GA->sreq [dir])));
- }
- if (GH->commmodel == PUGH_ALLOCATEDBUFFERS)
- {
- if (GA->connectivity->dim == 3)
- {
- PostSendGA3(GH,GA,dir);
- }
- else if (GA->connectivity->dim == 1)
- {
- PostSendGA1(GH,GA,dir);
- }
- else
+ istart = GA->extras->overlap[GA->stagger][0][dir];
+ iend = GA->extras->overlap[GA->stagger][1][dir];
+ iterator = GA->extras->iterator;
+
+ /* set iterator to the start vector */
+ for(dim = 0; dim < GA->extras->dim; dim++)
{
- CCTK_WARN(0,"PUGH does not yet support this dimension");
+ iterator[dim] = istart[dim];
}
- tp = MPI_Wtime();
-
- /* post send */
- CACTUS_MPI_ERROR (MPI_Isend (GA->send_buffer [dir],
- GA->buffer_sz [dir], mpi_type,
- GA->connectivity->neighbours [GH->myproc][dir], stag,
- GH->PUGH_COMM_WORLD, &(GA->sreq [dir])));
- tw = MPI_Wtime();
-#ifdef COMM_TIMING
- printf ("GA %s Dir %d Time %f\n", GA->name, dir, tw-ts);
-#endif
- }
-#endif
-}
+ /* get the source and the number of bytes to copy in the lowest dimension */
+ copy_to = comm->send_buffer[dir];
+ copy_bytes = (iend[0] - istart[0]) * GA->varsize;
-void PostSendGA3(pGH *GH, pGA *GA, int dir)
-{
-
- int ii,jj,kk,xx;
- int istart, iend;
- int jstart, jend;
- int kstart, kend;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy my information into the send buffer */
- istart = GA->extras->overlap[GA->stagger][0][dir][0];
- iend = GA->extras->overlap[GA->stagger][1][dir][0];
- jstart = GA->extras->overlap[GA->stagger][0][dir][1];
- jend = GA->extras->overlap[GA->stagger][1][dir][1];
- kstart = GA->extras->overlap[GA->stagger][0][dir][2];
- kend = GA->extras->overlap[GA->stagger][1][dir][2];
-
- /* Great now copy */
- xx = 0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- char_data [xx++] =
- ((CCTK_CHAR *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
+ /* now do the nested loops starting with the innermost */
+ dim = 1;
+ while (1)
+ {
+ /* check for end of current loop */
+ if (GA->extras->dim > 1 && iterator[dim] >= iend[dim])
{
- for (jj=jstart; jj<jend; jj++)
+ /* increment outermost loopers */
+ for (dim++; dim < GA->extras->dim; dim++)
{
- for (ii=istart; ii<iend; ii++)
- {
- int_data [xx++] =
- ((CCTK_INT *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
- }
- break;
-
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
- {
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- real_data [xx++] =
- ((CCTK_REAL *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
+ iterator[dim]++;
+ if (iterator[dim] < iend[dim])
+ break;
}
+
+ /* done if beyond outermost loop */
+ if (dim >= GA->extras->dim)
+ break;
+
+ /* reset innermost loopers */
+ for (dim--; dim > 0; dim--)
+ iterator[dim] = istart[dim];
+ dim = 1;
}
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->send_buffer [dir];
- for (kk=kstart; kk<kend; kk++)
+
+ /* get the linear offset */
+ offset = istart[0] * GA->varsize;
+ for (i = 1; i < GA->extras->dim; i++)
+ offset += iterator[i] * GA->extras->hyper_volume[i];
+
+ /* copy the data in dimension 0 */
+ for (i = comm->first_var; i < comm->first_var + comm->n_vars; i++)
{
- for (jj=jstart; jj<jend; jj++)
- {
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [xx++] =
- ((CCTK_COMPLEX *) GA->data) [DATINDEX (GA->extras,ii,jj,kk)];
- }
- }
+ copy_from = ((pGA *) pughGH->variables[i][comm->sync_timelevel])->data;
+ memcpy (copy_to, copy_from + offset, copy_bytes);
+ copy_to += copy_bytes;
}
- break;
+
+ /* increment current looper */
+ iterator[dim]++;
+
+ } /* end of nested loops over all dimensions */
+
+ /* now post send */
+ CACTUS_MPI_ERROR (MPI_Isend (comm->send_buffer[dir],
+ comm->buffer_sz[dir], comm->mpi_type,
+ neighbour, stag,
+ pughGH->PUGH_COMM_WORLD, &comm->sreq[dir]));
}
-}
+#ifdef PUGH_WITH_DERIVED_DATATYPES
+ else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
+ {
+ int var;
+ MPI_Datatype *send_dt;
+ switch (GA->vtype)
+ {
-void PostSendGA1(pGH *GH, pGA *GA, int dir)
-{
+ case CCTK_VARIABLE_CHAR:
+ send_dt = pughGH->send_char_dt[GA->stagger];
+ break;
- int ii,xx;
- int istart, iend;
- CCTK_CHAR *char_data;
- CCTK_INT *int_data;
- CCTK_REAL *real_data;
- CCTK_COMPLEX *complex_data;
-
- /* Copy my information into the send buffer */
- istart = GA->extras->overlap[GA->stagger][0][dir][0];
- iend = GA->extras->overlap[GA->stagger][1][dir][0];
-
- /* Great now copy */
- xx = 0;
- switch (GA->vtype)
- {
- case CCTK_VARIABLE_CHAR:
- char_data = (CCTK_CHAR *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- char_data [xx++] = ((CCTK_CHAR *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_INT:
- int_data = (CCTK_INT *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- int_data [xx++] = ((CCTK_INT *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_REAL:
- real_data = (CCTK_REAL *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- real_data [xx++] = ((CCTK_REAL *) GA->data) [ii];
- }
- break;
-
- case CCTK_VARIABLE_COMPLEX:
- complex_data = (CCTK_COMPLEX *) GA->send_buffer [dir];
- for (ii=istart; ii<iend; ii++)
- {
- complex_data [xx++] = ((CCTK_COMPLEX *) GA->data) [ii];
- }
- break;
+ case CCTK_VARIABLE_INT:
+ send_dt = pughGH->send_int_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_REAL:
+ send_dt = pughGH->send_real_dt[GA->stagger];
+ break;
+
+ case CCTK_VARIABLE_COMPLEX:
+ send_dt = pughGH->send_complex_dt[GA->stagger];
+ break;
+
+ default:
+ CCTK_WARN (1, "Unknown variable type in PostSendGA");
+ return;
+
+ }
+
+ for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
+ {
+ pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
+
+
+ CACTUS_MPI_ERROR (MPI_Isend (GA->data,
+ 1, send_dt[dir],
+ neighbour, (stag + var) % 32768,
+ pughGH->PUGH_COMM_WORLD,
+ &GA->comm->sreq[dir]));
+ }
}
+#endif
}
+#endif /* CCTK_MPI */
diff --git a/src/Reduction.c b/src/Reduction.c
index a9243ab..a91d469 100644
--- a/src/Reduction.c
+++ b/src/Reduction.c
@@ -37,9 +37,6 @@ CCTK_FILEVERSION(CactusPUGH_PUGH_Reduction_c)
#define SQR(x) ((x) * (x))
-/* index into GH->extensions[] to get the PUGH extension handle */
-extern int pugh_GHExtension;
-
/* local function prototypes */
int PUGH_Norm1_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal);
int PUGH_Norm2_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal);
@@ -142,12 +139,10 @@ void PUGH_ReduceArrayMinVal (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayMinVal: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -265,12 +260,10 @@ void PUGH_ReduceArrayMaxVal (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayMaxVal: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array "
+ "with %d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -386,12 +379,10 @@ void PUGH_ReduceArraySum (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArraySum: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -511,12 +502,10 @@ void PUGH_ReduceArrayNorm1 (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayNorm1: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -636,12 +625,10 @@ void PUGH_ReduceArrayNorm2 (cGH *GH, int proc, int nDims, int dims [],
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceArrayNorm2: don't know how to reduce a "
- "%d-dimensional array with %d elements to an output "
- "array of %d elements", nDims, nPoints, nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a %d-dimensional array with "
+ "%d elements to an output array of %d elements",
+ nDims, nPoints, nOutVals);
return;
}
nPoints = 1;
@@ -725,12 +712,12 @@ void PUGH_ReduceArrayNorm2 (cGH *GH, int proc, int nDims, int dims [],
@endvar
@@*/
void PUGH_ReduceMinVal (cGH *GH,
- int proc,
- int nOutVals,
+ int proc,
+ int nOutVals,
int outType,
- void *outVals,
- int numinvars,
- int *varlist)
+ void *outVals,
+ int numinvars,
+ int *varlist)
{
int i;
@@ -750,11 +737,9 @@ void PUGH_ReduceMinVal (cGH *GH,
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceMinVal: don't know how to reduce a GF "
- "to an output array of %d elements", nOutVals);
- CCTK_WARN (1, msg);
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a GF to an output array "
+ "of %d elements", nOutVals);
return;
}
nPoints = 1;
@@ -884,12 +869,9 @@ void PUGH_ReduceMaxVal (cGH *GH, int proc, int nOutVals,
{
if (nOutVals != nPoints)
{
- char msg [200];
-
- sprintf (msg, "PUGH_ReduceMaxVal: don't know how to reduce a GF "
- "to an output array of %d elements", nOutVals);
- CCTK_WARN (1, msg);
- return;
+ CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Don't know how to reduce a GF to an output array "
+ "of %d elements", nOutVals);
}
nPoints = 1;
}
@@ -2118,9 +2100,6 @@ int PUGH_Norm2_GF (cGH *GH, int index, int proc, CCTK_REAL *outVal)
CCTK_INT np;
CCTK_REAL local_result;
#endif
- int dim;
-
- dim = GH->cctk_dim;
timelevel = CCTK_NumTimeLevelsFromVarI (index) - 1;
if (timelevel > 0)
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index 812aae4..7bba580 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -36,13 +36,10 @@ CCTK_FILEVERSION (CactusPUGH_PUGH_SetupGroup_c)
@@*/
-int PUGH_SetupScalarGroup
- (
- pGH *newGH,
- int vtype,
- int n_variables,
- int n_timelevels
- )
+int PUGH_SetupScalarGroup(pGH *newGH,
+ int vtype,
+ int n_variables,
+ int n_timelevels)
{
int retval=PUGH_SUCCESS;
int variable;
@@ -67,7 +64,7 @@ int PUGH_SetupScalarGroup
for(level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
- (void *)malloc(var_size);
+ (void *)malloc(var_size);
}
newGH->nvariables++;
}
@@ -105,16 +102,14 @@ int PUGH_SetupScalarGroup
@@*/
int PUGH_SetupArrayGroup(pGH *newGH,
- int *nsize,
- int *ghostsize,
- int vtype,
- int dim,
- int n_variables,
- int staggercode,
- int n_timelevels)
+ int *nsize,
+ int *ghostsize,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggercode,
+ int n_timelevels)
{
-
- int i;
int variable;
int var_size = CCTK_VarTypeSize(vtype);
int retval = PUGH_SUCCESS;
@@ -122,34 +117,16 @@ int PUGH_SetupArrayGroup(pGH *newGH,
void ***temp;
pConnectivity *connectivity;
pGExtras *extras;
+ pComm *groupcomm;
int *perme;
int *nprocs;
- /* Arrays can't (yet) have periodicity */
- perme = (int *) malloc(dim*sizeof(int));
- if (perme)
- {
- for (i=0;i<dim;i++)
- {
- perme[i]=0;
- }
- }
- else
- {
- CCTK_WARN(0,"Memory error in PUGH");
- }
-
- /* Arrays can't (yet) have manual set up */
- nprocs = (int *) malloc(dim*sizeof(int));
- if (nprocs)
- {
- for (i=0;i<dim;i++)
- {
- nprocs[i]=0;
- }
- }
- else
+ /* Arrays can't (yet) have periodicity and manual setup,
+ so initialize them to zero */
+ perme = (int *) calloc(dim, sizeof(int));
+ nprocs = (int *) calloc(dim, sizeof(int));
+ if (! (perme && nprocs))
{
CCTK_WARN(0,"Memory error in PUGH");
}
@@ -168,6 +145,23 @@ int PUGH_SetupArrayGroup(pGH *newGH,
free(nprocs);
free(perme);
+ /* Set up the communication buffer used for all variables within this group.
+ Note: only with allocated buffers we can have group communication. */
+ if(newGH->commmodel == PUGH_ALLOCATEDBUFFERS)
+ {
+ groupcomm = PUGH_SetupGArrayGroupComm(newGH,
+ dim,
+ newGH->nvariables,
+ n_variables,
+ n_timelevels - 1,
+ vtype,
+ extras);
+ }
+ else
+ {
+ groupcomm = NULL;
+ }
+
temp = (void ***)realloc
(newGH->variables,(newGH->nvariables+n_variables)*sizeof(void **));
@@ -185,14 +179,15 @@ int PUGH_SetupArrayGroup(pGH *newGH,
for(level = 0; level < n_timelevels; level++)
{
newGH->variables[newGH->nvariables][level] =
- PUGH_SetupPGA(newGH,
- extras,
- connectivity,
- CCTK_VarName(newGH->nvariables),
- newGH->nvariables,
- var_size,
- vtype,
- staggercode);
+ PUGH_SetupGArray(newGH,
+ extras,
+ connectivity,
+ groupcomm,
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode);
}
newGH->nvariables++;
}
@@ -241,6 +236,15 @@ int PUGH_SetupGFGroup(pGH *newGH,
int retval=PUGH_SUCCESS;
int level;
void ***temp;
+ pComm *groupcomm;
+
+ groupcomm = PUGH_SetupGArrayGroupComm(newGH,
+ dim,
+ newGH->nvariables,
+ n_variables,
+ n_timelevels - 1,
+ vtype,
+ newGH->GFExtras[dim-1]);
temp = (void ***)realloc(newGH->variables,
(newGH->nvariables+n_variables)*sizeof(void **));
@@ -258,15 +262,16 @@ int PUGH_SetupGFGroup(pGH *newGH,
{
for(level = 0; level < n_timelevels; level++)
{
- newGH->variables[newGH->nvariables][level] =
- PUGH_SetupPGA(newGH,
- newGH->GFExtras[dim-1],
- newGH->Connectivity[dim-1],
- CCTK_VarName(newGH->nvariables),
- newGH->nvariables,
- var_size,
- vtype,
- staggercode);
+ newGH->variables[newGH->nvariables][level] =
+ PUGH_SetupGArray(newGH,
+ newGH->GFExtras[dim-1],
+ newGH->Connectivity[dim-1],
+ groupcomm,
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode);
}
newGH->nvariables++;
}
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index 441ed59..3d4b1b0 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -13,6 +13,11 @@
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
+#include <assert.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
#include "cctk.h"
#include "pugh.h"
@@ -46,7 +51,7 @@ pGH *PUGH_SetupPGH(void *callerid,
int *nsize,
int *nghostzones,
int staggertype,
- int *perme)
+ int *perme)
{
DECLARE_CCTK_PARAMETERS
@@ -77,28 +82,34 @@ pGH *PUGH_SetupPGH(void *callerid,
GH->Connectivity[idim-1]=
PUGH_SetupConnectivity(idim,
- GH->nprocs,
- nprocs,
- perme);
+ GH->nprocs,
+ nprocs,
+ perme);
free(nprocs);
GH->GFExtras[idim-1]=
PUGH_SetupPGExtras(idim,
- perme,
- staggertype,
- nsize,
- nghostzones,
- GH->nprocs,
- ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
- GH->myproc);
+ perme,
+ staggertype,
+ nsize,
+ nghostzones,
+ GH->nprocs,
+ ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
+ GH->myproc);
}
GH->active = 1;
GH->commmodel = PUGH_ALLOCATEDBUFFERS;
+
#if 0
- CactusResetTimer (&GH->comm_time);
+ /* create the timer for communication time */
+ if(comm_verbose)
+ {
+ GH->comm_time = CCTK_TimerCreateI ();
+ }
#endif
+
/* set staggering flag */
/*GH->stagger = staggertype; */
GH->level = 0;
@@ -201,7 +212,8 @@ void PUGH_DestroyPGH(pGH **GHin)
{
#ifdef DEBUG_PUGH
- printf("Calling Destroying Group %s \n",CCTK_GroupName(group));
+ printf("Calling Destroying Group %s\n", CCTK_GroupName(group));
+ fflush(stdout);
#endif
CCTK_GroupData(group,&pgroup);
@@ -214,7 +226,7 @@ void PUGH_DestroyPGH(pGH **GHin)
{
case GROUP_GF:
case CCTK_ARRAY:
- PUGH_DestroyPGA(&(((pGA ***)GH->variables)[variable][i]));
+ PUGH_DestroyGArray(&(((pGA ***)GH->variables)[variable][i]));
break;
case GROUP_SCALAR:
free(GH->variables[variable][i]);
@@ -231,13 +243,12 @@ void PUGH_DestroyPGH(pGH **GHin)
PUGH_DestroyPGExtras(&GH->GFExtras[i-1]);
}
+ free(GH->identity_string);
free(GH->Connectivity);
free(GH->GFExtras);
free(GH->variables);
free(GH);
*GHin = NULL;
-
- fflush(stdout);
}
void pGH_DumpInfo(pGH *GH)
@@ -323,3 +334,51 @@ int PUGH_SetupDefaultTopology(int dim, int *nprocs)
return retval;
}
+
+int PUGH_ParallelInit(cGH *GH)
+{
+ return 0;
+}
+
+int PUGH_Exit(int retval, cGH *GH)
+{
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Finalize());
+#endif
+ exit(retval);
+ return(retval);
+}
+int PUGH_Abort(cGH *GH)
+{
+ assert(0);
+ exit(0);
+ return(0);
+}
+
+
+int PUGH_MyProc(cGH *GH)
+{
+ int myproc;
+
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Comm_rank(PUGH_pGH(GH)->PUGH_COMM_WORLD, &myproc));
+#else
+ myproc = 0;
+#endif
+
+ return myproc;
+}
+
+
+int PUGH_nProcs(cGH *GH)
+{
+ int nprocs;
+
+#ifdef CCTK_MPI
+ CACTUS_MPI_ERROR(MPI_Comm_size(PUGH_pGH(GH)->PUGH_COMM_WORLD, &nprocs));
+#else
+ nprocs = 1;
+#endif
+
+ return nprocs;
+}
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 171aa7c..d27ace0 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -69,12 +69,12 @@ pGExtras *PUGH_SetupPGExtras(int dim,
this->dim = dim;
- PUGH_SetupPGExtrasSizes (dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc,this);
+ PUGH_SetupPGExtrasSizes(dim, perme, stagger, sh, nghosts,
+ total_procs, nprocs, this_proc,this);
PUGH_SetupPGExtrasOwnership(dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc, this);
+ total_procs, nprocs, this_proc, this);
PUGH_SetupPGExtrasStaggering(dim, perme, stagger, sh, nghosts,
- total_procs, nprocs, this_proc, this);
+ total_procs, nprocs, this_proc, this);
}
}
@@ -124,6 +124,8 @@ void PUGH_DestroyPGExtras(pGExtras **PGExtras)
free((*PGExtras)->nghostzones);
free((*PGExtras)->nsize);
free((*PGExtras)->lnsize);
+ free((*PGExtras)->iterator);
+ free((*PGExtras)->hyper_volume);
free(*PGExtras);
*PGExtras = NULL;
@@ -552,9 +554,11 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints = (int *) malloc(total_procs*sizeof(int));
/* Things just depending on dimension */
- this->nghostzones = (int *)malloc(dim*sizeof(int));
- this->nsize = (int *)malloc(dim*sizeof(int));
- this->lnsize = (int *)malloc(dim*sizeof(int));
+ this->nghostzones = (int *)malloc(dim*sizeof(int));
+ this->nsize = (int *)malloc(dim*sizeof(int));
+ this->lnsize = (int *)malloc(dim*sizeof(int));
+ this->iterator = (int *)malloc(dim*sizeof(int));
+ this->hyper_volume = (int *)malloc(dim*sizeof(int));
/* Check all the above succeeded and then get memory for
* arrays hanging off the above.
@@ -565,7 +569,9 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints &&
this->nghostzones &&
this->nsize &&
- this->lnsize)
+ this->lnsize &&
+ this->iterator &&
+ this->hyper_volume)
{
this->lb[0] = (int *)malloc(total_procs *dim*sizeof(int));
this->ub[0] = (int *)malloc(total_procs *dim*sizeof(int));
@@ -607,6 +613,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
}
else
@@ -626,6 +636,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
@@ -635,7 +649,9 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->rnpoints &&
this->nghostzones &&
this->nsize &&
- this->lnsize)
+ this->lnsize &&
+ this->iterator &&
+ this->hyper_volume)
{
retcode = 0;
for (i = 0 ; i < PUGH_NSTAGGER; i++)
@@ -735,6 +751,10 @@ int PUGH_SetupPGExtrasMemory(int dim,
this->nsize = NULL;
free(this->lnsize);
this->lnsize = NULL;
+ free(this->iterator);
+ this->iterator = NULL;
+ free(this->hyper_volume);
+ this->hyper_volume = NULL;
}
}
}
@@ -853,14 +873,14 @@ int PUGH_SetupPGExtrasSizes(int dim,
@@*/
int PUGH_SetupPGExtrasOwnership(int dim,
- int *perme,
- int stagger,
- int *sh,
- int *nghosts,
- int total_procs,
- int *nprocs,
- int this_proc,
- pGExtras *this)
+ int *perme,
+ int stagger,
+ int *sh,
+ int *nghosts,
+ int total_procs,
+ int *nprocs,
+ int this_proc,
+ pGExtras *this)
{
int tmp;
int dir, idir;
@@ -1180,394 +1200,307 @@ int PUGH_SetupRemoteSizes(int dim,
return 0;
}
+
/*@@
- @routine PUGH_SetupPGA
- @date Mon Nov 8 16:29:34 1999
- @author Tom Goodale
+ @routine PUGH_SetupGArrayGroupComm
+ @date Tue 06 Jun 2000
+ @author Thomas Radke
@desc
- Sets up a new pGA.
+ Sets up a communication buffer for a group of GAs.
@enddesc
@calls
@calledby
@history
@endhistory
-
@@*/
-pGA *PUGH_SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger)
+pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
+ int dim,
+ int first_var,
+ int n_vars,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras)
{
- pGA *this;
int i;
- this = (pGA *)malloc(sizeof(pGA));
+ pComm *this;
- if(this)
- {
- this->extras = extras;
- this->connectivity = connectivity;
- this->parent = parent;
- this->varsize = varsize;
- this->vtype = vtype;
- this->stagger = stagger;
- this->id = id;
- this->storage = PUGH_NOSTORAGE;
- this->commflag = PUGH_NOCOMM;
+ this = (pComm *) malloc (sizeof (pComm));
- this->name = (char *) malloc((strlen(name)+1)*sizeof(char));
- this->buffer_sz = (int *) malloc(2*extras->dim*sizeof(int));
- this->send_buffer = (void **)malloc(2*extras->dim*sizeof(void *));
- this->recv_buffer = (void **)malloc(2*extras->dim*sizeof(void *));
- this->docomm = (int *) malloc(2*extras->dim*sizeof(int));
- this->padddata = (void *) calloc(1, varsize);
- this->data = this->padddata;
-
+ if (this)
+ {
+ this->buffer_sz = (int *) malloc (2 * dim * sizeof (int));
+ this->send_buffer = (void **) malloc (2 * dim * sizeof (void *));
+ this->recv_buffer = (void **) malloc (2 * dim * sizeof (void *));
#ifdef CCTK_MPI
- this->sreq = (MPI_Request *)malloc(2*extras->dim*sizeof(MPI_Request));
- this->rreq = (MPI_Request *)malloc(2*extras->dim*sizeof(MPI_Request));
+ this->sreq = (MPI_Request *) malloc (2 * dim * sizeof (MPI_Request));
+ this->rreq = (MPI_Request *) malloc (2 * dim * sizeof (MPI_Request));
#endif
+ this->docomm = (int *) malloc (2 * dim * sizeof (int));
- if(this->name &&
- this->buffer_sz &&
+ if(this->buffer_sz &&
this->send_buffer &&
this->recv_buffer &&
- this->docomm &&
#ifdef CCTK_MPI
this->sreq &&
this->rreq &&
#endif
- this->padddata)
+ this->docomm)
{
- strcpy(this->name, name);
+ this->commflag = PUGH_NOCOMM;
+ this->first_var = first_var;
+ this->n_vars = n_vars;
+ this->sync_timelevel = sync_timelevel;
- for(i = 0; i < 2*extras->dim; i++)
+#ifdef CCTK_MPI
+ switch (vartype)
+ {
+ case CCTK_VARIABLE_CHAR:
+ this->mpi_type = PUGH_MPI_CHAR; break;
+ case CCTK_VARIABLE_INT4:
+ this->mpi_type = PUGH_MPI_INT4; break;
+ case CCTK_VARIABLE_INT:
+ this->mpi_type = PUGH_MPI_INT; break;
+ case CCTK_VARIABLE_REAL4:
+ this->mpi_type = PUGH_MPI_REAL4; break;
+ case CCTK_VARIABLE_REAL:
+ this->mpi_type = PUGH_MPI_REAL; break;
+ case CCTK_VARIABLE_COMPLEX:
+ this->mpi_type = pughGH->PUGH_mpi_complex; break;
+
+ default:
+ CCTK_WARN (1, "Unsupported variable type for PUGH communication");
+ break;
+ }
+#endif
+
+ for (i = 0; i < 2 * dim; i++)
{
this->buffer_sz[i] = 0;
this->send_buffer[i] = NULL;
this->recv_buffer[i] = NULL;
- this->docomm[i] = 0;
#ifdef CCTK_MPI
/* Null my send and receive requests */
- this->sreq[i] = MPI_REQUEST_NULL;
- this->rreq[i] = MPI_REQUEST_NULL;
+ this->sreq[i] = MPI_REQUEST_NULL;
+ this->rreq[i] = MPI_REQUEST_NULL;
#endif
+ this->docomm[i] = PUGH_NOCOMM;
}
}
else
{
- free(this->name);
- free(this->buffer_sz);
- free(this->send_buffer);
- free(this->recv_buffer);
- free(this->docomm);
- free(this->padddata);
-
+ if (this->buffer_sz)
+ {
+ free (this->buffer_sz);
+ }
+ if (this->send_buffer)
+ {
+ free (this->send_buffer);
+ }
+ if (this->recv_buffer)
+ {
+ free (this->recv_buffer);
+ }
#ifdef CCTK_MPI
- free(this->sreq);
- free(this->rreq);
+ if (this->sreq)
+ {
+ free (this->sreq);
+ }
+ if (this->rreq)
+ {
+ free (this->rreq);
+ }
#endif
- free(this);
+ if (this->docomm)
+ {
+ free (this->docomm);
+ }
+
+ free (this);
this = NULL;
}
}
+ /* Now set the hyper_volume vector in the extras structure
+ which is used to copy the ghostzones from/to the comm buffers.
+ NOTE: Sizes are already in bytes here. hyper_volume[0] is unused.
+ */
+ if(this)
+ {
+ extras->hyper_volume[0] = CCTK_VarTypeSize(vartype);
+ for (i = 1; i < extras->dim; i++)
+ {
+ extras->hyper_volume[i] = extras->hyper_volume[i-1] * extras->lnsize[i-1];
+ }
+ }
+
return this;
}
+
/*@@
- @routine PUGH_DestroyPGA
- @date Mar 12 2000
+ @routine PUGH_SetupGArrayComm
+ @date Tue 06 Jun 2000
@author Thomas Radke
@desc
- Destroys a pGA object.
+ Sets up a communication buffer for a GA.
@enddesc
@calls
@calledby
@history
@endhistory
-
@@*/
-void PUGH_DestroyPGA(pGA **GA)
+pComm *PUGH_SetupGArrayComm(pGH *pughGH,
+ int dim,
+ int var,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras)
{
- if(GA && *GA)
- {
- if((*GA)->storage != PUGH_NOSTORAGE)
- {
- PUGH_DisableGADataStorage(*GA);
- }
- free((*GA)->name);
- free((*GA)->buffer_sz);
- free((*GA)->send_buffer);
- free((*GA)->recv_buffer);
- free((*GA)->docomm);
- free((*GA)->padddata);
-
-#ifdef CCTK_MPI
- free((*GA)->sreq);
- free((*GA)->rreq);
-#endif
- free(*GA);
- *GA = NULL;
- }
+ return PUGH_SetupGArrayGroupComm(pughGH,
+ dim,
+ var,
+ 1,
+ sync_timelevel,
+ vartype,
+ extras);
}
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing)
+
+ /*@@
+ @routine PUGH_DestroyComm
+ @date Tue 06 Jun 2000
+ @author Thomas Radke
+ @desc
+ Destroys a communication buffer.
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
+@@*/
+void PUGH_DestroyComm(pComm **comm)
{
- int retval;
- int i;
- int dir;
- int sz;
- int ldir;
- char *temp_byte;
-#if 0
- int special_pad, cache_size;
- long start;
+ free ((*comm)->buffer_sz);
+ free ((*comm)->send_buffer);
+ free ((*comm)->recv_buffer);
+#ifdef CCTK_MPI
+ free ((*comm)->sreq);
+ free ((*comm)->rreq);
#endif
+ free ((*comm)->docomm);
+ free (*comm);
- retval = 0;
- if(GA->storage == PUGH_NOSTORAGE)
- {
- /* Allocate memory for communication buffers */
- for(i = 0; i < 2*GA->extras->dim; i++)
- {
- dir = i/2;
-
- sz = GA->extras->nghostzones[dir];
- for(ldir = 0; ldir < GA->extras->dim; ldir++)
- {
- sz *= GA->extras->lnsize[ldir];
- }
-
- sz /= GA->extras->lnsize[dir];
-
- if(GA->connectivity->neighbours[this_proc][i] >= 0)
- {
- GA->buffer_sz[i] = sz;
- GA->send_buffer[i] = malloc(sz * GA->varsize);
- GA->recv_buffer[i] = malloc(sz * GA->varsize);
-
- if(!(GA->send_buffer[i]
- && GA->recv_buffer[i]))
- {
- retval = -1;
- break;
- }
- }
- else
- {
- GA->buffer_sz[i] = 0;
- GA->send_buffer[i] = NULL;
- GA->recv_buffer[i] = NULL;
- }
- }
-
- if(retval)
- {
- for(; i >=0 ; i--)
- {
- free(GA->send_buffer[i]);
- free(GA->recv_buffer[i]);
- GA->buffer_sz[i] = 0;
- }
- }
-
- /* Now assign memory for the variable itself */
- if(!retval)
- {
- if(GA->padddata)
- {
- free(GA->padddata);
- }
+ *comm = NULL;
+}
- if(!padding_active)
- {
- /* Easy case. */
- if(zero_memory)
- {
- GA->padddata = calloc(GA->extras->npoints, GA->varsize);
- }
- else
- {
- GA->padddata = malloc(GA->extras->npoints * GA->varsize);
- }
- GA->data = GA->padddata;
- }
- else
- {
+ /*@@
+ @routine PUGH_SetupGArray
+ @date Mon Nov 8 16:29:34 1999
+ @author Tom Goodale
+ @desc
+ Sets up a new pGA.
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
- /* Use the Cactus Cache alignment function */
- GA->data = Util_CacheMalloc(GA->id,
- GA->extras->npoints * GA->varsize,
- &(GA->padddata));
-
- /* Zero the memory if desired. */
- if(GA->data && zero_memory)
- {
- for(temp_byte = (char *)GA->data;
- temp_byte < ((char *)GA->data)+GA->extras->npoints * GA->varsize;
- temp_byte++)
- {
- *temp_byte = 0;
- }
- }
+@@*/
+pGA *PUGH_SetupGArray(void *parent,
+ pGExtras *extras,
+ pConnectivity *connectivity,
+ pComm *groupcomm,
+ const char *name,
+ int id,
+ int varsize,
+ int vtype,
+ int stagger)
+{
+ pGA *this;
- }
- }
- }
+ this = (pGA *)malloc(sizeof(pGA));
- if (!GA->padddata)
+ if(this)
{
- CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
- "FATAL ERROR: Cannot allocate data for %s [%d]\n",
- GA->name, GA->id);
- }
-
- GA->storage = PUGH_STORAGE;
-
- return retval;
-
-}
+ this->extras = extras;
+ this->connectivity = connectivity;
+ this->comm = NULL;
+ this->groupcomm = groupcomm;
+ this->parent = parent;
+ this->varsize = varsize;
+ this->vtype = vtype;
+ this->stagger = stagger;
+ this->id = id;
-int PUGH_DisableGADataStorage(pGA *GA)
-{
- int retval;
- int i;
+ this->storage = PUGH_NOSTORAGE;
- if (GA->storage == PUGH_STORAGE)
- {
- if (GA->padddata)
+ this->name = (char *) malloc((strlen(name)+1)*sizeof(char));
+ this->padddata = (void *) calloc(1, varsize);
+ this->data = this->padddata;
+
+ if(this->name && this->padddata)
{
- free(GA->padddata);
- GA->padddata = NULL;
- GA->data = NULL;
+ strcpy(this->name, name);
}
-
- for (i = 0; i < 2*GA->extras->dim; i++)
+ else
{
- if (GA->send_buffer[i])
+ if(this->name)
{
-#ifdef CCTK_MPI
- if(GA->sreq[i] != MPI_REQUEST_NULL)
- {
- CACTUS_MPI_ERROR(MPI_Request_free(&(GA->sreq[i])));
- }
-#endif
- free(GA->send_buffer[i]);
- GA->send_buffer[i] = NULL;
+ free(this->name);
}
- }
-
- for (i = 0; i < 2*GA->extras->dim; i++)
- {
- if (GA->recv_buffer[i])
+ if(this->padddata)
{
- free(GA->recv_buffer[i]);
- GA->recv_buffer[i] = NULL;
+ free(this->padddata);
}
+ free(this);
+ this = NULL;
}
-
- GA->padddata = calloc(1, GA->varsize);
- GA->data = GA->padddata;
-
- GA->storage = PUGH_NOSTORAGE;
- retval = 0;
- }
- else
- {
- retval = 1;
}
- return retval;
+ return this;
}
-
/*@@
- @routine PUGH_GAComm
- @date Sun Jan 23 12:46:23 2000
- @author Gabrielle Allen
+ @routine PUGH_DestroyGArray
+ @date Mar 12 2000
+ @author Thomas Radke
@desc
- This sets the docomm[2*dim] array of the GA based
- on the setting of the comm flag.
+ Destroys a pGA object.
@enddesc
-@@*/
-
+ @calls
+ @calledby
+ @history
+
+ @endhistory
-void PUGH_GAComm(pGA *GA, int docomm)
+@@*/
+void PUGH_DestroyGArray(pGA **GA)
{
- int idir;
- int dim = GA->extras->dim;
-
- /* Copy docomm */
- GA->commflag = docomm;
-
- /* First set all communcation off */
- for (idir=0;idir<2*dim;idir++)
- GA->docomm[idir] = 0;
-
- if (docomm == PUGH_NOCOMM)
- {
- for (idir=0;idir<2*dim;idir++)
- GA->docomm[idir] = 0;
- }
- else if (docomm == PUGH_ALLCOMM)
- {
- for (idir=0;idir<2*dim;idir++)
- {
- GA->docomm[idir] = 1;
- }
- }
- else if (docomm == PUGH_PLUSFACESCOMM)
- {
- for (idir=0;idir<dim;idir++)
- {
- GA->docomm[2*idir+1] = 1;
- }
- }
- else if (docomm == PUGH_MINUSFACESCOMM)
+ if(GA && *GA)
{
- for (idir=0;idir<dim;idir++)
+ if((*GA)->storage != PUGH_NOSTORAGE)
{
- GA->docomm[2*idir] = 1;
+ PUGH_DisableGArrayDataStorage(*GA);
}
- }
- else
- {
- for (idir=0;idir<dim;idir++)
+ /* destroy group comm buffers only once per group */
+ if((*GA)->groupcomm && (*GA)->groupcomm->first_var == (*GA)->id)
{
- if (docomm == PUGH_COMM(idir))
- {
- GA->docomm[2*idir] = 1;
- GA->docomm[2*idir+1] = 1;
- }
+ PUGH_DestroyComm(&(*GA)->groupcomm);
}
- }
-
- /* FIXME Add back the check that you have a valid COMM model: Gab */
-
- /* Handle nsize = 1 type cases. This is only important for one
- processor MPI periodic boundaries */
-
- for (idir=0;idir<dim;idir++)
- {
- if (GA->extras->nsize[idir] == 1)
+ if((*GA)->comm)
{
- GA->docomm[2*idir] = 0;
- GA->docomm[2*idir+1] = 0;
+ PUGH_DestroyComm(&(*GA)->comm);
}
+ free((*GA)->name);
+ free((*GA)->padddata);
+
+ free(*GA);
+ *GA = NULL;
}
-
}
diff --git a/src/Startup.c b/src/Startup.c
index fb5a51a..10a0740 100644
--- a/src/Startup.c
+++ b/src/Startup.c
@@ -17,15 +17,10 @@
#include "pugh_Comm.h"
#include "pugh_reductions.h"
-static char *WhatIsPugh(void);
-
static char *rcsid="$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_Startup_c)
-/* Store the handle in a global variable for the moment. */
-int pugh_GHExtension;
-
/*@@
@routine PUGH_Startup
@@ -43,6 +38,7 @@ int pugh_GHExtension;
@@*/
int PUGH_Startup(void)
{
+ int pugh_GHExtension;
char *banner;
/* Register the stuff in the GH extension. */
@@ -94,7 +90,7 @@ int PUGH_Startup(void)
}
-
+#if 0
/*@@
@routine WhatIsPugh
@date Mon Apr 14 22:13:12 1997
@@ -140,4 +136,4 @@ static char *WhatIsPugh(void)
return (pughnames[which]);
}
-
+#endif
diff --git a/src/Storage.c b/src/Storage.c
new file mode 100644
index 0000000..801f1a4
--- /dev/null
+++ b/src/Storage.c
@@ -0,0 +1,561 @@
+ /*@@
+ @file Storage.c
+ @date Tue Jun 13 2000
+ @author Thomas Radke
+ @desc
+ Pugh storage functions
+ @enddesc
+ @version $Header$
+ @@*/
+
+/*#define DEBUG_PUGH*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+#include "pugh.h"
+#include "pughi.h"
+
+static char *rcsid="$Header$";
+
+CCTK_FILEVERSION(CactusPUGH_PUGH_Storage_c)
+
+static int totalstorage = 0; /* Storage for GAs in Bytes */
+static int totalnumber = 0; /* Number of stored GAs */
+
+
+/*@@
+ @routine PUGH_ArrayGroupSize
+ @author Gabrielle Allen
+ @date 11 Apr 1999
+ @desc
+ Returns size of arrays in a group, in a given direction
+ @enddesc
+ @calls
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @vcomment either index or groupname must be present
+ @endvar
+ @var group
+ @vdesc index of group
+ @vtype int
+ @vio in
+ @vcomment either index or groupname must be given
+ @endvar
+ @var dir
+ @vdesc direction
+ @vtype int
+ @vio in
+ @@*/
+
+const int *PUGH_ArrayGroupSize(cGH *GH, int dir, int group, const char *groupname)
+{
+ int *sizep;
+ pGA *GA;
+
+ if (groupname)
+ {
+ group = CCTK_GroupIndex(groupname);
+ }
+
+ /* get pointer to pGA for a timelevel of first variable in this group */
+ GA = (pGA *) PUGH_pGH(GH)->variables[CCTK_FirstVarIndexI(group)][0];
+
+ if (GA->storage == PUGH_STORAGE)
+ {
+ if (dir >= 0 && dir < GA->extras->dim)
+ {
+ sizep = &GA->extras->lnsize[dir];
+ }
+ else
+ {
+ sizep = NULL;
+ CCTK_WARN(1, "Wrong value for dir");
+ }
+ }
+ else
+ {
+ sizep = &_cctk_one;
+ }
+
+ return (sizep);
+}
+
+
+/*@@
+ @routine PUGH_QueryGroupStorage
+ @author Gabrielle Allen
+ @date 13 Apr 1999
+ @desc
+ Returns true if group has storage assigned, false otherwise
+ @enddesc
+ @calls CCTK_DecomposeName
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be queried
+ @vtype const char *
+ @vio in
+ @vcomment either index or groupname must be present
+ @endvar
+ @var group
+ @vdesc index of group
+ @vtype int
+ @vio in
+ @vcomment either index or groupname must be given
+ @endvar
+ @@*/
+
+int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
+{
+
+ int first;
+ int storage=PUGH_UNDEFINEDSTORAGE;
+ int retval;
+ int grouptype;
+
+ if (groupname)
+ {
+ group = CCTK_GroupIndex(groupname);
+ }
+
+ /* get first variable in group */
+ first = CCTK_FirstVarIndexI(group);
+ if (first < 0)
+ {
+ CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "Invalid group ID %d in PUGH_QueryGroupStorage", group);
+ }
+
+ grouptype = CCTK_GroupTypeI(group);
+
+ if (grouptype == CCTK_SCALAR)
+ {
+ storage = PUGH_STORAGE;
+ }
+ else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
+ {
+ storage = ((pGA *) PUGH_pGH(GH)->variables[first][0])->storage;
+ }
+ else
+ {
+ CCTK_WARN(0,"Unknown group type in PUGH_QueryGroupStorage");
+ }
+
+ if (storage == PUGH_STORAGE)
+ {
+ retval = 1;
+ }
+ else if (storage == PUGH_NOSTORAGE)
+ {
+ retval = 0;
+ }
+ else
+ {
+ CCTK_WARN(0,"Inconsistency in PUGH_QueryGroupStorage");
+ retval = PUGH_ERROR;
+ }
+
+ return (retval);
+}
+
+
+/*@@
+ @routine PUGH_EnableGroupStorage
+ @author Tom Goodale
+ @date 30 Mar 1999
+ @desc
+ Enables storage for all variables in the group indicated by groupname.
+ @enddesc
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
+ PUGH_EnableGArrayGroupStorage
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @endvar
+ @@*/
+
+int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
+{
+
+ DECLARE_CCTK_PARAMETERS
+
+ int group; /* group index */
+ int first_var; /* first variable's index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
+ pGA *GA;
+ pGH *pughGH;
+
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGroupStorage: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
+
+ pughGH = PUGH_pGH(GH);
+ group = CCTK_GroupIndex(groupname);
+ first_var = CCTK_FirstVarIndexI(group);
+
+ /* get the group info from its index */
+ CCTK_GroupData(group, &pgroup);
+
+ /* SCALAR types have always switched on memory */
+ if (pgroup.grouptype == CCTK_SCALAR)
+ {
+ rc = 1;
+ }
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
+ {
+ rc = PUGH_EnableGArrayGroupStorage(pughGH,
+ first_var,
+ pgroup.numvars,
+ pgroup.numtimelevels);
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupStorage");
+ rc = -1;
+ }
+
+ /* Report on memory usage */
+ if (storage_verbose)
+ {
+ if (rc == 0)
+ {
+ /* get GA pointer of first var in group */
+ GA = (pGA *) pughGH->variables[first_var][0];
+
+ /* Memory toggled */
+ totalnumber += pgroup.numvars;
+ totalstorage += GA->extras->npoints * GA->varsize *
+ pgroup.numtimelevels * pgroup.numvars;
+ printf("Switched memory on for %s \n [Num Arrays: %d Total Size: %d]\n",
+ groupname,totalnumber,totalstorage);
+ }
+ else if (rc == 1)
+ {
+ /* Memory already on */
+ printf("Memory already on for %s\n",groupname);
+ }
+ }
+
+ return (rc);
+
+}
+
+
+/*@@
+ @routine PUGH_DisableGroupStorage
+ @author Tom Goodale
+ @date 30 Mar 1999
+ @desc
+ Disables storage for all variables in the group indicated by groupname.
+ @enddesc
+ @calls CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
+ @history
+
+ @endhistory
+ @var GH
+ @vdesc Pointer to CCTK grid hierarchy
+ @vtype cGH
+ @vio in
+ @endvar
+ @var groupname
+ @vdesc name of the group to be synchronized
+ @vtype const char *
+ @vio in
+ @endvar
+ @@*/
+
+int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
+{
+ DECLARE_CCTK_PARAMETERS
+
+ int group; /* group index */
+ cGroup pgroup; /* group information */
+ int rc; /* return code */
+ pGH *pughGH; /* PUGH extension reference */
+ pGA *GA;
+ int level;
+ int first_var,var;
+ int unchanged; /* count how many aren't toggled */
+
+ /* get PUGH extension handle */
+ pughGH = PUGH_pGH(GH);
+
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
+ fflush(stdout);
+#endif
+
+ /* get the group info from its index */
+ group = CCTK_GroupIndex(groupname);
+ CCTK_GroupData(group, &pgroup);
+
+ /* get global index of first variable in group */
+ first_var = CCTK_FirstVarIndexI(group);
+
+ unchanged = 0;
+
+ if (pgroup.grouptype == CCTK_SCALAR)
+ {
+ rc = 1;
+ }
+ else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
+ {
+ for (var = first_var; var < first_var+pgroup.numvars; var++)
+ {
+ for (level = 0; level < pgroup.numtimelevels; level++)
+ {
+ unchanged = unchanged +
+ PUGH_DisableGArrayDataStorage((pGA *)(pughGH->variables[var][level]));
+ }
+ }
+
+ rc = 1;
+ }
+ else
+ {
+ CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupStorage");
+ rc = 0;
+ }
+
+ /* Report on memory usage */
+ if (storage_verbose)
+ {
+ if (unchanged == 0)
+ {
+ GA = (pGA *) pughGH->variables[first_var][0];
+
+ /* Memory toggled */
+ totalnumber -= pgroup.numvars;
+ totalstorage -= GA->extras->npoints * GA->varsize *
+ pgroup.numtimelevels * pgroup.numvars;
+ printf("Switched memory off for %s \n [Num Arrays: %d Total Size: %d]\n",
+ groupname,totalnumber,totalstorage);
+ }
+ else if (unchanged == pgroup.numvars)
+ {
+ /* Memory already off */
+ printf("Memory already off for %s\n",groupname);
+ }
+ else
+ {
+ CCTK_WARN(1,"Inconsistency in group memory assignment");
+ }
+ }
+
+ return (rc);
+}
+
+
+/*****************************************************************************/
+/* local functions */
+/*****************************************************************************/
+
+int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
+ int first_var,
+ int n_variables,
+ int n_timelevels)
+{
+ DECLARE_CCTK_PARAMETERS
+ int nstorage; /* Number of Arrays for which storage was set */
+ int nnostorage; /* Number of Arrays for which no storage was set */
+ int retval;
+ int var;
+ pGA *GA;
+ int level;
+
+ nstorage = 0;
+ nnostorage = 0;
+
+ for (var = first_var; var < first_var + n_variables; var++)
+ {
+ for (level = 0; level < n_timelevels; level++)
+ {
+ GA = (pGA *) pughGH->variables[var][level];
+
+ if (! GA->storage)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayGroupStorage: request for var '%s' "
+ "timelevel %d\n", GA->name, level);
+ fflush(stdout);
+#endif
+ PUGH_EnableGArrayDataStorage(GA,
+ pughGH->myproc,
+ zero_memory,
+ padding_active,
+ padding_cacheline_bits,
+ padding_size,
+ padding_address_spacing);
+
+ ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
+
+ nnostorage++;
+ }
+ else
+ {
+ ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
+ nstorage++;
+ }
+ }
+ }
+
+ if (nstorage > 0 && nnostorage > 0)
+ {
+ CCTK_WARN(0, "Group storage violation in PUGH_EnableGArrayGroupStorage");
+ retval = -1;
+ }
+ else if (nstorage > 0)
+ {
+ retval = 1;
+ }
+ else if (nnostorage > 0)
+ {
+ retval = 0;
+ }
+ else
+ {
+ CCTK_WARN(0, "Problem in PUGH_EnableGArrayGroupStorage");
+ retval = -1;
+ }
+
+ return retval;
+
+}
+
+
+int PUGH_EnableGArrayDataStorage(pGA *GA,
+ int this_proc,
+ int zero_memory,
+ int padding_active,
+ int padding_cacheline_bits,
+ int padding_size,
+ int padding_address_spacing)
+{
+
+ if(GA->storage == PUGH_NOSTORAGE)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_EnableGArrayDataStorage: allocating storage "
+ "for var '%s'\n", GA->name);
+ fflush(stdout);
+#endif
+
+ /* Now assign memory for the variable itself */
+ if(GA->padddata)
+ {
+ free(GA->padddata);
+ }
+
+ if(!padding_active)
+ {
+ /* Easy case. */
+ if(zero_memory)
+ {
+ GA->padddata = calloc(GA->extras->npoints, GA->varsize);
+ }
+ else
+ {
+ GA->padddata = malloc(GA->extras->npoints * GA->varsize);
+ }
+
+ GA->data = GA->padddata;
+ }
+ else
+ {
+
+ /* Use the Cactus Cache alignment function */
+ GA->data = Util_CacheMalloc(GA->id,
+ GA->extras->npoints * GA->varsize,
+ &(GA->padddata));
+
+ /* Zero the memory if desired. */
+ if(GA->data && zero_memory)
+ {
+ memset(GA->data, 0, GA->extras->npoints * GA->varsize);
+ }
+
+ }
+ }
+
+ if (!GA->padddata)
+ {
+ CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+ "FATAL ERROR: Cannot allocate data for %s [%d]\n",
+ GA->name, GA->id);
+ }
+
+ GA->storage = PUGH_STORAGE;
+
+ return 0;
+
+}
+
+
+int PUGH_DisableGArrayDataStorage(pGA *GA)
+{
+ int retval;
+
+ if (GA->storage == PUGH_STORAGE)
+ {
+#ifdef DEBUG_PUGH
+ printf(" PUGH_DisableGArrayDataStorage: freeing storage for var '%s'\n",
+ GA->name);
+ fflush(stdout);
+#endif
+ if (GA->padddata)
+ {
+ free(GA->padddata);
+ GA->padddata = NULL;
+ GA->data = NULL;
+ }
+
+ GA->padddata = calloc(1, GA->varsize);
+ GA->data = GA->padddata;
+
+ GA->storage = PUGH_NOSTORAGE;
+ retval = 0;
+ }
+ else
+ {
+ retval = 1;
+ }
+
+ return retval;
+}
diff --git a/src/include/pGH.h b/src/include/pGH.h
index 9fe17e7..d5139d9 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -13,10 +13,6 @@
typedef struct PGH
{
-#ifdef CCTK_MPI
- MPI_Comm PUGH_COMM_WORLD; /* A MPIcommunicator */
-#endif
-
/* pGH identifier */
void *callerid;
int dim; /* The dimension of the GH */
@@ -55,12 +51,16 @@ typedef struct PGH
/* 0 -> GH is skipped for conv.test */
/* is used in NanCheckGH.c */
+#if 1
/* FIXME */
double comm_time; /* time spent in communication */
+#endif
#ifdef CCTK_MPI
- /* the complex datatype defined in SetupPGH.c */
- MPI_Datatype PUGH_mpi_complex;
+ MPI_Datatype PUGH_mpi_complex;/* the MPI datatype for COMPLEX types */
+ MPI_Comm PUGH_COMM_WORLD; /* the MPI communicator */
+
+#ifdef PUGH_WITH_DERIVED_DATATYPES
/* Derived data types for communication */
MPI_Datatype send_char_dt [PUGH_NSTAGGER][6];
MPI_Datatype recv_char_dt [PUGH_NSTAGGER][6];
@@ -71,15 +71,14 @@ typedef struct PGH
MPI_Datatype send_complex_dt [PUGH_NSTAGGER][6];
MPI_Datatype recv_complex_dt [PUGH_NSTAGGER][6];
#endif
+#endif
/* Used for all grid functions */
pGExtras **GFExtras; /* [dim] stagger ? */
pConnectivity **Connectivity; /* [dim] */
- /* FIXME : should be a dynamically allocated string.
- * If you change this, please change the line in PUGH_SetupGH.
- */
- char identity_string[20];
+ char *identity_string; /* identifier for this pGH */
+
} pGH;
#define XDP 1
diff --git a/src/include/pGV.h b/src/include/pGV.h
index 841ecb0..e86afb1 100644
--- a/src/include/pGV.h
+++ b/src/include/pGV.h
@@ -49,6 +49,10 @@ typedef struct PGExtras
int *rnpoints; /* Number of points on each proc */
int **rnsize; /* [#points on a proc][in each dir] */
+ /* Copying ghostzones to/from comm buffers */
+ int *iterator; /* Iterator for copying ghostzones [dim] */
+ int *hyper_volume; /* Points per subcube dimension [dim] */
+
/* Ghosts and overlaps. */
int *nghostzones; /* Width of ghost zone */
int *ownership[PUGH_NSTAGGER][2];
@@ -65,19 +69,34 @@ typedef struct PGExtras
} pGExtras;
+typedef struct PComm
+{
+ /* buffer_sz[2*dim] */
+ int *buffer_sz; /* Size of the face ghost zones */
+ void **send_buffer; /* Storage for buffered Comm if */
+ void **recv_buffer; /* we don't use derived types */
+ int commflag; /* What is the comm flag set to? */
+ /* do_comm[2*dim] */
+ int *docomm; /* Do we do comm or not? */
+
+ int first_var; /* first variable to be synced */
+ int n_vars; /* number of variables to be synced */
+ int sync_timelevel; /* the timelevel to be synced */
+
+#ifdef CCTK_MPI
+ MPI_Request *sreq, *rreq; /* Comm requests and statuses. */
+ MPI_Status ms;
+ MPI_Datatype mpi_type; /* MPI datatype to use for communication */
+#endif
+
+} pComm;
+
typedef struct PGA
{
char *name; /* The name of the grid function */
int id; /* My ID number in my GH parent. */
void *padddata; /* Storage for the data. */
void *data; /* See the note above. */
- /* buffer_sz[2*dim] */
- int *buffer_sz; /* Size of the face ghost zones */
- void **send_buffer; /* Storage for buffered Comm if */
- void **recv_buffer; /* we don't use derived types */
- int commflag; /* What is the comm flag set to? */
- /* do_comm[2*dim] */
- int *docomm; /* Do we do comm or not? */
int storage; /* Do we have storage or not? */
int stagger; /* Only Vertex Centered now... */
@@ -92,13 +111,12 @@ typedef struct PGA
int varsize; /* The size of the data */
int vtype; /* The type of the data */
-#ifdef CCTK_MPI
- MPI_Request *sreq, *rreq; /* Comm requests and statuses. */
- MPI_Status ms;
-#endif
-
pGExtras *extras;
pConnectivity *connectivity;
+
+ pComm *comm; /* comm buffer for single variable */
+ pComm *groupcomm; /* comm buffer for a variable group */
+
} pGA;
typedef struct PGV
diff --git a/src/include/pugh.h b/src/include/pugh.h
index ce8c89b..7408c0f 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -72,18 +72,21 @@
#ifdef CCTK_MPI
-#define CACTUS_MPI_ERROR(xf) do {int errcode; \
- if((errcode = xf) != MPI_SUCCESS) \
- { \
- char mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
- int resultlen; \
- MPI_Error_string(errcode, mpi_error_string, &resultlen);\
- fprintf(stderr, "MPI Call %s returned error code %d (%s)\n", \
- #xf, errcode, mpi_error_string); \
- fprintf(stderr, "At line %d of file %s\n", \
- __LINE__, __FILE__); \
- } \
- } while (0)
+#define CACTUS_MPI_ERROR(fn_call) \
+ do { \
+ int errcode; \
+ \
+ if ((errcode = fn_call) != MPI_SUCCESS) \
+ { \
+ char mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
+ int resultlen; \
+ \
+ MPI_Error_string (errcode, mpi_error_string, &resultlen); \
+ fprintf (stderr, "MPI call '%s' returned error code %d (%s)\n", \
+ #fn_call, errcode, mpi_error_string); \
+ fprintf(stderr, "At line %d of file %s\n", __LINE__, __FILE__); \
+ } \
+ } while (0)
#endif
#ifdef _cplusplus
@@ -93,21 +96,21 @@ extern "C"
int PUGH_SetupGroup(pGH *newGH,
- int *nsize,
- int *nghostsize,
- int gtype,
- int vtype,
- int dim,
- int n_variables,
- int staggertype,
- int n_timelevels);
+ int *nsize,
+ int *nghostsize,
+ int gtype,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggertype,
+ int n_timelevels);
pGH *PUGH_SetupPGH(void *callerid,
int dim,
int *nsize,
int *nghostzones,
- int staggertype,
- int *perme);
+ int staggertype,
+ int *perme);
int PUGH_GFSize(int dim,
int *nsize);
@@ -115,7 +118,7 @@ int PUGH_GFSize(int dim,
int PUGH_GFGhostsize(int dim,
int *ghostsize);
int PUGH_GFPeriodic(int dim,
- int *perme);
+ int *perme);
pGH *PUGH_pGH(cGH *GH);
diff --git a/src/include/pughi.h b/src/include/pughi.h
index 8defba7..f6fdf1c 100644
--- a/src/include/pughi.h
+++ b/src/include/pughi.h
@@ -30,22 +30,22 @@ pGExtras *PUGH_SetupPGExtras(int dim,
int *nprocs,
int this_proc);
-pGA *SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger);
-
-
-void pGH_DumpInfo(pGH *GH);
+pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
+ int dim,
+ int first_var,
+ int n_vars,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras);
+
+void pGH_DumpInfo(pGH *pughGH);
int PUGH_SetupDefaultTopology(int dim, int *nprocs);
-int pGH_SetupnProcs(pGH *GH,int dim);
-void PUGH_DestroyPGA(pGA **GA);
+int pGH_SetupnProcs(pGH *pughGH,int dim);
+void PUGH_DestroyGArray(pGA **GA);
+void PUGH_DestroyComm(pComm **comm);
void PUGH_DestroyConnectivity(pConnectivity **conn);
void PUGH_DestroyPGExtras(pGExtras **PGExtras);
+void PUGH_DestroyPGH(pGH **pughGH);
int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs);
@@ -104,25 +104,6 @@ int PUGH_SetupRemoteSizes(int dim,
int *nprocs,
pGExtras *this);
-pGA *PUGH_SetupPGA(void *parent,
- pGExtras *extras,
- pConnectivity *connectivity,
- const char *name,
- int id,
- int varsize,
- int vtype,
- int stagger);
-
-int PUGH_EnablePGAStorage(pGA *GA,
- int this_proc,
- int zero_memory,
- int padding_active,
- int padding_cacheline_bits,
- int padding_size,
- int padding_address_spacing);
-
-int PUGH_DisableGADataStorage(pGA *GA);
-
int PUGH_SetupPGExtrasStaggering(int dim,
int *perme,
int stagger,
@@ -133,6 +114,51 @@ int PUGH_SetupPGExtrasStaggering(int dim,
int this_proc,
pGExtras *this);
+pGA *PUGH_SetupGArray(void *parent,
+ pGExtras *extras,
+ pConnectivity *connectivity,
+ pComm *comm,
+ const char *name,
+ int id,
+ int varsize,
+ int vtype,
+ int stagger);
+
+pComm *PUGH_SetupGArrayComm(pGH *pughGH,
+ int dim,
+ int var,
+ int sync_timelevel,
+ int vartype,
+ pGExtras *extras);
+
+int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
+ int first_var,
+ int n_variables,
+ int n_timelevels);
+
+int PUGH_EnableGArrayDataStorage(pGA *GA,
+ int this_proc,
+ int zero_memory,
+ int padding_active,
+ int padding_cacheline_bits,
+ int padding_size,
+ int padding_address_spacing);
+
+int PUGH_DisableGArrayDataStorage(pGA *GA);
+
+int PUGH_EnableGArrayComm(pGA *GA,
+ int commflag);
+
+int PUGH_DisableGArrayComm(pGA *GA);
+
+int PUGH_SyncGArray(pGA *GA);
+
+#ifdef CCTK_MPI
+void PostSendGA (pGH *pughGH, int dir, pComm *comm);
+void PostReceiveGA (pGH *pughGH, int dir, pComm *comm);
+void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm);
+#endif
+
#ifdef _cplusplus
}
#endif
diff --git a/src/make.code.defn b/src/make.code.defn
index d28f187..f5cda42 100644
--- a/src/make.code.defn
+++ b/src/make.code.defn
@@ -1,8 +1,4 @@
# Main make.code.defn file for thorn pugh
# : /usr/users/cactus/CCTK/lib/make/new_thorn.pl,v 1.1 1999/02/03 17:00:50 goodale Exp n
# Source files in this directory
-SRCS = Evolve.c Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
-
-# Subdirectories containing source files
-SUBDIRS =
-
+SRCS = Evolve.c Startup.c GHExtension.c Comm.c Storage.c SetupPGH.c SetupGroup.c Reduction.c PughUtils.c SetupPGV.c PostSendGA.c PostReceiveGA.c FinishReceiveGA.c LoadAware.c
diff --git a/src/pugh_extension.h b/src/pugh_extension.h
index b2b2b1b..62cf29b 100644
--- a/src/pugh_extension.h
+++ b/src/pugh_extension.h
@@ -11,8 +11,6 @@
#ifndef _PUGH_EXTENSION_H_
#define _PUGH_EXTENSION_H_ 1
-extern int pugh_GHExtension;
-
#ifdef _cplusplus
extern "C"
{
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-27 09:25:24 +0000