aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--param.ccl33
-rw-r--r--src/Comm.c373
-rw-r--r--src/GHExtension.c311
-rw-r--r--src/SetupGroup.c424
-rw-r--r--src/SetupPGF.c1
-rw-r--r--src/SetupPGH.c221
-rw-r--r--src/SetupPGV.c123
-rw-r--r--src/include/pGH.h5
-rw-r--r--src/include/pGV.h6
-rw-r--r--src/include/pugh.h1
-rw-r--r--src/include/pugh_constants.h5
11 files changed, 969 insertions, 534 deletions
diff --git a/param.ccl b/param.ccl
index 418ea2e..6de1f76 100644
--- a/param.ccl
+++ b/param.ccl
@@ -73,6 +73,8 @@ BOOLEAN enable_all_storage "Enable storage for all GFs?"
{
} "no"
+# Processor topology
+
KEYWORD proc_topology "How to determine the processor topology"
{
"manual" :: "Specified by proc_top_nx etc"
@@ -95,6 +97,37 @@ INT proc_top_nz "No of Procs in Z direction"
0:* :: "See proc_topology"
} 0
+INT processor_topology_1d_x "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+INT processor_topology_2d_x "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+INT processor_topology_2d_y "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+INT processor_topology_3d_x "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+INT processor_topology_3d_y "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+INT processor_topology_3d_z "No of Procs in X direction"
+{
+ 0:* :: "See proc_topology"
+} 0
+
+
# padding stuff
BOOLEAN padding_active "Padd 3D arrays so they line up on cache lines?"
diff --git a/src/Comm.c b/src/Comm.c
index cda777e..78f8571 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -7,7 +7,8 @@
@enddesc
@@*/
-/*#define DEBUG_COMM*/
+/*#define DEBUG_PUGH*/
+
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
@@ -22,8 +23,17 @@
extern int PUGH_GHExtension;
/* local function prototypes */
-int pugh_SyncGroupArray (cGH *GH, int group, int vtype, int n_vars, int timelevel);
-int pugh_SyncGroupGF (cGH *GH, int group, int vtype, int n_vars, int timelevel);
+
+int pugh_SyncGroupArray (cGH *GH,
+ int group,
+ int vtype,
+ int n_vars,
+ int timelevel);
+int pugh_SyncGroupGF (cGH *GH,
+ int group,
+ int vtype,
+ int n_vars,
+ int timelevel);
int pugh_SyncGFs(pGH *pughGH, int first_var, int n_vars, int timelevel);
void DisableGFDataStorage(pGH *GH, pGF *GF);
void EnableGFDataStorage(pGH *GH, pGF *GF);
@@ -55,6 +65,7 @@ void pGF_FinishRecv(pGH *GH, pGF *GF, int dir);
void pGF_PostRecv(pGH *GH, pGF *GF, int dir);
void pGF_PostSend(pGH *GH, pGF *GF, int dir);
void SetGFComm(pGH *GH, pGF *res, int docomm);
+int pugh_GAComm(pGA *GA, int docomm);
/*@@
@routine pugh_SyncGroup
@@ -90,16 +101,6 @@ int pugh_SyncGroup (cGH *GH, const char *groupname)
group = CCTK_GroupIndex(groupname);
- if (group < 0) {
- char *msg;
-
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_SyncGroup", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- return (0);
- }
-
/* get the group info from its index */
CCTK_GroupData (group,&pgroup);
@@ -116,11 +117,11 @@ int pugh_SyncGroup (cGH *GH, const char *groupname)
break;
case GROUP_ARRAY :
- rc = pugh_SyncGroupArray (GH,
- group,
- pgroup.vartype,
- pgroup.numvars,
- timelevel);
+ /* rc = pugh_SyncGA (GH,
+ group,
+ pgroup.vartype,
+ pgroup.numvars,
+ timelevel);*/
rc = 1;
break;
@@ -181,18 +182,11 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
int *sizep;
pGH *pughGH;
pGF *pughGF;
+ pGA *pughGA;
if (groupname)
{
group = CCTK_GroupIndex(groupname);
-
- if (group < 0) {
- char *msg;
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_ArrayGroupSize", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- }
}
/* get PUGH extension handle */
@@ -200,33 +194,65 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
/* get first variable in group */
first = CCTK_FirstVarIndexI(group);
-
- /* get pointer to pGF for a timelevel of this variable */
- pughGF = (pGF *)(pughGH->variables[first][0]);
- if (pughGF->storage == PUGH_STORAGE)
+ if (CCTK_GroupTypeI(group)==GROUP_GF)
{
- switch (dir)
+ /* get pointer to pGF for a timelevel of this variable */
+ pughGF = (pGF *)(pughGH->variables[first][0]);
+
+ if (pughGF->storage == PUGH_STORAGE)
+ {
+ switch (dir)
+ {
+ case 0:
+ sizep = &pughGH->lnsize[0];
+ break;
+ case 1:
+ sizep = &pughGH->lnsize[1];
+ break;
+ case 2:
+ sizep = &pughGH->lnsize[2];
+ break;
+ default :
+ sizep = NULL;
+ printf("Wrong value for dir in pugh_ArrayGroupSize\n");
+ break;
+ }
+ }
+ else
+ {
+ sizep = &_cctk_one;
+ }
+ }
+ else if (CCTK_GroupTypeI(group)==GROUP_ARRAY)
{
- case 0:
- sizep = &pughGH->lnsize[0];
- break;
- case 1:
- sizep = &pughGH->lnsize[1];
- break;
- case 2:
- sizep = &pughGH->lnsize[2];
- break;
- default :
- sizep = NULL;
- printf("Wrong value for dir in pugh_ArrayGroupSize\n");
- break;
+ /* get pointer to pGF for a timelevel of this variable */
+ pughGA = (pGA *)(pughGH->variables[first][0]);
+
+ if (pughGA->storage == PUGH_STORAGE)
+ {
+ switch (dir)
+ {
+ case 0:
+ sizep = &pughGA->extras->lnsize[0];
+ break;
+ case 1:
+ sizep = &pughGA->extras->lnsize[1];
+ break;
+ case 2:
+ sizep = &pughGA->extras->lnsize[2];
+ break;
+ default :
+ sizep = NULL;
+ printf("Wrong value for dir in pugh_ArrayGroupSize\n");
+ break;
+ }
+ }
+ else
+ {
+ sizep = &_cctk_one;
+ }
}
- }
- else
- {
- sizep = &_cctk_one;
- }
return (sizep);
@@ -265,35 +291,17 @@ int *pugh_ArrayGroupSize (cGH *GH, int dir, int group, const char *groupname)
int pugh_QueryGroupStorage(cGH *GH, int group, const char *groupname)
{
- int first;
+
+ int first,storage;
int retval;
- int group_from_name;
+ int grouptype;
pGH *pughGH;
- pGF *pughGF;
if (groupname)
{
- group_from_name = CCTK_GroupIndex(groupname);
-
- if (group>=0 && group_from_name != group)
- {
- CCTK_WARN(1,"Name and Index differ in pugh_QueryGroupStorage, using name");
- }
-
- group = group_from_name;
-
- if (group < 0) {
- char *msg;
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_QueryGroupStorage", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- }
+ group = CCTK_GroupIndex(groupname);
}
- /* get PUGH extension handle */
- pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
-
/* get first variable in group */
first = CCTK_FirstVarIndexI(group);
if (first < 0) {
@@ -304,43 +312,36 @@ int pugh_QueryGroupStorage(cGH *GH, int group, const char *groupname)
free (msg);
}
- /* scalars have always storage assigned */
- if (CCTK_GroupTypeFromVarI (first) == GROUP_SCALAR)
- return (1);
+ /* get PUGH extension handle */
+ pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
- /* get pointer to pGF for a timelevel of this variable */
- pughGF = (pGF *)(pughGH->variables[first][0]);
+ grouptype = CCTK_GroupTypeI(group);
- if (pughGF->storage == PUGH_STORAGE)
+ if (grouptype == CCTK_SCALAR)
{
+ storage = PUGH_STORAGE;
+ }
+ else if (grouptype==CCTK_GF)
+ {
+ storage = ((pGF *)(pughGH->variables[first][0]))->storage;
+ }
+ else if (grouptype==CCTK_ARRAY)
+ {
+ storage = ((pGA *)(pughGH->variables[first][0]))->storage;
+ }
-#ifdef DEBUG_COMM
- printf("--------------------------------------------------------------\n");
- printf("In QueryGroupStorage\n");
- printf("--------------------\n");
- printf(" Storage assigned for %s\n",groupname);
- printf("--------------------------------------------------------------\n");
-#endif
-
+ if (storage == PUGH_STORAGE)
+ {
retval = 1;
}
- else if (pughGF->storage == PUGH_NOSTORAGE)
+ else if (storage == PUGH_NOSTORAGE)
{
-
-#ifdef DEBUG_COMM
- printf("--------------------------------------------------------------\n");
- printf("In QueryGroupStorage\n");
- printf("--------------------\n");
- printf(" Storage NOT assigned for %s\n",groupname);
- printf("--------------------------------------------------------------\n");
-#endif
-
retval = 0;
}
else
{
CCTK_WARN(0,"Inconsistency in pugh_QueryGroupStorage");
- retval = -1;
+ retval = PUGH_ERROR;
}
return (retval);
@@ -386,26 +387,16 @@ int pugh_EnableGroupStorage(cGH *GH, const char *groupname)
group = CCTK_GroupIndex(groupname);
- if (group < 0) {
- char *msg;
-
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_EnableGroupStorage", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- return (0);
- }
-
/* get the group info from its index */
CCTK_GroupData (group, &pgroup);
/* branch to "enable storage" function according to group type */
switch (pgroup.grouptype) {
- case GROUP_SCALAR :
+ case CCTK_SCALAR :
rc = 1;
break;
- case GROUP_ARRAY :
+ case CCTK_ARRAY :
rc = pugh_EnableArrayGroupStorage (GH,
group,
pgroup.vartype,
@@ -414,7 +405,7 @@ int pugh_EnableGroupStorage(cGH *GH, const char *groupname)
pgroup.numtimelevels);
break;
- case GROUP_GF :
+ case CCTK_GF :
rc = pugh_EnableGFGroupStorage (GH,
group,
pgroup.vartype,
@@ -470,21 +461,12 @@ int pugh_DisableGroupStorage(cGH *GH, const char *groupname)
group = CCTK_GroupIndex(groupname);
- if (group < 0) {
- char *msg;
-
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_DisableGroupStorage", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- return (0);
- }
-
/* get the group info from its index */
CCTK_GroupData (group,&pgroup);
- /* branch to "disable storage" function according to group type */
- switch (pgroup.grouptype) {
+ switch (pgroup.grouptype)
+ {
+
case GROUP_SCALAR :
rc = 1;
break;
@@ -549,19 +531,8 @@ int pugh_EnableGroupComm (cGH *GH, const char *groupname)
int first_var; /* global index of first variable in group */
int rc; /* return code */
-
group = CCTK_GroupIndex(groupname);
- if (group < 0) {
- char *msg;
-
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_EnableGroupComm", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- return (0);
- }
-
/* get the group info from its index */
CCTK_GroupData (group,&pgroup);
@@ -572,20 +543,33 @@ int pugh_EnableGroupComm (cGH *GH, const char *groupname)
pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
/* branch to "enable comm" function according to group type */
- switch (pgroup.grouptype) {
+ switch (pgroup.grouptype)
+ {
case GROUP_SCALAR :
rc = 1;
break;
case GROUP_ARRAY :
- CCTK_WARN (1, "GROUP_ARRAY type not yet supported in pugh_EnableGroupComm");
+ for (var = first_var; var < first_var+pgroup.numvars; var++)
+ {
+ for (level = 0; level < pgroup.numtimelevels; level++)
+ {
+ pugh_GAComm ((pGA *)(pughGH->variables[var][level]),
+ PUGH_ALLCOMM);
+ }
+ }
rc = 1;
break;
case GROUP_GF :
for (var = first_var; var < first_var+pgroup.numvars; var++)
+ {
for (level = 0; level < pgroup.numtimelevels; level++)
- SetGFComm (pughGH, (pGF *)(pughGH->variables[var][level]), PUGH_ALLCOMM);
+ {
+ SetGFComm (pughGH, (pGF *)(pughGH->variables[var][level]),
+ PUGH_ALLCOMM);
+ }
+ }
rc = 1;
break;
@@ -625,7 +609,7 @@ int pugh_EnableGroupComm (cGH *GH, const char *groupname)
int pugh_DisableGroupComm(cGH *GH, const char *groupname)
{
int group; /* group index */
- cGroup pgroup; /* pointer to group information */
+ cGroup pgroup; /* pointer to group information */
pGH *pughGH; /* PUGH extension reference */
int var, level; /* loop variables */
int first_var; /* global index of first variable in group */
@@ -633,16 +617,6 @@ int pugh_DisableGroupComm(cGH *GH, const char *groupname)
group = CCTK_GroupIndex(groupname);
- if (group < 0) {
- char *msg;
-
- msg = (char *) malloc (strlen (groupname) + 80);
- sprintf (msg, "Unknown group \"%s\" in pugh_DisableGroupComm", groupname);
- CCTK_WARN (1, msg);
- free (msg);
- return (0);
- }
-
/* get the group info from its index */
CCTK_GroupData (group,&pgroup);
@@ -653,13 +627,22 @@ int pugh_DisableGroupComm(cGH *GH, const char *groupname)
pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
/* branch to "disable comm" function according to group type */
- switch (pgroup.grouptype) {
+ switch (pgroup.grouptype)
+ {
+
case GROUP_SCALAR :
rc = 1;
break;
case GROUP_ARRAY :
- CCTK_WARN (1, "GROUP_ARRAY type not yet supported in pugh_DisableGroupComm");
+ for (var = first_var; var < first_var+pgroup.numvars; var++)
+ {
+ for (level = 0; level < pgroup.numtimelevels; level++)
+ {
+ pugh_GAComm ((pGA *)(pughGH->variables[var][level]),
+ PUGH_NOCOMM);
+ }
+ }
rc = 1;
break;
@@ -746,7 +729,7 @@ int pugh_SyncGroupArray (cGH *GH, int group, int vtype, int n_vars, int timeleve
}
/*@@
- @routine pugh_SyncGroupGF
+ @routine pugh_SyncGroup
@author Paul Walker
@date March 1997
@desc
@@ -827,7 +810,8 @@ int pugh_SyncGroupGF (cGH *GH, int group, int vtype, int n_vars, int timelevel)
int first_var = CCTK_FirstVarIndexI (group);
/* Say which grid functions will actually be synchronised */
- if (pughGH->sync_verbose && pughGH->myproc == 0) {
+ if (pughGH->sync_verbose && pughGH->myproc == 0)
+ {
char *gname = CCTK_GroupName (group);
printf("Syncing %d grid functions in group %s\n", n_vars, gname);
@@ -1001,7 +985,86 @@ int pugh_EnableArrayGroupStorage(cGH *GH,
int n_variables,
int n_timelevels)
{
- return 0;
+ int nstorage; /* Number of Arrays for which storage was set */
+ int nnostorage; /* Number of Arrays for which no storage was set */
+ int retval;
+ int first_var;
+ int var;
+ pGH *mypGH;
+ pGA *GA;
+ int level;
+ int zero_memory=0;
+ int padding_active=0;
+ int padding_cacheline_bits=0;
+ int padding_size=0;
+ int padding_address_spacing=0;
+
+ nstorage = 0;
+ nnostorage = 0;
+
+ mypGH = (pGH *)GH->extensions[PUGH_GHExtension];
+
+ first_var = CCTK_FirstVarIndexI(group);
+
+ for(var = first_var; var < first_var+n_variables; var++)
+ {
+ for(level = 0; level < n_timelevels; level++)
+ {
+ GA = (pGA *)(mypGH->variables[var][level]);
+
+ if(!GA->storage)
+ {
+#ifdef DEBUG_PUGH
+ {
+ const char *name;
+ name = CCTK_FullName(var);
+ printf("Storage on in pugh_EnableArrayGroupStorage for %s\n",name);
+ }
+#endif
+ pugh_EnablePGAStorage
+ (
+ GA,
+ mypGH->myproc,
+ zero_memory,
+ padding_active,
+ padding_cacheline_bits,
+ padding_size,
+ padding_address_spacing
+ );
+
+ GH->data[var][level] = GA->data;
+
+ nnostorage++;
+ }
+ else
+ {
+ GH->data[var][level] = GA->data;
+ nstorage++;
+ }
+ }
+ }
+
+ if (nstorage > 0 && nnostorage > 0)
+ {
+ CCTK_WARN(0,"Group storage violation in pugh_EnableArrayGroupStorage");
+ retval = -1;
+ }
+ else if (nstorage > 0)
+ {
+ retval = 1;
+ }
+ else if (nnostorage > 0)
+ {
+ retval = 0;
+ }
+ else
+ {
+ CCTK_WARN(0,"Problem in pugh_EnableArrayGroupStorage");
+ retval = -1;
+ }
+
+ return retval;
+
}
@@ -1044,15 +1107,12 @@ int pugh_EnableGFGroupStorage(cGH *GH,
}
#endif
EnableGFDataStorage(mypGH,GF);
-
GH->data[var][level] = GF->data;
-
nnostorage++;
}
else
{
- GH->data[var][level] = GF->data;
-
+ GH->data[var][level] = GF->data;
nstorage++;
}
}
@@ -1121,7 +1181,6 @@ int pugh_DisableGFGroupStorage(cGH *GH,
{
GF = (pGF *)(mypGH->variables[var][level]);
-
if(GF->storage)
{
#ifdef DEBUG_PUGH
@@ -1132,7 +1191,6 @@ int pugh_DisableGFGroupStorage(cGH *GH,
printf(" for %s\n",gfname);
}
#endif
-
DisableGFDataStorage(mypGH,GF);
number_disabled++;
GH->data[var][level] = GF->data;
@@ -1178,7 +1236,6 @@ int pugh_MyProc(cGH *GH)
#ifdef MPI
pGH *pughGH;
-
pughGH = (pGH *) GH->extensions [PUGH_GHExtension];
CACTUS_MPI_ERROR(MPI_Comm_rank(pughGH->PUGH_COMM_WORLD, &myproc));
#else
diff --git a/src/GHExtension.c b/src/GHExtension.c
index fb32503..b2bd9d2 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -15,144 +15,127 @@
#include "cctk.h"
#include "pugh.h"
-#include "cctk_parameters.h"
+#include "cctk_Parameters.h"
static char *rcsid = "$Header$";
extern int PUGH_GHExtension;
-int pugh_SetupGroup(pGH *newGH,
- int gtype,
- int vtype,
- int dim,
- int n_variables,
- int staggertype,
- int n_timelevels);
-
-pGH *SetupPGH(void *callerid,
- int dim,
- int *nsize,
- int *ghostsize);
-
-void *pugh_SetupGH(tFleshConfig *config,
- int convergence_level,
- cGH *GH)
+int PUGH_SetupGroup
+ (
+ pGH *newGH,
+ int *nsize,
+ int *nghostsize,
+ int gtype,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggertype,
+ int n_timelevels
+ );
+
+pGH *PUGH_SetupPGH
+ (
+ void *callerid,
+ int dim,
+ int *nsize,
+ int *ghostsize
+ );
+
+int pugh_GFSize(int dim, int *nsize);
+int pugh_GFGhostsize(int dim,int *ghostsize);
+
+
+void *pugh_SetupGH
+ (
+ tFleshConfig *config,
+ int convergence_level,
+ cGH *GH
+ )
{
DECLARE_CCTK_PARAMETERS
+ int group;
+ int dim;
+ int *nsize;
+ int *ghostsize;
+ int *groupghostsize;
+ int *groupsize;
+ int dir;
cGroup pgroup;
pGH *newGH;
- int group;
- int dim;
- int *nsize;
- int *ghostsize;
- int dir;
+
+ /* Set up the GH */
dim = CCTK_MaxDim();
- nsize = (int *)malloc(dim*sizeof(int));
+ nsize = (int *)malloc(dim*sizeof(int));
+ pugh_GFSize(dim,nsize);
+
ghostsize = (int *)malloc(dim*sizeof(int));
+ pugh_GFGhostsize(dim,ghostsize);
- if (global_nsize > 0)
+ newGH = PUGH_SetupPGH(GH, dim, nsize, ghostsize);
+ if (!newGH)
{
- for (dir=0;dir<dim;dir++)
- {
- nsize[dir] = global_nsize;
- }
+ CCTK_WARN(0,"Failed to allocate memory for a pGH");
}
- else
+
+ /* Set up groups on the GH */
+
+ for (group = 0; group < CCTK_NumGroups(); group++)
{
- if (local_nsize > 0)
+ CCTK_INT **size;
+ int i;
+
+ CCTK_GroupData(group,&pgroup);
+
+ /* Set the global size of the variables in the group */
+ size=CCTK_GroupSizesI(group);
+
+ if (size)
{
- for (dir=0;dir<dim;dir++)
+ groupsize = (int *)malloc(pgroup.dim*sizeof(int));
+ for (i=0;i<pgroup.dim;i++)
{
- nsize[dir] = -local_nsize;
+ groupsize[i]=*size[i];
}
}
else
{
- if (dim>0)
- {
- if (local_nx > 0)
- {
- nsize[0] = -local_nx;
- }
- else
- {
- nsize[0] = global_nx;
- }
- }
+ groupsize = NULL;
+ }
- if (dim>1)
- {
- if (local_ny > 0)
- {
- nsize[1] = -local_ny;
- }
- else
- {
- nsize[1] = global_ny;
- }
- }
+ /* Set the ghostzone size of the variables in the group */
+ size=CCTK_GroupGhostsizesI(group);
- if (dim>2)
+ if (size)
+ {
+ groupghostsize = (int *)malloc(pgroup.dim*sizeof(int));
+ for (i=0;i<pgroup.dim;i++)
{
- if (local_nz > 0)
- {
- nsize[2] = -local_nz;
- }
- else
- {
- nsize[2] = global_nz;
- }
+ groupghostsize[i]=*size[i];
}
}
- }
-
- if (ghost_size>-1)
- {
- for (dir=0;dir<dim;dir++)
- {
- ghostsize[dir] = ghost_size;
- }
- }
- else
- {
- switch(dim)
+ else
{
- case 3:
- ghostsize[2] = ghost_size_z;
- case 2:
- ghostsize[1] = ghost_size_y;
- case 1:
- ghostsize[0] = ghost_size_x;
- default:
- ;
+ groupghostsize = NULL;
}
- }
-
- newGH = SetupPGH(GH, dim, nsize, ghostsize);
-
- if(!newGH) CCTK_WARN(0,"PUGH: Failed to allocate memory for a pGH !\n");
-
- for(group = 0; group < CCTK_NumGroups(); group++)
- {
-
-#ifdef DEBUG_PUGH
- printf("Calling pugh_SetupGroup for %s [%d]\n",
- CCTK_GroupName(group),group);
-#endif
- CCTK_GroupData(group,&pgroup);
- pugh_SetupGroup(newGH,
+ PUGH_SetupGroup(newGH,
+ groupsize,
+ groupghostsize,
pgroup.grouptype,
pgroup.vartype,
pgroup.dim,
pgroup.numvars,
pgroup.stagtype,
pgroup.numtimelevels);
+
+ if (groupsize) free(groupsize);
+ if (groupghostsize) free(groupghostsize);
}
free(nsize);
@@ -171,15 +154,17 @@ int pugh_InitGH(cGH *GH)
int pugh_rfrTraverseGH(cGH *GH, int rfrpoint)
{
- int var;
- int gtype;
+ int var;
+ int gtype;
+ int ntimelevels;
+ int level;
+ int idir;
+ int istag;
pGH *mypGH;
- int ntimelevels;
- int level,idir,istag;
mypGH = (pGH *)GH->extensions[PUGH_GHExtension];
- /* Fix me */
+ /* FIXME */
GH->cctk_convlevel = 1;
for (idir=0;idir<GH->cctk_dim;idir++)
@@ -190,7 +175,7 @@ int pugh_rfrTraverseGH(cGH *GH, int rfrpoint)
GH->cctk_gsh[idir] = mypGH->nsize[idir];
GH->cctk_bbox[2*idir] = mypGH->lb[mypGH->myproc][idir] == 0;
GH->cctk_bbox[2*idir+1] = mypGH->ub[mypGH->myproc][idir]
- == mypGH->nsize[idir]-1;
+ == mypGH->nsize[idir]-1;
GH->cctk_lbnd[idir] = mypGH->lb[mypGH->myproc][idir];
GH->cctk_ubnd[idir] = mypGH->ub[mypGH->myproc][idir];
@@ -221,14 +206,19 @@ int pugh_rfrTraverseGH(cGH *GH, int rfrpoint)
{
switch(gtype)
{
- case GROUP_SCALAR :
- GH->data[var][level] = mypGH->variables[var][level]; break;
- case GROUP_GF :
- GH->data[var][level] = ((pGF ***)(mypGH->variables))[var][level]->data; break;
- case GROUP_ARRAY :
- GH->data[var][level] = NULL; break;
+ case CCTK_SCALAR :
+ GH->data[var][level] = mypGH->variables[var][level];
+ break;
+ case CCTK_GF :
+ GH->data[var][level] =
+ ((pGF ***)(mypGH->variables))[var][level]->data;
+ break;
+ case CCTK_ARRAY :
+ GH->data[var][level] =
+ ((pGA ***)(mypGH->variables))[var][level]->data;
+ break;
default:
- CCTK_WARN(1,"Unknown group type in pugh_rfrTraverse\n");
+ CCTK_WARN(1,"Unknown group type in pugh_rfrTraverse");
}
}
}
@@ -237,3 +227,102 @@ int pugh_rfrTraverseGH(cGH *GH, int rfrpoint)
return 0;
}
+
+int pugh_GFSize(int dim, int *nsize)
+{
+
+ DECLARE_CCTK_PARAMETERS
+
+ int dir;
+
+ if (global_nsize > 0)
+ {
+ for (dir=0;dir<dim;dir++)
+ {
+ nsize[dir] = global_nsize;
+ }
+ }
+ else
+ {
+ if (local_nsize > 0)
+ {
+ for (dir=0;dir<dim;dir++)
+ {
+ nsize[dir] = -local_nsize;
+ }
+ }
+ else
+ {
+ if (dim>0)
+ {
+ if (local_nx > 0)
+ {
+ nsize[0] = -local_nx;
+ }
+ else
+ {
+ nsize[0] = global_nx;
+ }
+ }
+
+ if (dim>1)
+ {
+ if (local_ny > 0)
+ {
+ nsize[1] = -local_ny;
+ }
+ else
+ {
+ nsize[1] = global_ny;
+ }
+ }
+
+ if (dim>2)
+ {
+ if (local_nz > 0)
+ {
+ nsize[2] = -local_nz;
+ }
+ else
+ {
+ nsize[2] = global_nz;
+ }
+ }
+ }
+ }
+ USE_CCTK_PARAMETERS
+}
+
+int pugh_GFGhostsize(int dim, int *ghostsize)
+{
+
+ DECLARE_CCTK_PARAMETERS
+
+ int dir;
+
+ if (ghost_size>-1)
+ {
+ for (dir=0;dir<dim;dir++)
+ {
+ ghostsize[dir] = ghost_size;
+ }
+ }
+ else
+ {
+ switch(dim)
+ {
+ case 3:
+ ghostsize[2] = ghost_size_z;
+ ghostsize[1] = ghost_size_y;
+ ghostsize[0] = ghost_size_x;
+ case 2:
+ ghostsize[1] = ghost_size_y;
+ ghostsize[0] = ghost_size_x;
+ case 1:
+ ghostsize[0] = ghost_size_x;
+ default:
+ ghostsize = NULL;
+ }
+ }
+ USE_CCTK_PARAMETERS
+}
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index e6ad426..de5d508 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -15,64 +15,48 @@
static char *rcisd = "$Header$";
-pGF *SetupPGF(pGH *GH,
- const char *name,
- int dim,
- int varsize,
- int vtype,
- int staggercode);
-
-int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timelevels);
-int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables,
- int staggergroup, int n_timelevels);
-int pugh_SetupGFGroup(pGH *newGH, int vtype, int dim, int n_variables,
- int staggercode, int n_timelevels);
+pGExtras *pugh_SetupPGExtras
+ (
+ int dim,
+ int periodic,
+ int stagger,
+ int *sh,
+ int *nghosts,
+ int total_procs,
+ int *nprocs,
+ int this_proc
+ );
+
+pConnectivity *pugh_SetupConnectivity
+ (
+ int dim,
+ int total_procs,
+ int *nprocs,
+ int periodic
+ );
+
+pGF *SetupPGF
+ (
+ pGH *GH,
+ const char *name,
+ int dim,
+ int varsize,
+ int vtype,
+ int staggercode
+ );
+
+pGA *SetupPGA
+ (
+ void *parent,
+ pGExtras *extras,
+ pConnectivity *connectivity,
+ const char *name,
+ int id,
+ int varsize,
+ int vtype,
+ int stagger
+ );
- /*@@
- @routine pugh_SetupGroup
- @date Mon Feb 8 19:37:55 1999
- @author Tom Goodale
- @desc
- Sets up a group on a pGH
- @enddesc
- @calls
- @calledby
- @history
-
- @endhistory
-
-@@*/
-int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables,
- int staggercode, int n_timelevels)
-{
- int returncode;
- switch(gtype)
- {
- case GROUP_SCALAR : returncode = pugh_SetupScalarGroup(newGH,
- vtype,
- n_variables,
- n_timelevels);
- break;
- case GROUP_ARRAY : returncode = pugh_SetupArrayGroup(newGH,
- vtype,
- dim,
- n_variables,
- staggercode,
- n_timelevels);
- break;
- case GROUP_GF : returncode = pugh_SetupGFGroup(newGH,
- vtype,
- dim,
- n_variables,
- staggercode,
- n_timelevels);
- break;
- default : CCTK_WARN(0, "Unknown group type in pugh_SetupGroup");
- returncode = 1;
- }
-
- return returncode;
-}
/*@@
@routine pugh_SetupScalarGroup
@@ -88,81 +72,23 @@ int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables,
@endhistory
@@*/
-int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timelevels)
+
+int pugh_SetupScalarGroup
+ (
+ pGH *newGH,
+ int vtype,
+ int n_variables,
+ int n_timelevels
+ )
{
+ int retval=PUGH_SUCCESS;
int variable;
- char *msg;
- int var_size;
-
+ int var_size = CCTK_VarTypeSize(vtype);
int level;
void ***temp;
- int returncode;
-
- switch(vtype)
- {
- case CCTK_VARIABLE_CHAR:
- var_size = sizeof(CCTK_CHAR);
- break;
-
- case CCTK_VARIABLE_INT:
- var_size = sizeof(CCTK_INT) ;
- break;
-
-#ifdef CCTK_INT2
- case CCTK_VARIABLE_INT2:
- var_size = sizeof(CCTK_INT2);
- break;
-#endif
-
-#ifdef CCTK_INT4
- case CCTK_VARIABLE_INT4:
- var_size = sizeof(CCTK_INT4);
- break;
-#endif
-
-#ifdef CCTK_INT8
- case CCTK_VARIABLE_INT8:
- var_size = sizeof(CCTK_INT8);
- break;
-#endif
-
- case CCTK_VARIABLE_REAL:
- var_size = sizeof(CCTK_REAL);
- break;
-
-#ifdef CCTK_REAL4
- case CCTK_VARIABLE_REAL4:
- var_size = sizeof(CCTK_REAL4);
- break;
-#endif
-
-#ifdef CCTK_REAL8
- case CCTK_VARIABLE_REAL8:
- var_size = sizeof(CCTK_REAL8);
- break;
-#endif
-
-#ifdef CCTK_REAL16
- case CCTK_VARIABLE_REAL16:
- var_size = sizeof(CCTK_REAL16);
- break;
-#endif
-
- case CCTK_VARIABLE_COMPLEX:
- var_size = sizeof(CCTK_COMPLEX);
- break;
-
- default:
- msg = malloc( 200*sizeof(char) );
- sprintf(msg,"Unknown variable type (%d) in pugh_SetupScalarGroup",vtype);
- CCTK_WARN(0,msg);
- free(msg);
- var_size = 1;
-
- }
-
- temp = (void ***)realloc(newGH->variables, (newGH->nvariables+n_variables)*sizeof(void **));
+ temp = (void ***)realloc
+ (newGH->variables, (newGH->nvariables+n_variables)*sizeof(void **));
if(temp)
{
@@ -170,7 +96,8 @@ int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timeleve
for(variable = 0; variable < n_variables; variable++)
{
- newGH->variables[newGH->nvariables] = (void **)malloc(n_timelevels*sizeof(void *));
+ newGH->variables[newGH->nvariables] =
+ (void **)malloc(n_timelevels*sizeof(void *));
if(newGH->variables[newGH->nvariables])
{
@@ -182,38 +109,97 @@ int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timeleve
}
else
{
- CCTK_WARN(1," Memory A error in SetupScalarGroup: ");
+ CCTK_WARN(1,"Memory error in SetupScalarGroup");
+ retval = PUGH_ERRORMEMORY;
break;
}
}
}
else
{
- CCTK_WARN(1," Memory A error in SetupScalarGroup: ");
+ CCTK_WARN(1,"Memory error in SetupScalarGroup");
+ retval = PUGH_ERRORMEMORY;
}
- return 0;
+ return retval;
}
-int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables,
- int staggercode, int n_timelevels)
-{
- int variable;
- char *msg;
- int var_size;
- void ***temp;
+ /*@@
+ @routine pugh_SetupArrayGroup
+ @date January 19 2000
+ @author Gabrielle Allen
+ @desc
+ Set up a group of array variables on a pGH.
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
- int returncode;
+@@*/
- int level;
+int pugh_SetupArrayGroup
+ (
+ pGH *newGH,
+ int *nsize,
+ int *ghostsize,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggercode,
+ int n_timelevels
+ )
+{
+
+ int i;
+ int variable;
+ int var_size = CCTK_VarTypeSize(vtype);
+ int retval;
+ int level;
+ char *msg;
+ void ***temp;
+ pConnectivity *connectivity;
+ pGExtras *extras;
+
+ int periodic=0;
+ int *nprocs;
+ int *nghostzones;
+
+ /* FIXME: Arrays can't have manual set up yet GAB */
+ nprocs = (int *) malloc(dim*sizeof(int));
+ for (i=0;i<dim;i++)
+ {
+ nprocs[i]=0;
+ }
- var_size = CCTK_VarTypeSize(vtype);
+ connectivity = pugh_SetupConnectivity(dim,newGH->nprocs,nprocs,periodic);
+
+ /* FIXME: For now default ghostzones 0 for array GAB */
+ nghostzones = (int *)malloc(dim*sizeof(int));
+ for (i=0;i<dim;i++)
+ {
+ nghostzones[i]=0;
+ }
- /* Call Extras and (connectivity) */
+ extras = pugh_SetupPGExtras
+ (
+ dim,
+ periodic,
+ staggercode,
+ nsize,
+ nghostzones,
+ newGH->nprocs,
+ connectivity->nprocs,
+ newGH->myproc
+ );
- temp = (void ***)realloc(newGH->variables,
- (newGH->nvariables+n_variables)*sizeof(void **));
+ free(nprocs);
+ free(nghostzones);
+
+ temp = (void ***)realloc
+ (newGH->variables,(newGH->nvariables+n_variables)*sizeof(void **));
if(temp)
{
@@ -228,25 +214,38 @@ int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables,
{
for(level = 0; level < n_timelevels; level++)
{
- /* newGH->variables[newGH->nvariables][level] =
- SetupPGV(newGH,
- CCTK_VarName(newGH->nvariables),
- dim,
- var_size,
- vtype,
- PUGH_VERTEXCTR);*/
+ newGH->variables[newGH->nvariables][level] = SetupPGA
+ (
+ newGH,
+ extras,
+ connectivity,
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode
+ );
}
newGH->nvariables++;
}
+ else
+ {
+ CCTK_WARN(1,"Memory error in SetupArrayGroup");
+ retval = PUGH_ERRORMEMORY;
+ break;
+ }
}
}
-
- fprintf(stderr, "PUGH doesn't yet support array groups, sorry !\n");
- exit(1);
-
- return 0;
+ else
+ {
+ CCTK_WARN(1,"Memory error in SetupArrayGroup");
+ retval = PUGH_ERRORMEMORY;
+ }
+
+ return retval;
}
+
/*@@
@routine pugh_SetupGFGroup
@date Wed Feb 17 04:45:49 1999
@@ -270,18 +269,12 @@ int pugh_SetupGFGroup(pGH *newGH,
{
int variable;
+ int var_size=CCTK_VarTypeSize(vtype);
+ int retval=PUGH_SUCCESS;
+ int level;
char *msg;
- int var_size;
-
void ***temp;
- int returncode;
-
- int level;
-
-
- var_size = CCTK_VarTypeSize(vtype);
-
temp = (void ***)realloc(newGH->variables,
(newGH->nvariables+n_variables)*sizeof(void **));
@@ -305,10 +298,109 @@ int pugh_SetupGFGroup(pGH *newGH,
var_size,
vtype,
staggercode);
+ /* newGH->variables[newGH->nvariables][level] =
+ SetupPGA(newGH,
+ newGH->GFExtras[dim-1],
+ newGH->Connectivity[dim-1],
+ CCTK_VarName(newGH->nvariables),
+ newGH->nvariables,
+ var_size,
+ vtype,
+ staggercode);*/
}
newGH->nvariables++;
}
+ else
+ {
+ CCTK_WARN(1,"Memory error in SetupGFGroup");
+ retval = PUGH_ERRORMEMORY;
+ break;
+ }
}
}
- return 0;
+ else
+ {
+ CCTK_WARN(1,"Memory error in SetupGFGroup");
+ retval = PUGH_ERRORMEMORY;
+ }
+
+ return retval;
+
+}
+
+
+ /*@@
+ @routine pugh_SetupGroup
+ @date Mon Feb 8 19:37:55 1999
+ @author Tom Goodale
+ @desc
+ Sets up a group on a pGH
+ @enddesc
+ @calls
+ @calledby
+ @history
+
+ @endhistory
+
+@@*/
+
+int PUGH_SetupGroup
+ (
+ pGH *newGH,
+ int *nsize,
+ int *nghostsize,
+ int gtype,
+ int vtype,
+ int dim,
+ int n_variables,
+ int staggercode,
+ int n_timelevels
+ )
+{
+
+ int returncode=PUGH_SUCCESS;
+
+ switch(gtype)
+ {
+ case CCTK_SCALAR :
+ returncode = pugh_SetupScalarGroup
+ (
+ newGH,
+ vtype,
+ n_variables,
+ n_timelevels
+ );
+ break;
+ case CCTK_ARRAY :
+ returncode = pugh_SetupArrayGroup
+ (
+ newGH,
+ nsize,
+ nghostsize,
+ vtype,
+ dim,
+ n_variables,
+ staggercode,
+ n_timelevels
+ );
+ break;
+ case CCTK_GF :
+ returncode = pugh_SetupGFGroup
+ (
+ newGH,
+ vtype,
+ dim,
+ n_variables,
+ staggercode,
+ n_timelevels
+ );
+ break;
+ default :
+ CCTK_WARN(0, "Unknown group type in pugh_SetupGroup");
+ returncode = PUGH_ERROR;
+ }
+
+ return returncode;
+
}
+
diff --git a/src/SetupPGF.c b/src/SetupPGF.c
index 8592d0f..220f3c8 100644
--- a/src/SetupPGF.c
+++ b/src/SetupPGF.c
@@ -129,7 +129,6 @@ pGF *SetupPGF(pGH *GH, const char *name, int dim, int varsize, int vtype, int st
{
res->storage = PUGH_NOSTORAGE; /* Fake it out... */
EnableGFDataStorage(GH, res);
- printf("Enabling storage in pugh\n");
}
else
{
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index ff4a57d..1ee5559 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -25,25 +25,17 @@ void Neighbours1D(pGH *GH, int *nproc, int *maxpoints, int *minpoints, double *a
void Neighbours3D(pGH *GH, int *nproc, int *maxpoints, int *minpoints, double *avgpoints);
void DestroyPGF(pGH *GH, pGF **GFin);
void pGH_DumpInfo(pGH *GH);
+int SetupDefaultTopology(int dim, int *nprocs);
+int pGH_SetupnProcs(pGH *GH,int dim);
/*@@
- @routine SetupPGH
+ @routine PUGH_SetupPGH
@date Fri Feb 21 10:21:36 1997
- @author Paul Walker
+ @author Gabrielle Allen, Tom Goodale, Paul Walker
@desc
Sets up the PGH distribution based on processor number and
- problem size. This amounts to:
- <ul>
- <li> Allocating and copying some stuff
- <li> Figuring out an optimal domain decomposition
- <li> Setting up ghosts and overlaps.
- </ul>
- Of course the latter two are hard, so some more comments
- are in order.
- <p>
- The second is in @seeroutine ProcTop now.
- <p>
- The third should be obvious ;-)
+ problem size. This includes setting an optimal domain
+ decomposition for each dimension, and setting ghosts and overlaps.
@enddesc
@hdate 4 June 1999
@hauthor Thomas Radke
@@ -52,21 +44,89 @@ void pGH_DumpInfo(pGH *GH);
@hdate 30 June 1999
@@*/
-pGH *SetupPGH(void *callerid, int dim, int *nsize, int *nghostzones,int staggertype)
+pGH *PUGH_SetupPGH(void *callerid,
+ int dim,
+ int *nsize,
+ int *nghostzones,
+ int staggertype)
{
pGH *GH;
+ int idim;
void pGH_SetupBasics(pGH *, int, int *, int *, int);
void pGH_SetupBounds(pGH *GH);
void pGH_SetupOwnership(pGH *,int dim);
void pGH_SetupStaggering(pGH *GH, int dim);
+ pConnectivity *pugh_SetupConnectivity
+ (
+ int dim,
+ int total_procs,
+ int *nprocs,
+ int periodic
+ );
+
+ pGExtras *pugh_SetupPGExtras
+ (
+ int dim,
+ int periodic,
+ int stagger,
+ int *sh,
+ int *nghosts,
+ int total_procs,
+ int *nprocs,
+ int this_proc
+ );
+
/* Allocate for myself */
GH = (pGH*) malloc(sizeof(pGH));
/* Set an identifier for my parent */
GH->callerid = callerid ;
+ GH->GFExtras = (pGExtras **)malloc(dim*sizeof(pGExtras*));
+ GH->Connectivity = (pConnectivity **)malloc(dim*sizeof(pConnectivity*));
+
+ /* Set the total number of processors */
+ pGH_SetupnProcs(GH,dim);
+
+ /* Set up connectivity and extras for each dimension */
+ for (idim=1;idim<=dim;idim++)
+ {
+ int periodic=0;
+ int *nprocs;
+
+ nprocs = (int *) malloc(idim*sizeof(int));
+
+ SetupDefaultTopology(idim,nprocs);
+
+ GH->Connectivity[idim-1]=pugh_SetupConnectivity
+ (
+ idim,
+ GH->nprocs,
+ nprocs,
+ periodic
+ );
+
+ /* FIXME : Should not be on GH */
+ GH->nproc = ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs;
+
+ GH->GFExtras[idim-1]=pugh_SetupPGExtras
+ (
+ idim,
+ periodic,
+ staggertype,
+ nsize,
+ nghostzones,
+ GH->nprocs,
+ ((pConnectivity **)GH->Connectivity)[idim-1]->nprocs,
+ GH->myproc
+ );
+
+ free(nprocs);
+
+ }
+
pGH_SetupBasics(GH,dim,nsize,nghostzones,staggertype);
pGH_SetupBounds(GH);
pGH_SetupOwnership(GH,dim);
@@ -77,11 +137,7 @@ pGH *SetupPGH(void *callerid, int dim, int *nsize, int *nghostzones,int staggert
}
-void pGH_SetupBasics(pGH *GH,
- int dim,
- int *nsize,
- int *nghostzones,
- int staggertype)
+int pGH_SetupnProcs(pGH *GH, int dim)
{
int i,j,k,idir;
@@ -91,12 +147,8 @@ void pGH_SetupBasics(pGH *GH,
CCTK_REAL4 sum_sizes [2], compiled_sizes [2];
#endif
- void ProcTop(int nprocs, int dim, int *procs);
-
DECLARE_CCTK_PARAMETERS;
- nproc = (int *)malloc(dim*sizeof(int));
-
GH->dim = dim;
#ifdef MPI
@@ -135,13 +187,25 @@ void pGH_SetupBasics(pGH *GH,
GH->myproc = 0;
#endif
- /* Activate GH (conv.grids can be inactivatet by nancheck so the evolution will
- carry on) */
+}
+
+void pGH_SetupBasics(pGH *GH,
+ int dim,
+ int *nsize,
+ int *nghostzones,
+ int staggertype)
+{
+
+ int i,j,k,idir;
+
+ DECLARE_CCTK_PARAMETERS;
+
+ /* Activate GH (conv.grids can be inactivated by nancheck
+ so the evolution will carry on) */
GH->active=1;
/* Set up arrays for ownership,overlap and ghosts */
for (i=0;i<PUGH_NSTAGGER;i++)
- {
for (j=0;j<2;j++)
{
GH->ownership[i][j] = (int *)malloc(dim*sizeof(int));
@@ -151,43 +215,10 @@ void pGH_SetupBasics(pGH *GH,
GH->overlap[i][j] = (int **)malloc(2*dim*sizeof(int *));
for (k=0;k<2*dim;k++)
GH->overlap[i][j][k] = (int *)malloc(dim*sizeof(int));
-
}
- }
- /* Processor topology to have constant work per processor */
- ProcTop(GH->nprocs,dim,nproc);
-
- GH->nproc = (int *)malloc(dim*sizeof(int));
- for (idir=0;idir<dim;idir++)
- GH->nproc[idir] = nproc[idir];
- free(nproc);
-
- /* Set up the global size, using cheap method */
- GH->nsize = (int *)malloc(dim*sizeof(int));
- for (idir=0;idir<dim;idir++)
- {
- if (nsize[idir] < 0 && GH->nproc[idir] > 1)
- {
- GH->nsize[idir] = (GH->nproc[idir]-2) *
- (-nsize[idir] - 2*nghostzones[idir]) +
- 2 * (-nsize[idir] - nghostzones[idir]);
- if (staggertype == PUGH_STAGGER)
- {
- GH->nsize[idir] -= GH->nproc[idir]-1;
- }
- }
- else
- {
- GH->nsize[idir] = abs(nsize[idir]);
- }
- }
-
- GH->nghostzones = (int *)malloc(dim*sizeof(int));
- for (idir=0;idir<dim;idir++)
- {
- GH->nghostzones[idir] = nghostzones[idir];
- }
+ GH->nsize = GH->GFExtras[dim-1]->nsize;
+ GH->nghostzones = GH->GFExtras[dim-1]->nghostzones;
/*
if (Contains("commmodel","loc") != 0) {
@@ -274,7 +305,6 @@ void pGH_SetupBounds(pGH *GH)
-
void pGH_SetupOwnership(pGH *GH,int dim) {
int idir,i,j,k,dir,staglcv;
int *one;
@@ -317,16 +347,12 @@ void pGH_SetupOwnership(pGH *GH,int dim) {
}
avgpoints = (1.0 * avgpoints) / (GH->nprocs);
+ GH->maxskew = (maxpoints-minpoints)/avgpoints * 100.0;
/* Local information */
- GH->npoints = GH->rnpoints[GH->myproc];
+ GH->npoints = GH->GFExtras[dim-1]->npoints;
+ GH->lnsize = GH->GFExtras[dim-1]->lnsize;
- GH->lnsize = (int *)malloc(dim*sizeof(int));
- for (i=0;i<dim;i++)
- {
- GH->lnsize[i] = GH->rnsize[GH->myproc][i];
- }
- GH->maxskew = (maxpoints-minpoints)/avgpoints * 100.0;
/* So now set up the ownership, ghosts, and overlap */
@@ -364,7 +390,6 @@ void pGH_SetupOwnership(pGH *GH,int dim) {
}
}
-
/* Great now set up the overlap region. This is the region we own,
but which is covered by our neighbors ghost. So this is what we
want to SEND
@@ -736,6 +761,9 @@ void DestroyPGH(pGH **GHin)
DestroyPGF(GH, &(((pGF ***)GH->variables)[variable][i]));
}
break;
+ case CCTK_ARRAY:
+ CCTK_WARN(1,"Need to add code for Destroying an Pugh array");
+ break;
case GROUP_SCALAR:
for(i = 0; i < pgroup.numtimelevels; i++)
{
@@ -767,7 +795,8 @@ void DestroyPGH(pGH **GHin)
fflush(stdout);
}
-void pGH_DumpInfo(pGH *GH) {
+void pGH_DumpInfo(pGH *GH)
+{
int i,j,k;
printf("INFO: \n");
printf(" myproc: %d/%d \n",GH->myproc,GH->nprocs);
@@ -787,3 +816,51 @@ void pGH_DumpInfo(pGH *GH) {
printf(" GH->overlap[%d][0/1][%d][%d]: %d %d \n",
i,k,j,GH->overlap[i][0][j][k],GH->overlap[i][1][k][j]);
}
+
+
+ /*@@
+ @routine SetupDefaultTopology
+ @date Sun 23 Jan
+ @author Gabrielle Allen
+ @desc
+ Use PUGH parameters to set the processor decomposition
+ is required
+ @enddesc
+ @hdate
+ @hauthor
+ @hdesc
+ @hdate
+@@*/
+
+int SetupDefaultTopology(int dim, int *nprocs)
+{
+
+ DECLARE_CCTK_PARAMETERS
+
+ int retval=PUGH_SUCCESS;
+
+ switch(dim)
+ {
+ case 1:
+ nprocs[0] = processor_topology_1d_x;
+ break;
+ case 2:
+ nprocs[0] = processor_topology_2d_x;
+ nprocs[1] = processor_topology_2d_y;
+ break;
+ case 3:
+ nprocs[0] = processor_topology_3d_x;
+ nprocs[1] = processor_topology_3d_y;
+ nprocs[2] = processor_topology_3d_z;
+ break;
+ default:
+ nprocs=NULL;
+ retval=PUGH_ERROR;
+ }
+
+ return retval;
+
+ USE_CCTK_PARAMETERS
+
+}
+
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index b1187d2..3b1d334 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -9,6 +9,8 @@
static char *rcsid = "$Header$";
+/*#define DEBUG_PUGH*/
+
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
@@ -94,7 +96,7 @@ pGA *SetupPGA(void *parent,
int id,
int varsize,
int vtype,
- int stagger) ;
+ int stagger);
int pugh_EnablePGAStorage(pGA *GA,
int this_proc,
@@ -263,6 +265,7 @@ int pugh_GenerateTopology(int dim, int total_procs, int *nprocs)
int free_dims;
used_procs = 0;
+ free_procs = total_procs;
retval = 0;
@@ -270,7 +273,7 @@ int pugh_GenerateTopology(int dim, int total_procs, int *nprocs)
for(i=0; i < dim; i++)
{
- if((nprocs[i]))
+ if((nprocs[i])>0)
{
free_dims--;
if(used_procs)
@@ -281,37 +284,39 @@ int pugh_GenerateTopology(int dim, int total_procs, int *nprocs)
{
used_procs = nprocs[i];
}
+ if (total_procs%used_procs)
+ {
+ CCTK_WARN(0, "Inconsistent PUGH topology");
+ retval = 1;
+ }
+ else
+ {
+ free_procs = total_procs/used_procs;
+ }
}
}
- if(total_procs%used_procs)
- {
- fprintf(stderr, "Error: Inconsistent PUGH topology\n");
- retval = 1;
- }
-
- free_procs = total_procs/used_procs;
-
/* Ok calculate topology if necessary */
if(free_dims && ! retval)
{
- if(total_procs%used_procs)
- {
- fprintf(stderr, "Error: Inconsistent PUGH topology\n");
- retval = 1;
- }
-
- free_procs = total_procs/used_procs;
-
/* This algorithm gives the most number of processors
- * in the highest diemsion.
+ * in the highest dimension.
*/
+
+#ifdef DEBUG_PUGH
+ printf("Processor topology for dimension %d\n",dim);
+#endif
+
for(i = 0; i < dim; i++)
{
- if(!nprocs[i])
+ if(nprocs[i]==0)
{
nprocs[i] = pow(free_procs, 1.0/free_dims);
+#ifdef DEBUG_PUGH
+ printf(" nprocs[%d] = %d\n",i,nprocs[i]);
+#endif
+
while(free_procs % nprocs[i]) nprocs[i]--;
free_procs /= nprocs[i];
@@ -1069,6 +1074,7 @@ int pugh_SetupBoundingBox(int dim,
for(pnum = 0; pnum < total_procs; pnum++)
{
+
pugh_DecomposeIJK(dim, pnum,nprocs, pos);
for(dir = 0 ; dir < dim; dir++)
@@ -1439,3 +1445,80 @@ int pugh_DisableGADataStorage(pGA *GA)
return retval;
}
+
+
+ /*@@
+ @routine pugh_GAComm
+ @date Sun Jan 23 12:46:23 2000
+ @author Gabrielle Allen
+ @desc
+ This sets the docomm[2*dim] array of the GA based
+ on the setting of the comm flag.
+ @enddesc
+@@*/
+
+
+void pugh_GAComm(pGA *GA, int docomm)
+{
+ int idir;
+ int dim = GA->extras->dim;
+
+ /* Copy docomm */
+ GA->commflag = docomm;
+
+ /* First set all communcation off */
+ for (idir=0;idir<2*dim;idir++)
+ GA->docomm[idir] = 0;
+
+ if (docomm == PUGH_NOCOMM)
+ {
+ for (idir=0;idir<2*dim;idir++)
+ GA->docomm[idir] = 0;
+ }
+ else if (docomm == PUGH_ALLCOMM)
+ {
+ for (idir=0;idir<2*dim;idir++)
+ GA->docomm[idir] = 1;
+ }
+ else if (docomm == PUGH_PLUSFACESCOMM)
+ {
+ for (idir=0;idir<dim;idir++)
+ {
+ GA->docomm[2*idir+1] = 1;
+ }
+ }
+ else if (docomm == PUGH_MINUSFACESCOMM)
+ {
+ for (idir=0;idir<dim;idir++)
+ {
+ GA->docomm[2*idir] = 1;
+ }
+ }
+ else
+ {
+ for (idir=0;idir<dim;idir++)
+ {
+ if (docomm == PUGH_COMM(idir))
+ {
+ GA->docomm[2*idir] = 1;
+ GA->docomm[2*idir+1] = 1;
+ }
+ }
+ }
+
+ /* FIXME Add back the check that you have a valid COMM model: Gab */
+
+ /* Handle nsize = 1 type cases. This is only important for one
+ processor MPI periodic boundaries */
+
+ for (idir=0;idir<dim;idir++)
+ {
+ if (GA->extras->nsize[idir] == 1)
+ {
+ GA->docomm[2*idir] = 0;
+ GA->docomm[2*idir+1] = 0;
+ }
+ }
+
+}
+
diff --git a/src/include/pGH.h b/src/include/pGH.h
index 450bc35..f0b9b9f 100644
--- a/src/include/pGH.h
+++ b/src/include/pGH.h
@@ -123,8 +123,9 @@ typedef struct PGH
MPI_Datatype recv_complex_dt [PUGH_NSTAGGER][6];
#endif
- pGExtras **GFExtras;
- pConnectivity **Connectivity;
+ /* Used for all grid functions */
+ pGExtras **GFExtras; /* [dim] stagger ? */
+ pConnectivity **Connectivity; /* [dim] */
/* #include "pGH_Extensions.h" */
diff --git a/src/include/pGV.h b/src/include/pGV.h
index 51bfa03..e8dc3e7 100644
--- a/src/include/pGV.h
+++ b/src/include/pGV.h
@@ -65,16 +65,16 @@ typedef struct PGExtras
typedef struct PGA
{
char *name; /* The name of the grid function */
- int id; /* My ID number in my GH parent. */
+ int id; /* My ID number in my GH parent. */
void *padddata; /* Storage for the data. */
void *data; /* See the note above. */
/* buffer_sz[2*dim] */
- int *buffer_sz; /* Size of the face ghost zones */
+ int *buffer_sz; /* Size of the face ghost zones */
void **send_buffer; /* Storage for buffered Comm if */
void **recv_buffer; /* we don't use derived types */
int commflag; /* What is the comm flag set to? */
/* do_comm[2*dim] */
- int *docomm; /* Do we do comm or not? */
+ int *docomm; /* Do we do comm or not? */
int storage; /* Do we have storage or not? */
int stagger; /* Only Vertex Centered now... */
diff --git a/src/include/pugh.h b/src/include/pugh.h
index 3f11292..214917b 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -14,7 +14,6 @@
#ifndef _PUGH_H_
#define _PUGH_H_
-/* include the system dependent defines */
#include "cctk.h"
/* FIXME */
diff --git a/src/include/pugh_constants.h b/src/include/pugh_constants.h
index fa77ca2..ba5fdbd 100644
--- a/src/include/pugh_constants.h
+++ b/src/include/pugh_constants.h
@@ -43,3 +43,8 @@
#define TERMINATION_NOT_RAISED 0 /* no termination flag, inital setting */
#define TERMINATION_RAISED_LOCAL 1 /* raised on one pe, has to be broadcasted */
#define TERMINATION_RAISED_BRDCAST 2 /* flag now available on all pes: termiante */
+
+/* PUGH error flags */
+#define PUGH_SUCCESS 0
+#define PUGH_ERROR -1
+#define PUGH_ERRORMEMORY -2
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-01 04:01:23 +0000