From 1c77c3689546128269a540049eac430a92fec400 Mon Sep 17 00:00:00 2001 From: schnetter Date: Mon, 26 Sep 2005 00:27:59 +0000 Subject: Improve periodic boundaries on small domains: allow the number of ghost zones to be larger than the number of interior points if there is only one processor in the corresponding direction. This is necessary to allow two ghost zones when there is only one interior grid point, which is often used to set up lower-dimensional domains. git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@469 b61c5cb5-eaca-4651-9a7a-d64986f99364 --- src/Comm.c | 1 + src/SetupGroup.c | 52 +++++++++++-------- src/SetupPGV.c | 28 ++++++---- src/Storage.c | 153 ++++++++++++++++++++++++++++++------------------------- 4 files changed, 133 insertions(+), 101 deletions(-) diff --git a/src/Comm.c b/src/Comm.c index bd7f453..3d04ae1 100644 --- a/src/Comm.c +++ b/src/Comm.c @@ -981,6 +981,7 @@ static int PUGH_SyncSingleProc(pGH *pughGH, pComm *comm) char *data; int *istart_from, *iend_from, *iterator_from; int *istart_to, *iterator_to; + int pass, num_passes, pass_total, pass_each, pass_from, pass_to; GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel]; diff --git a/src/SetupGroup.c b/src/SetupGroup.c index 7a93391..fa4a922 100644 --- a/src/SetupGroup.c +++ b/src/SetupGroup.c @@ -131,36 +131,44 @@ static int PUGH_SetupGAGroup (pGH *newGH, if (gtype == CCTK_ARRAY || gtype == CCTK_SCALAR) { - /* Arrays can't (yet) have periodicity and manual setup, - so initialize them to zero */ - perme = calloc (dim, sizeof (int)); - nprocs = calloc (dim, sizeof (int)); - if (dim && ! (perme && nprocs)) + if (n_variables > 0) { - CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup"); - } + /* Arrays can't (yet) have periodicity and manual setup, + so initialize them to zero */ + perme = calloc (dim, sizeof (int)); + nprocs = calloc (dim, sizeof (int)); + if (dim && ! (perme && nprocs)) + { + CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup"); + } - PUGH_SetupDefaultTopology (dim, nprocs); + PUGH_SetupDefaultTopology (dim, nprocs); - /* Check that there are enough grid points in this dimension - * to make parallelising it worthwhile - */ - for (i = 0 ; i < dim; i++) - { - if (abs (nsize[i]) <= 2*ghostsize[i]) + /* Check that there are enough grid points in this dimension + * to make parallelising it worthwhile + */ + for (i = 0 ; i < dim; i++) { - nprocs[i] = 1; + if (abs (nsize[i]) <= 2*ghostsize[i]) + { + nprocs[i] = 1; + } } - } - connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme); + connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme); - extras = PUGH_SetupPGExtras (0, dim, perme, staggercode, nsize, ghostsize, - newGH->nprocs, connectivity->nprocs, - connectivity->neighbours, newGH->myproc); + extras = PUGH_SetupPGExtras (0, dim, perme, staggercode, nsize, ghostsize, + newGH->nprocs, connectivity->nprocs, + connectivity->neighbours, newGH->myproc); - free (nprocs); - free (perme); + free (nprocs); + free (perme); + } + else + { + connectivity = NULL; + extras = NULL; + } } else { diff --git a/src/SetupPGV.c b/src/SetupPGV.c index 49c87ea..b01403b 100644 --- a/src/SetupPGV.c +++ b/src/SetupPGV.c @@ -846,7 +846,7 @@ static int PUGH_SetupPGExtrasSizes(int is_gf, } else { - this->nsize[dir] = abs(sh[dir]); + this->nsize[dir] = sh[dir]; } } @@ -944,14 +944,20 @@ static int PUGH_SetupPGExtrasOwnership(int dim, /* Do sanity check for periodic BC */ if(perme[dir]) { - if (this->ownership[PUGH_VERTEXCTR][1][dir] - - this->ownership[PUGH_VERTEXCTR][0][dir] < this->nghostzones[dir]) + const int is_single_proc_in_dim + = (this->lb[this_proc][dir] == 0 + && this->ub[this_proc][dir] == this->nsize[dir]-1); + if (! is_single_proc_in_dim + && this->ownership[PUGH_VERTEXCTR][1][dir] - + this->ownership[PUGH_VERTEXCTR][0][dir] < this->nghostzones[dir]) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, "Not enough interior grid points in dimension %d ! " "Please set the number of global grid points (currently " "%d) to something >= (nprocs+2) * the number of ghostzones " - "(currently (%d+2)*%d=%d) !", + "(currently (%d+2)*%d=%d), " + "or make sure that there is only one processor in this " + "dimension.", dir, this->nsize[dir], nprocs, this->nghostzones[dir], (nprocs+2) * this->nghostzones[dir]); } @@ -1256,15 +1262,15 @@ pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH, #endif this->docomm = malloc (2 * dim * sizeof (int)); - if(this->buffer_sz && - this->send_buffer && - this->recv_buffer && + if((dim==0 || this->buffer_sz) && + (dim==0 || this->send_buffer) && + (dim==0 || this->recv_buffer) && #ifdef CCTK_MPI - this->sreq && - this->rreq && - this->sstatus && + (dim==0 || this->sreq) && + (dim==0 || this->rreq) && + (dim==0 || this->sstatus) && #endif - this->docomm) + (dim==0 || this->docomm)) { this->commflag = PUGH_NOCOMM; this->first_var = first_var; diff --git a/src/Storage.c b/src/Storage.c index c3f8b6f..066fe18 100644 --- a/src/Storage.c +++ b/src/Storage.c @@ -316,43 +316,52 @@ int PUGH_EnableGroupStorage (const cGH *GH, const char *groupname) { first_var = CCTK_FirstVarIndexI (group); - /* get the group info from its index */ - CCTK_GroupData (group, &pgroup); - - retval = PUGH_EnableGArrayGroupStorage (pughGH, - first_var, - pgroup.numvars, - pgroup.numtimelevels); - if (!CCTK_Equals(storage_verbose,"no") && retval == 0) + if (first_var >= 0) { - /* get GA pointer of first var in group */ - GA = (pGA *) pughGH->variables[first_var][0]; - if (pgroup.grouptype == CCTK_GF) - { - totalnumberGF += pgroup.numvars * pgroup.numtimelevels; - } - else - { - totalnumberGA += pgroup.numvars * pgroup.numtimelevels; - } - totalstorage += (GA->extras->npoints * GA->varsize * - pgroup.numtimelevels * pgroup.numvars) / - (float) (1024*1024); - if (totalstorage > maxstorage) - { - numberGF = totalnumberGF; - numberGA = totalnumberGA; - maxstorage = totalstorage; - } - /* Report on memory usage */ - if (CCTK_Equals(storage_verbose,"yes")) + /* get the group info from its index */ + CCTK_GroupData (group, &pgroup); + + retval = PUGH_EnableGArrayGroupStorage (pughGH, + first_var, + pgroup.numvars, + pgroup.numtimelevels); + if (!CCTK_Equals(storage_verbose,"no") && retval == 0) { - CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'" - " [GFs: %d GAs: %d Total Size: %6.2fMB]", - groupname, totalnumberGF, totalnumberGA, totalstorage); + /* get GA pointer of first var in group */ + GA = (pGA *) pughGH->variables[first_var][0]; + if (pgroup.grouptype == CCTK_GF) + { + totalnumberGF += pgroup.numvars * pgroup.numtimelevels; + } + else + { + totalnumberGA += pgroup.numvars * pgroup.numtimelevels; + } + totalstorage += (GA->extras->npoints * GA->varsize * + pgroup.numtimelevels * pgroup.numvars) / + (float) (1024*1024); + if (totalstorage > maxstorage) + { + numberGF = totalnumberGF; + numberGA = totalnumberGA; + maxstorage = totalstorage; + } + + /* Report on memory usage */ + if (CCTK_Equals(storage_verbose,"yes")) + { + CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'" + " [GFs: %d GAs: %d Total Size: %6.2fMB]", + groupname, totalnumberGF, totalnumberGA, totalstorage); + } } } + else + { + /* don't know what to say here; 0 seems safe */ + retval = 0; + } } else @@ -956,28 +965,32 @@ int PUGH_GroupStorageIncrease(const cGH *GH, int n_groups,const int *groups,cons first_var = CCTK_FirstVarIndexI (groups[group]); - /* Enable all timelevels or just some */ - if(timelevels[group] == -1) + if (first_var >= 0) { - tlevels = pgroup.numtimelevels; - } - else - { - tlevels = timelevels[group]; - } - previous = PUGHi_EnableGArrayGroupStorage (pughGH, - first_var, - pgroup.numvars, - pgroup.numtimelevels, - tlevels); + /* Enable all timelevels or just some */ + if(timelevels[group] == -1) + { + tlevels = pgroup.numtimelevels; + } + else + { + tlevels = timelevels[group]; + } + + previous = PUGHi_EnableGArrayGroupStorage (pughGH, + first_var, + pgroup.numvars, + pgroup.numtimelevels, + tlevels); - if(status) - { - status[group] = previous; - } + if(status) + { + status[group] = previous; + } - retval += previous; + retval += previous; + } } } @@ -1066,28 +1079,32 @@ int PUGH_GroupStorageDecrease(const cGH *GH, int n_groups,const int *groups,cons first_var = CCTK_FirstVarIndexI (groups[group]); - /* Disable all timelevels or just some */ - if(timelevels[group] == -1) + if (first_var >= 0) { - tlevels = 0; - } - else - { - tlevels = timelevels[group]; - } - previous = PUGHi_DisableGArrayGroupStorage (pughGH, - first_var, - pgroup.numvars, - pgroup.numtimelevels, - tlevels); + /* Disable all timelevels or just some */ + if(timelevels[group] == -1) + { + tlevels = 0; + } + else + { + tlevels = timelevels[group]; + } - if(status) - { - status[group] = previous; - } + previous = PUGHi_DisableGArrayGroupStorage (pughGH, + first_var, + pgroup.numvars, + pgroup.numtimelevels, + tlevels); + + if(status) + { + status[group] = previous; + } - retval += previous; + retval += previous; + } } } -- cgit v1.2.3