Improve periodic boundaries on small domains: allow the number of

ghost zones to be larger than the number of interior points if there
is only one processor in the corresponding direction.  This is
necessary to allow two ghost zones when there is only one interior
grid point, which is often used to set up lower-dimensional domains.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@469 b61c5cb5-eaca-4651-9a7a-d64986f99364

4 files changed, 133 insertions, 101 deletions

diff --git a/src/Comm.c b/src/Comm.c
index bd7f453..3d04ae1 100644
--- a/src/Comm.c
+++ b/src/Comm.c
@@ -981,6 +981,7 @@ static int PUGH_SyncSingleProc(pGH *pughGH, pComm *comm)
   char *data;
   int *istart_from, *iend_from, *iterator_from;
   int *istart_to, *iterator_to;
+  int pass, num_passes, pass_total, pass_each, pass_from, pass_to;
 
 
   GA = (pGA *) pughGH->variables [comm->first_var][comm->sync_timelevel];
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index 7a93391..fa4a922 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -131,36 +131,44 @@ static int PUGH_SetupGAGroup (pGH *newGH,
 
   if (gtype == CCTK_ARRAY || gtype == CCTK_SCALAR)
   {
-    /* Arrays can't (yet) have periodicity and manual setup,
-       so initialize them to zero */
-    perme = calloc (dim, sizeof (int));
-    nprocs = calloc (dim, sizeof (int));
-    if (dim && ! (perme && nprocs))
+    if (n_variables > 0)
     {
-      CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
-    }
+      /* Arrays can't (yet) have periodicity and manual setup,
+         so initialize them to zero */
+      perme = calloc (dim, sizeof (int));
+      nprocs = calloc (dim, sizeof (int));
+      if (dim && ! (perme && nprocs))
+      {
+        CCTK_WARN (0, "Memory allocation error in PUGH_SetupGAGroup");
+      }
 
-    PUGH_SetupDefaultTopology (dim, nprocs);
+      PUGH_SetupDefaultTopology (dim, nprocs);
 
-    /* Check that there are enough grid points in this dimension
-     * to make parallelising it worthwhile
-     */
-    for (i = 0 ; i < dim; i++)
-    {
-      if (abs (nsize[i]) <= 2*ghostsize[i])
+      /* Check that there are enough grid points in this dimension
+       * to make parallelising it worthwhile
+       */
+      for (i = 0 ; i < dim; i++)
       {
-        nprocs[i] = 1;
+        if (abs (nsize[i]) <= 2*ghostsize[i])
+        {
+          nprocs[i] = 1;
+        }
       }
-    }
 
-    connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);
+      connectivity = PUGH_SetupConnectivity (dim, newGH->nprocs, nprocs, perme);
 
-    extras = PUGH_SetupPGExtras (0, dim, perme, staggercode, nsize, ghostsize,
-                                 newGH->nprocs, connectivity->nprocs, 
-                                 connectivity->neighbours, newGH->myproc);
+      extras = PUGH_SetupPGExtras (0, dim, perme, staggercode, nsize, ghostsize,
+                                   newGH->nprocs, connectivity->nprocs, 
+                                   connectivity->neighbours, newGH->myproc);
 
-    free (nprocs);
-    free (perme);
+      free (nprocs);
+      free (perme);
+    }
+    else
+    {
+      connectivity = NULL;
+      extras = NULL;
+    }
   }
   else
   {
diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 49c87ea..b01403b 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -846,7 +846,7 @@ static int PUGH_SetupPGExtrasSizes(int is_gf,
     }
     else
     {
-      this->nsize[dir] = abs(sh[dir]);
+      this->nsize[dir] = sh[dir];
     }
   }
 
@@ -944,14 +944,20 @@ static int PUGH_SetupPGExtrasOwnership(int dim,
     /* Do sanity check for periodic BC */
     if(perme[dir])
     {
-      if (this->ownership[PUGH_VERTEXCTR][1][dir] -
-          this->ownership[PUGH_VERTEXCTR][0][dir] < this->nghostzones[dir])
+      const int is_single_proc_in_dim
+	= (this->lb[this_proc][dir] == 0
+	   && this->ub[this_proc][dir] == this->nsize[dir]-1);
+      if (! is_single_proc_in_dim
+          && this->ownership[PUGH_VERTEXCTR][1][dir] -
+             this->ownership[PUGH_VERTEXCTR][0][dir] < this->nghostzones[dir])
       {
         CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                     "Not enough interior grid points in dimension %d ! "
                     "Please set the number of global grid points (currently "
                     "%d) to something >= (nprocs+2) * the number of ghostzones "
-                    "(currently (%d+2)*%d=%d) !",
+                    "(currently (%d+2)*%d=%d), "
+                    "or make sure that there is only one processor in this "
+                    "dimension.",
                     dir, this->nsize[dir], nprocs, this->nghostzones[dir],
                     (nprocs+2) * this->nghostzones[dir]);
       }
@@ -1256,15 +1262,15 @@ pComm *PUGH_SetupGArrayGroupComm(pGH *pughGH,
 #endif
     this->docomm      = malloc (2 * dim * sizeof (int));
 
-    if(this->buffer_sz   &&
-       this->send_buffer &&
-       this->recv_buffer &&
+    if((dim==0 || this->buffer_sz)   &&
+       (dim==0 || this->send_buffer) &&
+       (dim==0 || this->recv_buffer) &&
 #ifdef CCTK_MPI
-       this->sreq        &&
-       this->rreq        &&
-       this->sstatus     &&
+       (dim==0 || this->sreq)        &&
+       (dim==0 || this->rreq)        &&
+       (dim==0 || this->sstatus)     &&
 #endif
-       this->docomm)
+       (dim==0 || this->docomm))
     {
       this->commflag       = PUGH_NOCOMM;
       this->first_var      = first_var;
diff --git a/src/Storage.c b/src/Storage.c
index c3f8b6f..066fe18 100644
--- a/src/Storage.c
+++ b/src/Storage.c
@@ -316,43 +316,52 @@ int PUGH_EnableGroupStorage (const cGH *GH, const char *groupname)
   {
     first_var = CCTK_FirstVarIndexI (group);
 
-    /* get the group info from its index */
-    CCTK_GroupData (group, &pgroup);
-
-    retval = PUGH_EnableGArrayGroupStorage (pughGH,
-                                            first_var,
-                                            pgroup.numvars,
-                                            pgroup.numtimelevels);
-    if (!CCTK_Equals(storage_verbose,"no") && retval == 0)
+    if (first_var >= 0)
     {
-      /* get GA pointer of first var in group */
-      GA = (pGA *) pughGH->variables[first_var][0];
-      if (pgroup.grouptype == CCTK_GF)
-      {
-        totalnumberGF  += pgroup.numvars * pgroup.numtimelevels;
-      }
-      else
-      {
-        totalnumberGA  += pgroup.numvars * pgroup.numtimelevels;
-      }
-      totalstorage += (GA->extras->npoints * GA->varsize *
-                       pgroup.numtimelevels * pgroup.numvars) /
-                      (float) (1024*1024);
-      if (totalstorage > maxstorage)
-      {
-        numberGF = totalnumberGF;
-        numberGA = totalnumberGA;
-        maxstorage = totalstorage;
-      }
 
-      /* Report on memory usage */
-      if (CCTK_Equals(storage_verbose,"yes"))
+      /* get the group info from its index */
+      CCTK_GroupData (group, &pgroup);
+
+      retval = PUGH_EnableGArrayGroupStorage (pughGH,
+                                              first_var,
+                                              pgroup.numvars,
+                                              pgroup.numtimelevels);
+      if (!CCTK_Equals(storage_verbose,"no") && retval == 0)
       {
-        CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'"
-                    "  [GFs: %d GAs: %d Total Size: %6.2fMB]",
-                    groupname, totalnumberGF, totalnumberGA, totalstorage);
+        /* get GA pointer of first var in group */
+        GA = (pGA *) pughGH->variables[first_var][0];
+        if (pgroup.grouptype == CCTK_GF)
+        {
+          totalnumberGF  += pgroup.numvars * pgroup.numtimelevels;
+        }
+        else
+        {
+          totalnumberGA  += pgroup.numvars * pgroup.numtimelevels;
+        }
+        totalstorage += (GA->extras->npoints * GA->varsize *
+                         pgroup.numtimelevels * pgroup.numvars) /
+                        (float) (1024*1024);
+        if (totalstorage > maxstorage)
+        {
+          numberGF = totalnumberGF;
+          numberGA = totalnumberGA;
+          maxstorage = totalstorage;
+        }
+
+        /* Report on memory usage */
+        if (CCTK_Equals(storage_verbose,"yes"))
+        {
+          CCTK_VInfo (CCTK_THORNSTRING, "Switched memory on for group '%s'"
+                      "  [GFs: %d GAs: %d Total Size: %6.2fMB]",
+                      groupname, totalnumberGF, totalnumberGA, totalstorage);
+        }
       }
     }
+    else
+    {
+      /* don't know what to say here; 0 seems safe */
+      retval = 0;
+    }
 
   }
   else
@@ -956,28 +965,32 @@ int PUGH_GroupStorageIncrease(const cGH *GH, int n_groups,const int *groups,cons
 
       first_var = CCTK_FirstVarIndexI (groups[group]);
 
-      /* Enable all timelevels or just some */
-      if(timelevels[group] == -1)
+      if (first_var >= 0)
       {
-        tlevels = pgroup.numtimelevels;
-      }
-      else
-      {
-        tlevels = timelevels[group];
-      }
 
-      previous = PUGHi_EnableGArrayGroupStorage (pughGH,
-                                                 first_var,
-                                                 pgroup.numvars,
-                                                 pgroup.numtimelevels,
-                                                 tlevels);
+        /* Enable all timelevels or just some */
+        if(timelevels[group] == -1)
+        {
+          tlevels = pgroup.numtimelevels;
+        }
+        else
+        {
+          tlevels = timelevels[group];
+        }
+
+        previous = PUGHi_EnableGArrayGroupStorage (pughGH,
+                                                   first_var,
+                                                   pgroup.numvars,
+                                                   pgroup.numtimelevels,
+                                                   tlevels);
 
-      if(status)
-      {
-        status[group] = previous;
-      }
+        if(status)
+        {
+          status[group] = previous;
+        }
 
-      retval += previous;
+        retval += previous;
+      }
     }
   }
 
@@ -1066,28 +1079,32 @@ int PUGH_GroupStorageDecrease(const cGH *GH, int n_groups,const int *groups,cons
 
       first_var = CCTK_FirstVarIndexI (groups[group]);
 
-      /* Disable all timelevels or just some */
-      if(timelevels[group] == -1)
+      if (first_var >= 0)
       {
-        tlevels = 0;
-      }
-      else
-      {
-        tlevels = timelevels[group];
-      }
 
-      previous = PUGHi_DisableGArrayGroupStorage (pughGH,
-                                                  first_var,
-                                                  pgroup.numvars,
-                                                  pgroup.numtimelevels,
-                                                  tlevels);
+        /* Disable all timelevels or just some */
+        if(timelevels[group] == -1)
+        {
+          tlevels = 0;
+        }
+        else
+        {
+          tlevels = timelevels[group];
+        }
 
-      if(status)
-      {
-        status[group] = previous;
-      }
+        previous = PUGHi_DisableGArrayGroupStorage (pughGH,
+                                                    first_var,
+                                                    pgroup.numvars,
+                                                    pgroup.numtimelevels,
+                                                    tlevels);
+
+        if(status)
+        {
+          status[group] = previous;
+        }
 
-      retval += previous;
+        retval += previous;
+      }
     }
   }