CarpetLib: Add boolean parameter "omit_prolongation_points_when_restricting"

Add a parameter "omit_prolongation_points_when_restricting" that
controls whether to restrict to points that are used for boundary
prolongation.  The default is "no", restoring Carpet's behaviour from
before the patch "CarpetLib: Do not restrict to points that are used
for boundary prolongation".

When set to yes, then there is still a logic error in the code on
multiple processors, leading to different restriction regions on
different numbers of processors.

darcs-hash:20060503235940-dae7b-4c124e68e4c2519c0f97d416e0a7fa3489c1441d.gz

2 files changed, 27 insertions, 12 deletions

diff --git a/Carpet/CarpetLib/param.ccl b/Carpet/CarpetLib/param.ccl
index f4cd70439..45c38a83e 100644
--- a/Carpet/CarpetLib/param.ccl
+++ b/Carpet/CarpetLib/param.ccl
@@ -32,6 +32,12 @@ BOOLEAN fast_recomposing "Take shortcuts during recomposing (EXPERIMENTAL)" STEE
 
 
 
+BOOLEAN omit_prolongation_points_when_restricting "Do not restrict to points which are used to prolongate the boundary" STEERABLE=recover
+{
+} "no"
+
+
+
 INT print_memstats_every "Report periodically how much memory is used per process" STEERABLE=always
 {
   0   :: "don't report"
diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index 57628ef14..d54f7dee8 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -369,6 +369,8 @@ void dh::setup_refinement_boundary_prolongation_boxes (dh::dboxes & box,
 void dh::setup_refinement_restriction_boxes (dh::dboxes & box,
                                              int const rl, int const c, int const ml)
 {
+  DECLARE_CCTK_PARAMETERS;
+
   const ibbox& intr = box.interior;
 
   // Refinement boxes
@@ -407,17 +409,22 @@ void dh::setup_refinement_restriction_boxes (dh::dboxes & box,
           const ibbox recv = (*si).contracted_for(intr);
           recvs |= recv;
         }
-        // remove what is sent during boundary prolongation
-        const int pss = prolongation_stencil_size();
-        for (iblistvect::const_iterator slvi = box.send_ref_bnd_fine.begin();
-             slvi != box.send_ref_bnd_fine.end(); ++ slvi)
-        {
-          const iblist& sendlist = * slvi;
-          for (iblist::const_iterator sli = sendlist.begin();
-               sli != sendlist.end(); ++sli)
+        if (omit_prolongation_points_when_restricting) {
+          // remove what is sent during boundary prolongation
+          const int pss = prolongation_stencil_size();
+          // TODO: this needs to remove what is sent from other
+          // processors as well, not only what is sent from processor
+          // c
+          for (iblistvect::const_iterator slvi = box.send_ref_bnd_fine.begin();
+               slvi != box.send_ref_bnd_fine.end(); ++ slvi)
           {
-            const ibbox& send = * sli;
-            recvs -= send.expand(pss,pss);
+            const iblist& sendlist = * slvi;
+            for (iblist::const_iterator sli = sendlist.begin();
+                 sli != sendlist.end(); ++sli)
+            {
+              const ibbox& send = * sli;
+              recvs -= send.expand(pss,pss);
+            }
           }
         }
         recvs.normalize();
@@ -479,7 +486,9 @@ void dh::trim_unsynced_boundaries (dh::dboxes & box,
 void dh::check_bboxes (dh::dboxes & box,
                        int const rl, int const c, int const ml)
 {
-// Assert that all boundaries are synced or received
+  DECLARE_CCTK_PARAMETERS;
+
+  // Assert that all boundaries are synced or received
   {
     const ibset& sync_not = box.sync_not;
 #if 0
@@ -602,7 +611,7 @@ void dh::check_bboxes (dh::dboxes & box,
   
   // Assert that points which are used for boundary prolongation are
   // not restricted
-  {
+  if (omit_prolongation_points_when_restricting) {
     for (iblistvect::const_iterator slvi = box.send_ref_bnd_fine.begin();
          slvi != box.send_ref_bnd_fine.end(); ++ slvi)
     {