Carpet: bugfix for high-level synchronisation routines

In one of my previous patches I accidentally added two bugs when optimising
the high-level synchronisation routines in Carpet:

 * a SYNC statement did only synchronise the ghostzones
   but forgot to prolongate the boundaries for grid functions

 * SyncGroups() also tried to synchronise non-CCTK_GF variables
   at multigrid and refinement levels other than 0

darcs-hash:20050407150044-776a0-46b495efd5b68ab401ad00c5ac705f786069022c.gz

1 files changed, 161 insertions, 113 deletions

diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index c71458236..e7ec3f54e 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -12,76 +12,156 @@
 
 
 namespace Carpet {
-  
+
   using namespace std;
-  
-  
-  static int CheckSyncGroupConsistency (const cGH* cgh,
-                                        const char *groupname);
-  static void ProlongateGroupBoundaries (const cGH* cgh,
-                                         CCTK_REAL initial_time, int group,
-                                         int vartype);
-
-  
-  int SyncGroup (const cGH* cgh, const char* groupname)
+
+
+  static void ProlongateGroupBoundaries (const cGH* cctkGH,
+                                         CCTK_REAL initial_time,
+                                         group_set& groups);
+
+
+  // Carpet's overload function for CCTK_SyncGroup()
+  // which synchronises a single group
+  //
+  // returns 0 for success
+  //        -1 if the group doesn't have storage assigned
+  //        -2 if the given groupname is invalid
+  int SyncGroup (const cGH* cctkGH, const char* groupname)
   {
     DECLARE_CCTK_PARAMETERS;
-    int retval = 0;
+    int retval;
+
+    Checkpoint ("SyncGroup \"%s\" time=%g",
+                groupname, (double) cctkGH->cctk_time);
+
     const int group = CCTK_GroupIndex(groupname);
-    assert (group>=0 and group<CCTK_NumGroups());
-    assert (group<(int)arrdata.size());
-    const int grouptype = CCTK_GroupTypeI(group);
-    const int firstvar = CCTK_FirstVarIndexI (group);
-    const int vartype = CCTK_VarTypeI (firstvar);
-    assert(grouptype == CCTK_GF ||
-           grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY);
-    assert (CCTK_NumVarsInGroupI(group) != 0);
-
-    Checkpoint ("SyncGroup \"%s\" time=%g", groupname, (double)cgh->cctk_time);
-    
-    retval = CheckSyncGroupConsistency (cgh, groupname);
-    
-    if (retval == 0) {
-    
-      // Prolongate the boundaries
-      if (do_prolongate && grouptype == CCTK_GF) {
-        assert (reflevel>=0 and reflevel<reflevels);
-        if (reflevel > 0) {
-          ProlongateGroupBoundaries (cgh, cctk_initial_time, group, vartype);
-        }
-      }
-    
-      // Sync
+    if (group >= 0) {
+      assert (group < (int)arrdata.size());
+
+      const int firstvar = CCTK_FirstVarIndexI (group);
+      const int vartype = CCTK_VarTypeI (firstvar);
       const vector<int> members(1, group);
       group_set groups = {vartype, members};
-      SyncGroups (cgh, groups);
+      retval = SyncProlongateGroups (cctkGH, groups);
+    } else {
+      retval = -2;
     }
+
     return retval;
   }
-  
-  static void ProlongateGroupBoundaries (const cGH* cgh,
-                                         CCTK_REAL initial_time, int group,
-                                         int vartype)
+
+
+  // synchronises ghostzones and prolongates boundaries
+  // of a set of groups which all have the same vartype
+  //
+  // returns 0 for success and -1 if the set contains a group with no storage
+  int SyncProlongateGroups (const cGH* cctkGH, group_set& groups)
   {
+    int retval = 0;
     DECLARE_CCTK_PARAMETERS;
 
-    // use the current time here (which may be modified by the user)
-    const CCTK_REAL time = (cgh->cctk_time - initial_time) / delta_time;
+    assert (groups.members.size() > 0);
+
+    // check consistency of all groups:
+    // create a new set with empty and no-storage groups removed
+    group_set goodgroups = {groups.vartype};
+    for (size_t g = 0; g < groups.members.size(); g++) {
+      const int group = groups.members[g];
+      const int grouptype = CCTK_GroupTypeI (group);
+      char* groupname = CCTK_GroupName (group);
+
+      if (grouptype == CCTK_GF) {
+        if (reflevel == -1) {
+          CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                      "Cannot synchronise in global mode "
+                      "(Tried to synchronise group \"%s\")",
+                      groupname);
+        }
+        if (map != -1 and component == -1) {
+          if (maps == 1) {
+            CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                        "Synchronising group \"%s\" in singlemap mode",
+                        groupname);
+          } else {
+            CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                        "Cannot synchronise in singlemap mode "
+                        "(Tried to synchronise group \"%s\")",
+                        groupname);
+          }
+        }
+        if (component != -1) {
+          if (maps == 1 and vhh.at(map)->local_components(reflevel) == 1) {
+            CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                        "Synchronising group \"%s\" in local mode",
+                        groupname);
+          } else {
+            CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
+                        "Cannot synchronise in local mode "
+                        "(Tried to synchronise group \"%s\")",
+                        groupname);
+          }
+        }
+      }
+
+      if (! CCTK_QueryGroupStorageI (cctkGH, group)) {
+        CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "Cannot synchronise group \"%s\" because it has no storage",
+                    groupname);
+        retval = -1;
+      }
+      else if (CCTK_NumVarsInGroupI (group) > 0) {
+        goodgroups.members.push_back(group);
+      }
+
+      free (groupname);
+    }
+
+    if (goodgroups.members.size() > 0) {
+      // prolongate boundaries
+      if (do_prolongate && reflevel > 0) {
+        ProlongateGroupBoundaries (cctkGH, cctk_initial_time, goodgroups);
+      }
+
+      // synchronise ghostzones
+      SyncGroups (cctkGH, goodgroups);
+    }
+
+    return retval;
+  }
+
+  // Prolongate the boundaries of all CCTK_GF groups in the given set
+  static void ProlongateGroupBoundaries (const cGH* cctkGH,
+                                         CCTK_REAL initial_time,
+                                         group_set& groups)
+  {
+    DECLARE_CCTK_PARAMETERS;
     const int tl = 0;
-    
+
     // Use collective or single-component buffers for communication ?
-    if (! use_collective_communication_buffers) {
-      vartype = -1;
-    }
+    const int vartype =
+      use_collective_communication_buffers ? groups.vartype : -1;
+
+    // use the current time here (which may be modified by the user)
+    const CCTK_REAL time = (cctkGH->cctk_time - initial_time) / delta_time;
 
     if (use_collective_communication_buffers ||
         ! minimise_outstanding_communications) {
       for (comm_state state(vartype); ! state.done(); state.step()) {
-        for (int m = 0; m < arrdata.at(group).size(); ++m) {
-          for (int v = 0; v < arrdata.at(group).at(m).data.size(); ++v) {
-            ggf *const gv = arrdata.at(group).at(m).data.at(v);
-            for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
-              gv->ref_bnd_prolongate (state, tl, reflevel, c, mglevel, time);
+        for (int group = 0; group < groups.members.size(); ++group) {
+          const int g = groups.members.at(group);
+          const int grouptype = CCTK_GroupTypeI (g);
+          if (grouptype != CCTK_GF) {
+            continue;
+          }
+          assert (reflevel>=0 and reflevel<reflevels);
+
+          for (int m = 0; m < arrdata.at(g).size(); ++m) {
+            for (int v = 0; v < arrdata.at(g).at(m).data.size(); ++v) {
+              ggf *const gv = arrdata.at(g).at(m).data.at(v);
+              for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
+                gv->ref_bnd_prolongate (state, tl, reflevel, c, mglevel, time);
+              }
             }
           }
         }
@@ -89,12 +169,21 @@ namespace Carpet {
     } else {
       // make the comm_state loop the innermost
       // in order to minimise the number of outstanding communications
-      for (int m = 0; m < arrdata.at(group).size(); ++m) {
-        for (int v = 0; v < arrdata.at(group).at(m).data.size(); ++v) {
-          ggf *const gv = arrdata.at(group).at(m).data.at(v);
-          for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
-            for (comm_state state(vartype); ! state.done(); state.step()) {
-              gv->ref_bnd_prolongate (state, tl, reflevel, c, mglevel, time);
+      for (int group = 0; group < groups.members.size(); ++group) {
+        const int g = groups.members.at(group);
+        const int grouptype = CCTK_GroupTypeI (g);
+        if (grouptype != CCTK_GF) {
+          continue;
+        }
+        assert (reflevel>=0 and reflevel<reflevels);
+
+        for (int m = 0; m < arrdata.at(g).size(); ++m) {
+          for (int v = 0; v < arrdata.at(g).at(m).data.size(); ++v) {
+            ggf *const gv = arrdata.at(g).at(m).data.at(v);
+            for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
+              for (comm_state state(vartype); ! state.done(); state.step()) {
+                gv->ref_bnd_prolongate (state, tl, reflevel, c, mglevel, time);
+              }
             }
           }
         }
@@ -102,8 +191,9 @@ namespace Carpet {
     }
   }
 
+
   // synchronises a set of group which all have the same vartype
-  void SyncGroups (const cGH* cgh, group_set& groups)
+  void SyncGroups (const cGH* cctkGH, group_set& groups)
   {
     DECLARE_CCTK_PARAMETERS;
     const int tl = 0;
@@ -119,11 +209,14 @@ namespace Carpet {
       for (comm_state state(vartype); ! state.done(); state.step()) {
         for (int group = 0; group < groups.members.size(); ++group) {
           const int g = groups.members.at(group);
+          const int grouptype = CCTK_GroupTypeI (g);
+          const int ml = grouptype == CCTK_GF ? mglevel : 0;
+          const int rl = grouptype == CCTK_GF ? reflevel : 0;
           for (int m = 0; m < arrdata.at(g).size(); ++m) {
             for (int v = 0; v < arrdata.at(g).at(m).data.size(); ++v) {
               ggf *const gv = arrdata.at(g).at(m).data.at(v);
               for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
-                gv->sync (state, tl, reflevel, c, mglevel);
+                gv->sync (state, tl, rl, c, ml);
               }
             }
           }
@@ -134,12 +227,15 @@ namespace Carpet {
       // in order to minimise the number of outstanding communications
       for (int group = 0; group < groups.members.size(); ++group) {
         const int g = groups.members.at(group);
+        const int grouptype = CCTK_GroupTypeI (g);
+        const int ml = grouptype == CCTK_GF ? mglevel : 0;
+        const int rl = grouptype == CCTK_GF ? reflevel : 0;
         for (int m = 0; m < arrdata.at(g).size(); ++m) {
           for (int v = 0; v < arrdata.at(g).at(m).data.size(); ++v) {
             ggf *const gv = arrdata.at(g).at(m).data.at(v);
             for (int c = 0; c < vhh.at(m)->components(reflevel); ++c) {
               for (comm_state state(vartype); ! state.done(); state.step()) {
-                gv->sync (state, tl, reflevel, c, mglevel);
+                gv->sync (state, tl, rl, c, ml);
               }
             }
           }
@@ -149,64 +245,16 @@ namespace Carpet {
   }
 
 
-  int CheckSyncGroupConsistency ( const cGH* cgh,const char *groupname )
-  {
-    int retval = 0;
-    const int group = CCTK_GroupIndex(groupname);
-    const int grouptype = CCTK_GroupTypeI(group);
-    
-    if (grouptype == CCTK_GF) {
-      if (reflevel == -1) {
-        CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                    "Cannot synchronise in global mode "
-                    "(Tried to synchronise group \"%s\")",
-                    groupname);
-      }
-      if (map != -1 and component == -1) {
-        if (maps == 1) {
-          CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-                      "Synchronising group \"%s\" in singlemap mode",
-                      groupname);
-        } else {
-          CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                      "Cannot synchronise in singlemap mode "
-                      "(Tried to synchronise group \"%s\")",
-                      groupname);
-        }
-      }
-      if (component != -1) {
-        if (maps == 1 and vhh.at(map)->local_components(reflevel) == 1) {
-          CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-                      "Synchronising group \"%s\" in local mode",
-                      groupname);
-        } else {
-          CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                      "Cannot synchronise in local mode "
-                      "(Tried to synchronise group \"%s\")",
-                      groupname);
-        }
-      }
-    }
-    
-    if (! CCTK_QueryGroupStorageI(cgh, group)) {
-      CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING,
-		  "Cannot synchronise group \"%s\" because it has no storage",
-		  groupname);
-      retval = -1;
-    }
-    return retval;
-  }
-  
-  int EnableGroupComm (const cGH* cgh, const char* groupname)
+  int EnableGroupComm (const cGH* cctkGH, const char* groupname)
   {
     // Communication is always enabled
     return 0;
   }
-  
-  int DisableGroupComm (const cGH* cgh, const char* groupname)
+
+  int DisableGroupComm (const cGH* cctkGH, const char* groupname)
   {
     // Communication is always enabled
     return -1;
   }
-  
+
 } // namespace Carpet