Improve named barriers

Introduce dist::barrier as low-level implementation of a named
barrier.

Use it in Carpet::NamedBarrier.

Use the above in almost all barrier calls.

9 files changed, 41 insertions, 33 deletions

diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc
index 7c524c21e..936c8525d 100644
--- a/Carpet/Carpet/src/CallFunction.cc
+++ b/Carpet/Carpet/src/CallFunction.cc
@@ -297,7 +297,7 @@ namespace Carpet {
       buf << attribute->routine << "\n";
       string const str = buf.str();
       int const id = adler32(str.c_str(), str.length());
-      Carpet::NamedBarrier (NULL, id);
+      Carpet::NamedBarrier (NULL, id, "Carpet::CallFunction");
     }
     
     total_timer.stop();
diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index ce646e24a..9fa2893f3 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -149,7 +149,8 @@ namespace Carpet {
       // information about this ask Frank Loeffler
       // <knarf@cct.lsu.edu>.
 #ifdef CARPET_MPI_BARRIER_PROLONGATE_SYNC
-      CCTK_Barrier(cctkGH);
+      Carpet::NamedBarrier(cctkGH,
+                           8472211063, "CARPET_MPI_BARRIER_PROLONGATE_SYNC");
 #endif
       
       // synchronise ghostzones
diff --git a/Carpet/Carpet/src/functions.hh b/Carpet/Carpet/src/functions.hh
index 99cd24a09..6c1d3c84a 100644
--- a/Carpet/Carpet/src/functions.hh
+++ b/Carpet/Carpet/src/functions.hh
@@ -37,7 +37,7 @@ namespace Carpet {
   int GroupStorageDecrease (const cGH* cgh, int n_groups, const int* groups,
                             const int* timelevels, int* status);
   int Barrier (const cGH* cgh);
-  int NamedBarrier (const cGH* cgh, unsigned int id);
+  int NamedBarrier (const cGH* cgh, unsigned int id, const char* name);
   int Exit (const cGH* cgh, int retval);
   int Abort (const cGH* cgh, int retval);
   int MyProc (const cGH* cgh) CCTK_ATTRIBUTE_CONST;
diff --git a/Carpet/Carpet/src/helpers.cc b/Carpet/Carpet/src/helpers.cc
index 147cee109..afacfc802 100644
--- a/Carpet/Carpet/src/helpers.cc
+++ b/Carpet/Carpet/src/helpers.cc
@@ -198,23 +198,22 @@ namespace Carpet {
     return 0;
   }
 
-  int NamedBarrier (const cGH* const cgh, const unsigned int id)
+  int NamedBarrier (const cGH* const cgh,
+                    const unsigned int id, const char* name)
   {
     const void *dummy = &dummy;
     dummy = &cgh;
-
-    const int root = 0;
-    unsigned int my_id = dist::rank()==root ? id : id+1;
-    Checkpoint ("About to Bcast %ud", id);
-    MPI_Bcast (&my_id, 1, MPI_INT, root, dist::comm());
-    Checkpoint ("Finished Bcast");
-    if (my_id != id) {
-      CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
-                  "Wrong Barrier name: expected %ud, found %ud", id, my_id);
-    }
-    Checkpoint ("About to Barrier %ud", id);
-    MPI_Barrier (dist::comm());
+    
+    // Prevent recursive calls
+    static bool inside = false;
+    if (inside) return 0;
+    inside = true;
+    
+    Checkpoint ("About to Barrier %ud \"%s\"", id, name);
+    dist::barrier (dist::comm(), id, name);
     Checkpoint ("Finished Barrier");
+    
+    inside = false;
     return 0;
   }
 
@@ -222,7 +221,7 @@ namespace Carpet {
 
   int Exit (const cGH* cgh, int retval)
   {
-    CCTK_Barrier (cgh);
+    NamedBarrier (cgh, 792686462, "Carpet::Exit");
     dist::finalize();
     exit (retval);
     return -1;
@@ -446,7 +445,7 @@ namespace Carpet {
     va_end (args);
     CCTK_INFO (msg);
     if (barriers) {
-      MPI_Barrier (dist::comm());
+      NamedBarrier (NULL, 988121033, "Carpet::Output");
     }
   }
 
@@ -464,7 +463,7 @@ namespace Carpet {
       CCTK_INFO (msg);
     }
     if (barriers) {
-      MPI_Barrier (dist::comm());
+      NamedBarrier (NULL, 471561432, "Carpet::Waypoint");
     }
   }
 
@@ -482,7 +481,7 @@ namespace Carpet {
       CCTK_INFO (msg);
     }
     if (barriers) {
-      MPI_Barrier (dist::comm());
+      NamedBarrier (NULL, 572893143, "Carpet::Checkpoint");
     }
   }
 
diff --git a/Carpet/CarpetInterp/src/interp.cc b/Carpet/CarpetInterp/src/interp.cc
index b1cc62348..2a7a62fdb 100644
--- a/Carpet/CarpetInterp/src/interp.cc
+++ b/Carpet/CarpetInterp/src/interp.cc
@@ -329,15 +329,9 @@ namespace CarpetInterp {
 
 
     if (barriers) {
-      static unsigned int magic = 0x2ad072fbUL; // a random starting value
-      unsigned int recv = magic;
-      MPI_Bcast (& recv, 1, MPI_UNSIGNED, 0, dist::comm());
-      if (recv != magic) {
-        CCTK_WARN (CCTK_WARN_ABORT,
-                   "Inconsistent communication schedule: not all processes call CarpetInterp");
-      }
-      ++ magic;
-      MPI_Barrier (dist::comm());
+      Carpet::NamedBarrier (cctkGH,
+                            696681976,
+                            "CarpetInterp::Carpet_DriverInterpolate");
     }
 
     //////////////////////////////////////////////////////////////////////
diff --git a/Carpet/CarpetLib/src/commstate.cc b/Carpet/CarpetLib/src/commstate.cc
index 27f426890..19095864f 100644
--- a/Carpet/CarpetLib/src/commstate.cc
+++ b/Carpet/CarpetLib/src/commstate.cc
@@ -77,7 +77,7 @@ void comm_state::step ()
       CCTK_VInfo (CCTK_THORNSTRING,
                   "before MPI_Barrier; state=%s", tostring(thestate));
     }
-    MPI_Barrier (dist::comm());
+    dist::barrier (dist::comm(), 404924393, "CarpetLib::comm_state::step");
     if (commstate_verbose) {
       CCTK_INFO ("after MPI_Barrier");
     }
diff --git a/Carpet/CarpetLib/src/dist.cc b/Carpet/CarpetLib/src/dist.cc
index 6f8b759d3..dee79d5bf 100644
--- a/Carpet/CarpetLib/src/dist.cc
+++ b/Carpet/CarpetLib/src/dist.cc
@@ -237,6 +237,19 @@ namespace dist {
     MPI_Finalize ();
   }
   
+  void barrier (MPI_Comm const c, int const id, char const *const name)
+  {
+    int const root = 0;
+    unsigned int my_id = dist::rank()==root ? id : id+1;
+    MPI_Bcast (&my_id, 1, MPI_INT, root, c);
+    if (my_id != id) {
+      CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Wrong id for Barrier \"%s\": expected %ud, found %ud",
+                  name, id, my_id);
+    }
+    MPI_Barrier (c);
+  }
+
   
   
   // Create an MPI datatype from a C datatype description
@@ -412,7 +425,7 @@ namespace dist {
 	      rank, file, line);
     }
     if (barriers) {
-      MPI_Barrier (comm());
+      dist::barrier (comm(), 506877899+line, "CarpetLib::dist::checkpoint");
     }
   }
   
diff --git a/Carpet/CarpetLib/src/dist.hh b/Carpet/CarpetLib/src/dist.hh
index 83ebd3393..7fd084bb3 100644
--- a/Carpet/CarpetLib/src/dist.hh
+++ b/Carpet/CarpetLib/src/dist.hh
@@ -41,6 +41,7 @@ namespace dist {
   void init (int& argc, char**& argv);
   void pseudoinit (MPI_Comm const c);
   void finalize ();
+  void barrier (MPI_Comm const c, int const id, char const *const errormsg);
   
   
   
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index 2feda1882..cba9f1796 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -51,7 +51,7 @@ gdata::gdata (const int varindex_,
   allgdatai = allgdata.insert(allgdata.end(), this);
   
   if (barriers) {
-    MPI_Barrier (dist::comm());
+    dist::barrier (dist::comm(), 783988953, "CarpetLib::gdata::gdata");
   }
 }
 
@@ -63,7 +63,7 @@ gdata::~gdata ()
   allgdata.erase(allgdatai);
   
   if (barriers) {
-    MPI_Barrier (dist::comm());
+    dist::barrier (dist::comm(), 109687805, "CarpetLib::gdata::~gdata");
   }
 }