CarpetInterp2: Add parameter "interp_barrier" to add a barrier before interpolating

Interpolation is a global operation, so adding a barrier should not affect performance, and makes timers much more useful.

2 files changed, 13 insertions, 0 deletions

diff --git a/Carpet/CarpetInterp2/param.ccl b/Carpet/CarpetInterp2/param.ccl
index f7b5fffef..5c624daa2 100644
--- a/Carpet/CarpetInterp2/param.ccl
+++ b/Carpet/CarpetInterp2/param.ccl
@@ -7,3 +7,8 @@ BOOLEAN verbose "Produce info output" STEERABLE=always
 BOOLEAN veryverbose "Produce debugging output" STEERABLE=always
 {
 } "no"
+
+BOOLEAN interp_barrier "Wait for all processes before interpolating" STEERABLE=always
+{
+} "no"
+
diff --git a/Carpet/CarpetInterp2/src/fasterp.cc b/Carpet/CarpetInterp2/src/fasterp.cc
index c32721fc9..3d3ac530a 100644
--- a/Carpet/CarpetInterp2/src/fasterp.cc
+++ b/Carpet/CarpetInterp2/src/fasterp.cc
@@ -1619,6 +1619,14 @@ namespace CarpetInterp2 {
     assert (values.size() == nvars);
     
     if (nvars == 0) return;
+
+    if (interp_barrier)
+    {
+      static Timers::Timer barrier_timer ("Barrier");
+      barrier_timer.start();
+      CCTK_Barrier(cctkGH);
+      barrier_timer.stop();
+    }
     
     for (size_t v=0; v<values.size(); ++v) {
       int const vi = varinds.AT(v);