Carpet: Introduce more timers to measure communication time

3 files changed, 124 insertions, 3 deletions

diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index ab6106c2b..55be38251 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -3,6 +3,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <iostream>
+#include <vector>
 
 #include <cctk.h>
 #include <cctk_Parameters.h>
@@ -180,8 +181,29 @@ namespace Carpet {
 
     // use the current time here (which may be modified by the user)
     const CCTK_REAL time = cctkGH->cctk_time;
+
+    static vector<Timer*> timers;
+    if (timers.empty()) {
+      timers.push_back(new Timer("comm_state[0].create"));
+      for (astate state = static_cast<astate>(0);
+           state != state_done;
+           state = static_cast<astate>(static_cast<int>(state)+1))
+      {
+        ostringstream name1;
+        name1 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".user";
+        timers.push_back(new Timer(name1.str()));
+        ostringstream name2;
+        name2 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".step";
+        timers.push_back(new Timer(name2.str()));
+      }
+    }
     
+    vector<Timer*>::iterator ti = timers.begin();
+    (*ti)->start();
     for (comm_state state; not state.done(); state.step()) {
+      (*ti)->stop(); ++ti; (*ti)->start();
       for (int group = 0; group < (int)groups.size(); ++group) {
         const int g = groups.AT(group);
         const int grouptype = CCTK_GroupTypeI (g);
@@ -202,7 +224,10 @@ namespace Carpet {
           }
         }
       }
+      (*ti)->stop(); ++ti; (*ti)->start();
     }
+    (*ti)->stop();
+    ++ti; assert(ti == timers.end());
   }
 
 
@@ -219,7 +244,28 @@ namespace Carpet {
       Accelerator_PreSync(cctkGH, &groups.front(), groups.size());
     }
 
+    static vector<Timer*> timers;
+    if (timers.empty()) {
+      timers.push_back(new Timer("comm_state[0].create"));
+      for (astate state = static_cast<astate>(0);
+           state != state_done;
+           state = static_cast<astate>(static_cast<int>(state)+1))
+      {
+        ostringstream name1;
+        name1 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".user";
+        timers.push_back(new Timer(name1.str()));
+        ostringstream name2;
+        name2 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".step";
+        timers.push_back(new Timer(name2.str()));
+      }
+    }
+    
+    vector<Timer*>::iterator ti = timers.begin();
+    (*ti)->start();
     for (comm_state state; not state.done(); state.step()) {
+      (*ti)->stop(); ++ti; (*ti)->start();
       for (int group = 0; group < (int)groups.size(); ++group) {
         const int g = groups.AT(group);
         const int grouptype = CCTK_GroupTypeI (g);
@@ -235,8 +281,11 @@ namespace Carpet {
           }
         }
       }
+      (*ti)->stop(); ++ti; (*ti)->start();
     }
-
+    (*ti)->stop();
+    ++ti; assert(ti == timers.end());
+    
     if (CCTK_IsFunctionAliased("Accelerator_PostSync")) {
       Accelerator_PostSync(cctkGH, &groups.front(), groups.size());
     }
diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc
index 8ede1ebcd..e24198dcd 100644
--- a/Carpet/Carpet/src/Restrict.cc
+++ b/Carpet/Carpet/src/Restrict.cc
@@ -9,6 +9,7 @@
 #include <gh.hh>
 
 #include <carpet.hh>
+#include <Timers.hh>
 
 
 
@@ -51,10 +52,20 @@ namespace Carpet {
     }
 
     // Restrict
-    RestrictGroups (cctkGH, groups);
+    {
+      static Timer timer ("Restrict");
+      timer.start();
+      RestrictGroups (cctkGH, groups);
+      timer.stop();
+    }
 
     // Synchronise
-    SyncGroups (cctkGH, groups);
+    {
+      static Timer timer ("RestrictSync");
+      timer.start();
+      SyncGroups (cctkGH, groups);
+      timer.stop();
+    }
   }
   
 
@@ -62,7 +73,28 @@ namespace Carpet {
   static void RestrictGroups (const cGH* cctkGH, const vector<int>& groups) {
     DECLARE_CCTK_PARAMETERS;
 
+    static vector<Timer*> timers;
+    if (timers.empty()) {
+      timers.push_back(new Timer("comm_state[0].create"));
+      for (astate state = static_cast<astate>(0);
+           state != state_done;
+           state = static_cast<astate>(static_cast<int>(state)+1))
+      {
+        ostringstream name1;
+        name1 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".user";
+        timers.push_back(new Timer(name1.str()));
+        ostringstream name2;
+        name2 << "comm_state[" << timers.size() << "]"
+              << "." << tostring(state) << ".step";
+        timers.push_back(new Timer(name2.str()));
+      }
+    }
+
+    vector<Timer*>::iterator ti = timers.begin();
+    (*ti)->start();
     for (comm_state state; not state.done(); state.step()) {
+      (*ti)->stop(); ++ti; (*ti)->start();
       for (int group = 0; group < (int)groups.size(); ++group) {
         const int g = groups.AT(group);
         const int active_tl = CCTK_ActiveTimeLevelsGI (cctkGH, g);
@@ -75,7 +107,10 @@ namespace Carpet {
           }
         }
       } // loop over groups
+      (*ti)->stop(); ++ti; (*ti)->start();
     } // for state
+    (*ti)->stop();
+    ++ti; assert(ti == timers.end());
   }
 
 } // namespace Carpet
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index 2e814628f..6369c059d 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -493,6 +493,43 @@ namespace Carpet {
                      "The number of threads for this process is larger its number of cores. This may indicate a performance problem.");
         }
       }
+      
+#pragma omp parallel
+      for (int thread=0; thread<mynthreads; ++thread) {
+        if (thread == dist::thread_num()) {
+          vector<bool> mask(CPU_SETSIZE, false);
+          cpu_set_t cpumask;
+          int const ierr = sched_getaffinity (0, sizeof cpumask, &cpumask);
+          assert (not ierr);
+          for (int n=0; n<CPU_SETSIZE; ++n) {
+            if (CPU_ISSET(n, &cpumask)) {
+              mask.at(n) = true;
+            }
+          }
+          
+          ostringstream buf;
+          int num_cores = 0;
+          bool isfirst = true;
+          int first_active = -1;
+          for (int n=0; n<CPU_SETSIZE; ++n) {
+            if (mask.at(n)) ++num_cores;
+            if (first_active == -1 and mask.at(n)) {
+              if (not isfirst) buf << ", ";
+              isfirst = false;
+              buf << n;
+              first_active = n;
+            } else if (first_active >= 0 and not mask.at(n)) {
+              if (n-1 > first_active) buf << "-" << n-1;
+              first_active = -1;
+            }
+          }
+          CCTK_VInfo (CCTK_THORNSTRING,
+                      "Thread %d runs on %d core%s: %s",
+                      thread,
+                      num_cores, num_cores==1 ? "" : "s", buf.str().c_str());
+        }
+#pragma omp barrier
+      }
 #else
       CCTK_INFO ("Cannot determine core affinity");
 #endif