Carpet: Output load balance statistics for each refinement level

1 files changed, 45 insertions, 0 deletions

diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc
index 94b1f90f7..bb5f3ba20 100644
--- a/Carpet/Carpet/src/Recompose.cc
+++ b/Carpet/Carpet/src/Recompose.cc
@@ -7,6 +7,7 @@
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <limits>
 #include <list>
 #include <sstream>
 #include <string>
@@ -1016,6 +1017,38 @@ namespace Carpet {
     num_owned_cpu_points  /= num_steps * delta_time;
     num_total_cpu_points  /= num_steps * delta_time;
     
+    // Load balance statistics
+    CCTK_REAL const rzero = 0;
+    CCTK_REAL const rmax  = numeric_limits<CCTK_REAL>::max();
+    vector<CCTK_REAL> min_active (reflevels, rmax);
+    vector<CCTK_REAL> max_active (reflevels, 0);
+    vector<CCTK_REAL> avg_active (reflevels, 0);
+    vector<CCTK_REAL> sdv_active (reflevels, 0);
+    for (int rl=0; rl<reflevels; ++rl) {
+      vector<CCTK_REAL> num_active_per_proc (dist::size(), 0);
+      for (int m=0; m<maps; ++m) {
+        gh const * const hh = vhh.AT(m);
+        dh const * const dd = vdd.AT(m);
+        for (int ml=0; ml<mglevels; ++ml) {
+          for (int c=0; c<hh->components(rl); ++c) {
+            int const p = hh->processor(rl,c);
+            dh::dboxes const & b = dd->boxes.AT(ml).AT(rl).AT(c);
+            num_active_per_proc.AT(p) += num_gfs * b.active_size;
+          }
+        }
+      }
+      for (int p=0; p<dist::size(); ++p) {
+        min_active.AT(rl) = min (min_active.AT(rl), num_active_per_proc.AT(p));
+        max_active.AT(rl) = max (max_active.AT(rl), num_active_per_proc.AT(p));
+        avg_active.AT(rl) += num_active_per_proc.AT(p);
+        sdv_active.AT(rl) += pow (num_active_per_proc.AT(p), 2);
+      }
+      avg_active.AT(rl) /= dist::size();
+      sdv_active.AT(rl) /= dist::size();
+      sdv_active.AT(rl) =
+        sqrt (max (rzero, sdv_active.AT(rl) - pow (avg_active.AT(rl), 2)));
+    } // for rl
+    
     // Output
     CCTK_VInfo (CCTK_THORNSTRING,
                 "Grid structure statistics:");
@@ -1055,6 +1088,18 @@ namespace Carpet {
     CCTK_VInfo (CCTK_THORNSTRING,
                 "Total required memory: %.3f GByte (for GAs and currently active GFs)",
                 double ((size_total_array_points + size_total_mem_points) / 1e+9));
+    CCTK_VInfo (CCTK_THORNSTRING,
+                "Load balance:  min     avg     max     sdv      max/avg-1");
+    for (int rl=0; rl<reflevels; ++rl) {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "Level %2d:    %4.0fM   %4.0fM   %4.0fM   %4.0fM active   %4.0f%%",
+                  rl,
+                  double (min_active.AT(rl) / 1e+6),
+                  double (avg_active.AT(rl) / 1e+6),
+                  double (max_active.AT(rl) / 1e+6),
+                  double (sdv_active.AT(rl) / 1e+6),
+                  double (100 * (max_active.AT(rl) / avg_active.AT(rl) - 1)));
+    }
     
     // After this, we will begin to allocate memory for the grid
     // structure.  If we run out of memory, ensure that this output