diff options
author | Erik Schnetter <schnetter@cct.lsu.edu> | 2010-03-18 15:18:19 -0700 |
---|---|---|
committer | Barry Wardell <barry.wardell@gmail.com> | 2011-12-14 16:45:34 +0000 |
commit | 9bf1720e33624be28e978dc36760b783a0b1ef33 (patch) | |
tree | 996cc1faf71274f9ed944e44ffbf0010a86286d5 | |
parent | 0707cfeb82f565afec604f39202dafcbef889db0 (diff) |
Carpet: Output load balance statistics for each refinement level
-rw-r--r-- | Carpet/Carpet/src/Recompose.cc | 45 |
1 files changed, 45 insertions, 0 deletions
diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc index 94b1f90f7..bb5f3ba20 100644 --- a/Carpet/Carpet/src/Recompose.cc +++ b/Carpet/Carpet/src/Recompose.cc @@ -7,6 +7,7 @@ #include <fstream> #include <iomanip> #include <iostream> +#include <limits> #include <list> #include <sstream> #include <string> @@ -1016,6 +1017,38 @@ namespace Carpet { num_owned_cpu_points /= num_steps * delta_time; num_total_cpu_points /= num_steps * delta_time; + // Load balance statistics + CCTK_REAL const rzero = 0; + CCTK_REAL const rmax = numeric_limits<CCTK_REAL>::max(); + vector<CCTK_REAL> min_active (reflevels, rmax); + vector<CCTK_REAL> max_active (reflevels, 0); + vector<CCTK_REAL> avg_active (reflevels, 0); + vector<CCTK_REAL> sdv_active (reflevels, 0); + for (int rl=0; rl<reflevels; ++rl) { + vector<CCTK_REAL> num_active_per_proc (dist::size(), 0); + for (int m=0; m<maps; ++m) { + gh const * const hh = vhh.AT(m); + dh const * const dd = vdd.AT(m); + for (int ml=0; ml<mglevels; ++ml) { + for (int c=0; c<hh->components(rl); ++c) { + int const p = hh->processor(rl,c); + dh::dboxes const & b = dd->boxes.AT(ml).AT(rl).AT(c); + num_active_per_proc.AT(p) += num_gfs * b.active_size; + } + } + } + for (int p=0; p<dist::size(); ++p) { + min_active.AT(rl) = min (min_active.AT(rl), num_active_per_proc.AT(p)); + max_active.AT(rl) = max (max_active.AT(rl), num_active_per_proc.AT(p)); + avg_active.AT(rl) += num_active_per_proc.AT(p); + sdv_active.AT(rl) += pow (num_active_per_proc.AT(p), 2); + } + avg_active.AT(rl) /= dist::size(); + sdv_active.AT(rl) /= dist::size(); + sdv_active.AT(rl) = + sqrt (max (rzero, sdv_active.AT(rl) - pow (avg_active.AT(rl), 2))); + } // for rl + // Output CCTK_VInfo (CCTK_THORNSTRING, "Grid structure statistics:"); @@ -1055,6 +1088,18 @@ namespace Carpet { CCTK_VInfo (CCTK_THORNSTRING, "Total required memory: %.3f GByte (for GAs and currently active GFs)", double ((size_total_array_points + size_total_mem_points) / 1e+9)); + CCTK_VInfo (CCTK_THORNSTRING, + "Load balance: min avg max sdv max/avg-1"); + for (int rl=0; rl<reflevels; ++rl) { + CCTK_VInfo (CCTK_THORNSTRING, + "Level %2d: %4.0fM %4.0fM %4.0fM %4.0fM active %4.0f%%", + rl, + double (min_active.AT(rl) / 1e+6), + double (avg_active.AT(rl) / 1e+6), + double (max_active.AT(rl) / 1e+6), + double (sdv_active.AT(rl) / 1e+6), + double (100 * (max_active.AT(rl) / avg_active.AT(rl) - 1))); + } // After this, we will begin to allocate memory for the grid // structure. If we run out of memory, ensure that this output |