From 418c9491b1962e803dec4c31259b3a425e5fa1c4 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Sat, 3 Mar 2012 12:37:46 -0500
Subject: CarpetReduce: Warn about reductions in local mode

---
 Carpet/CarpetReduce/src/reduce.cc | 44 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

(limited to 'Carpet/CarpetReduce')

diff --git a/Carpet/CarpetReduce/src/reduce.cc b/Carpet/CarpetReduce/src/reduce.cc
index a849f100b..6042cc777 100644
--- a/Carpet/CarpetReduce/src/reduce.cc
+++ b/Carpet/CarpetReduce/src/reduce.cc
@@ -1202,12 +1202,54 @@ namespace CarpetReduce {
     
     
     
-    // meta mode
+    // can't have meta mode
     if (is_meta_mode()) {
       CCTK_WARN (CCTK_WARN_ABORT,
                  "Grid variable reductions are not possible in meta mode");
     }
     
+    // local mode may be trouble
+    if (is_local_mode()) {
+      assert (mc_grouptype >= 0);
+      if (mc_grouptype == CCTK_GF) {
+        int mynlc = 0;
+        for (int m=0; m<maps; ++m) {
+          mynlc += vhh.AT(m)->local_components(reflevel);
+        }
+        int nlc = mynlc;
+        MPI_Bcast(&nlc, 1, MPI_INT, 0, dist::comm());
+        if (nlc == mynlc) {
+          CCTK_WARN (CCTK_WARN_PICKY,
+                     "Reduction in local mode may lead to deadlock (if different processes own different numbers of components)");
+        } else {
+          CCTK_WARN (CCTK_WARN_ALERT,
+                     "Reduction in local mode will lead to deadlock if different processes own different numbers of components");
+        }
+      }
+    }
+    
+    // singlemap mode may also be trouble
+    if (is_singlemap_mode()) {
+      assert (mc_grouptype >= 0);
+      if (mc_grouptype == CCTK_GF) {
+        int mynlm = 0;
+        for (int m=0; m<maps; ++m) {
+          if (vhh.AT(m)->local_components(reflevel) > 0) {
+            ++mynlm;
+          };
+        }
+        int nlm = mynlm;
+        MPI_Bcast(&nlm, 1, MPI_INT, 0, dist::comm());
+        if (nlm == mynlm) {
+          CCTK_WARN (CCTK_WARN_PICKY,
+                     "Reduction in singlemap mode may lead to deadlock (if different processes own components on different numbers of maps)");
+        } else {
+          CCTK_WARN (CCTK_WARN_ALERT,
+                     "Reduction in singlemap mode will lead to deadlock if different processes own components on different numbers of maps");
+        }
+      }
+    }
+    
     bool const reduce_arrays = CCTK_GroupTypeFromVarI(vi) != CCTK_GF;
     bool const want_global_mode = is_global_mode() and not reduce_arrays;
     bool const want_level_mode = is_level_mode() and not reduce_arrays;
-- 
cgit v1.2.3