CarpetInterp: Poison coordinates and patch numbers

Poison coordinates and patch numbers when exchanging them between
processors.

2 files changed, 60 insertions, 0 deletions

diff --git a/Carpet/CarpetInterp/param.ccl b/Carpet/CarpetInterp/param.ccl
index 3f6b86379..3ba04386d 100644
--- a/Carpet/CarpetInterp/param.ccl
+++ b/Carpet/CarpetInterp/param.ccl
@@ -4,6 +4,16 @@ BOOLEAN barriers "Insert barriers at strategic places for debugging purposes (sl
 {
 } "no"
 
+CCTK_REAL poison "Poison value" STEERABLE=always
+{
+  *:* :: ""
+} -4.20042e+30
+
+CCTK_REAL ipoison "Integer poison value" STEERABLE=always
+{
+  *:* :: ""
+} -420042
+
 SHARES: Cactus
 
 USES CCTK_REAL cctk_initial_time
diff --git a/Carpet/CarpetInterp/src/interp.cc b/Carpet/CarpetInterp/src/interp.cc
index 2ff115029..e7d5e60f9 100644
--- a/Carpet/CarpetInterp/src/interp.cc
+++ b/Carpet/CarpetInterp/src/interp.cc
@@ -489,9 +489,33 @@ namespace CarpetInterp {
           assert (recvdispl.AT(n) + recvcnt.AT(n) <= recvbufsize);
         }
       }
+#ifndef _NDEBUG
+      for (size_t i=0; i<tmp.size(); ++i) {
+        tmp.AT(i) = poison;
+      }
+#endif
       MPI_Alltoallv (&coords_buffer[0], &sendcnt[0], &senddispl[0], datatype,
                      &tmp[0],           &recvcnt[0], &recvdispl[0], datatype,
                      dist::comm());
+#ifndef _NDEBUG
+      {
+        vector<bool> filled(tmp.size(), false);
+        for (size_t n=0; n<dist::size(); ++n) {
+          for (size_t i=0; i<recvcnt.AT(n); ++i) {
+            assert (not filled.AT(recvdispl.AT(n)+i));
+            filled.AT(recvdispl.AT(n)+i) = true;
+          }
+        }
+        for (size_t i=0; i<filled.size(); ++i) {
+          assert (filled.AT(i));
+        }
+      }
+#endif
+#ifndef _NDEBUG
+      for (size_t i=0; i<tmp.size(); ++i) {
+        assert (tmp.AT(i) != poison);
+      }
+#endif
       coords_buffer.swap (tmp);
     }
 
@@ -562,9 +586,33 @@ namespace CarpetInterp {
           assert (recvdispl.AT(n) + recvcnt.AT(n) <= recvbufsize);
         }
       }
+#ifndef _NDEBUG
+      for (size_t i=0; i<source_map.size(); ++i) {
+        source_map[i] = ipoison;
+      }
+#endif
       MPI_Alltoallv (&tmp[0],        &sendcnt[0], &senddispl[0], datatype,
                      &source_map[0], &recvcnt[0], &recvdispl[0], datatype,
                      dist::comm());
+#ifndef _NDEBUG
+      {
+        vector<bool> filled(source_map.size(), false);
+        for (size_t n=0; n<dist::size(); ++n) {
+          for (size_t i=0; i<recvcnt.AT(n); ++i) {
+            assert (not filled.AT(recvdispl.AT(n)+i));
+            filled.AT(recvdispl.AT(n)+i) = true;
+          }
+        }
+        for (size_t i=0; i<filled.size(); ++i) {
+          assert (filled.AT(i));
+        }
+      }
+#endif
+#ifndef _NDEBUG
+      for (size_t i=0; i<source_map.size(); ++i) {
+        assert (source_map[i] != poison);
+      }
+#endif
     } else {
       // No explicit source map specified
       if (Carpet::map != -1) {
@@ -1109,6 +1157,8 @@ namespace CarpetInterp {
       }
     }
 
+    // TODO: Don't use explicit mode switching; code the loops
+    // manually
     BEGIN_GLOBAL_MODE(cctkGH) {
       for (int rl=minrl; rl<maxrl; ++rl) {
         ENTER_LEVEL_MODE (cctkGH, rl) {