CarpetReduce: Support multi-block systems

2 files changed, 83 insertions, 45 deletions

diff --git a/Carpet/CarpetReduce/interface.ccl b/Carpet/CarpetReduce/interface.ccl
index 641ede55a..5e757d0e4 100644
--- a/Carpet/CarpetReduce/interface.ccl
+++ b/Carpet/CarpetReduce/interface.ccl
@@ -2,12 +2,15 @@
 
 IMPLEMENTS: reduce
 
+uses include header: nompi.h
+
 uses include header: defs.hh
 uses include header: dh.hh
 uses include header: dist.hh
 uses include header: vect.hh
 
 uses include header: carpet.hh
+uses include header: carpet.h
 
 uses include header: typecase.hh
 uses include header: typeprops.hh
@@ -21,28 +24,27 @@ CCTK_INT FUNCTION                                                \
 REQUIRES FUNCTION SymmetryTableHandleForGrid
 
 CCTK_INT FUNCTION                         \
+    MultiPatch_GetMaps                    \
+        (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION MultiPatch_GetMaps
+
+CCTK_INT FUNCTION                         \
     MultiPatch_GetMap                     \
         (CCTK_POINTER_TO_CONST IN cctkGH)
 USES FUNCTION MultiPatch_GetMap
 
-CCTK_INT FUNCTION                           \
-    MultiPatch_GetBoundarySpecification     \
-        (CCTK_INT IN map,                   \
-         CCTK_INT IN size,                  \
-         CCTK_INT OUT ARRAY nboundaryzones, \
-         CCTK_INT OUT ARRAY is_internal,    \
-         CCTK_INT OUT ARRAY is_staggered,   \
-         CCTK_INT OUT ARRAY shiftout)
-USES FUNCTION MultiPatch_GetBoundarySpecification
-
-CCTK_INT FUNCTION                           \
-    GetBoundarySpecification                \
-        (CCTK_INT IN size,                  \
-         CCTK_INT OUT ARRAY nboundaryzones, \
-         CCTK_INT OUT ARRAY is_internal,    \
-         CCTK_INT OUT ARRAY is_staggered,   \
-         CCTK_INT OUT ARRAY shiftout)
-REQUIRES FUNCTION GetBoundarySpecification
+CCTK_INT FUNCTION                          \
+    MultiPatch_GetDomainSpecification      \
+        (CCTK_INT IN map,                  \
+         CCTK_INT IN size,                 \
+         CCTK_REAL OUT ARRAY physical_min, \
+         CCTK_REAL OUT ARRAY physical_max, \
+         CCTK_REAL OUT ARRAY interior_min, \
+         CCTK_REAL OUT ARRAY interior_max, \
+         CCTK_REAL OUT ARRAY exterior_min, \
+         CCTK_REAL OUT ARRAY exterior_max, \
+         CCTK_REAL OUT ARRAY spacing)
+USES FUNCTION MultiPatch_GetDomainSpecification
 
 CCTK_INT FUNCTION                          \
     GetDomainSpecification                 \
diff --git a/Carpet/CarpetReduce/src/mask_test.c b/Carpet/CarpetReduce/src/mask_test.c
index fcaa74cc1..82fa4f306 100644
--- a/Carpet/CarpetReduce/src/mask_test.c
+++ b/Carpet/CarpetReduce/src/mask_test.c
@@ -2,6 +2,8 @@
 #include <cctk_Arguments.h>
 #include <cctk_Parameters.h>
 
+#include <carpet.h>
+
 #include <assert.h>
 #include <math.h>
 
@@ -44,44 +46,78 @@ MaskBase_TestMask (CCTK_ARGUMENTS)
   
   if (proc == -1 || CCTK_MyProc(cctkGH) == proc) {
     
-    CCTK_REAL physical_min[cctk_dim];
-    CCTK_REAL physical_max[cctk_dim];
-    CCTK_REAL interior_min[cctk_dim];
-    CCTK_REAL interior_max[cctk_dim];
-    CCTK_REAL exterior_min[cctk_dim];
-    CCTK_REAL exterior_max[cctk_dim];
-    CCTK_REAL spacing     [cctk_dim];
-    int const ierr = GetDomainSpecification (cctk_dim,
-                                             physical_min,
-                                             physical_max,
-                                             interior_min,
-                                             interior_max,
-                                             exterior_min,
-                                             exterior_max,
-                                             spacing);
-    assert (!ierr);
-    
-    CCTK_REAL domain_volume = 1.0;
-    for (int d=0; d<cctk_dim; ++d) {
-      domain_volume *= (physical_max[d] - physical_min[d]) / spacing[d];
+    if (verbose) {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "Reduction weight sum: %.17g", (double)sum_weight);
     }
     
     
     
-    int const there_is_a_problem =
-      fabs(sum_weight - domain_volume) > 1.0e-12 * (sum_weight + domain_volume);
+    CCTK_REAL domain_volume = 0.0;
     
+    int maps = 1;
+    if (CCTK_IsFunctionAliased ("MultiPatch_GetMaps")) {
+      maps = MultiPatch_GetMaps (cctkGH);
+    }
     
+    for (int m=0; m<maps; ++m) {
+      
+      CCTK_REAL physical_min[cctk_dim];
+      CCTK_REAL physical_max[cctk_dim];
+      CCTK_REAL interior_min[cctk_dim];
+      CCTK_REAL interior_max[cctk_dim];
+      CCTK_REAL exterior_min[cctk_dim];
+      CCTK_REAL exterior_max[cctk_dim];
+      CCTK_REAL spacing     [cctk_dim];
+      
+      if (CCTK_IsFunctionAliased ("MultiPatch_GetDomainSpecification")) {
+        int const ierr = MultiPatch_GetDomainSpecification
+          (m, cctk_dim,
+           physical_min, physical_max,
+           interior_min, interior_max,
+           exterior_min, exterior_max,
+           spacing);
+        assert (!ierr);
+      } else {
+        int const ierr = GetDomainSpecification
+          (cctk_dim,
+           physical_min, physical_max,
+           interior_min, interior_max,
+           exterior_min, exterior_max,
+           spacing);
+        assert (!ierr);
+      }
+      
+      CCTK_REAL map_volume = 1.0;
+      for (int d=0; d<cctk_dim; ++d) {
+        map_volume *= (physical_max[d] - physical_min[d]) / spacing[d];
+      }
     
-    if (verbose || there_is_a_problem) {
-      CCTK_VInfo (CCTK_THORNSTRING,
-                  "Simulation domain volume: %.15g", (double)domain_volume);
+      if (verbose) {
+        CCTK_VInfo (CCTK_THORNSTRING,
+                    "Volume of map #%d: %.17g", m, (double)map_volume);
+      }
+      
+      domain_volume += map_volume;
+      
+    } /* for m */
+    
+    if (verbose) {
       CCTK_VInfo (CCTK_THORNSTRING,
-                  "Reduction weight sum:     %.15g", (double)sum_weight);
+                  "Simulation domain volume: %.17g", (double)domain_volume);
     }
     
+    int const there_is_a_problem =
+      fabs(sum_weight - domain_volume) > 1.0e-12 * (sum_weight + domain_volume);
+    
     if (there_is_a_problem) {
-      CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
+      if (!verbose) {
+        CCTK_VInfo (CCTK_THORNSTRING,
+                    "Simulation domain volume: %.15g", (double)domain_volume);
+        CCTK_VInfo (CCTK_THORNSTRING,
+                    "Reduction weight sum:     %.15g", (double)sum_weight);
+      }
+      CCTK_VWarn (CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Simulation domain volume and reduction weight sum differ");
     }