Carpet: Remove dependency on LoopControl; use NoMPI

8 files changed, 74 insertions, 31 deletions

diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl
index 3dad22724..b4fa05583 100644
--- a/Carpet/Carpet/interface.ccl
+++ b/Carpet/Carpet/interface.ccl
@@ -7,6 +7,8 @@ include header: carpet_public.h in carpet.h
 
 include header: Timers.hh in CarpetTimers.hh
 
+uses include header: nompi.h
+
 uses include header: loopcontrol.h
 
 uses include header: mpi_string.hh
diff --git a/Carpet/Carpet/src/MultiModel.cc b/Carpet/Carpet/src/MultiModel.cc
index c59eb44aa..7f4db58bc 100644
--- a/Carpet/Carpet/src/MultiModel.cc
+++ b/Carpet/Carpet/src/MultiModel.cc
@@ -1,3 +1,5 @@
+#include <cctk.h>
+
 #include <cassert>
 #include <cstring>
 #include <iostream>
@@ -5,9 +7,11 @@
 #include <string>
 #include <vector>
 
-#include <mpi.h>
-
-#include <cctk.h>
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 #include <functions.hh>
 #include <mpi_string.hh>
diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc
index 257a92e8d..102e2445c 100644
--- a/Carpet/Carpet/src/Recompose.cc
+++ b/Carpet/Carpet/src/Recompose.cc
@@ -1,3 +1,6 @@
+#include <cctk.h>
+#include <cctk_Parameters.h>
+
 #include <algorithm>
 #include <cassert>
 #include <cmath>
@@ -16,10 +19,11 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include <mpi.h>
-
-#include <cctk.h>
-#include <cctk_Parameters.h>
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 #include <loopcontrol.h>
 
@@ -1954,14 +1958,10 @@ namespace Carpet {
             BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
               DECLARE_CCTK_ARGUMENTS;
 #pragma omp parallel
-              LC_LOOP3 (CarpetClassifyPoints,
-                        i,j,k,
-                        0,0,0, cctk_lsh[0],cctk_lsh[1],cctk_lsh[2],
-                        cctk_lsh[0],cctk_lsh[1],cctk_lsh[2])
-              {
+              CCTK_LOOP3_ALL(CarpetClassifyPoints, cctkGH, i,j,k) {
                 int const ind = CCTK_GFINDEX3D (cctkGH, i, j, k);
                 point_class[ind] = 1;
-              } LC_ENDLOOP3(CarpetClassifyPoints);
+              } CCTK_ENDLOOP3_ALL(CarpetClassifyPoints);
             } END_LOCAL_COMPONENT_LOOP;
           } END_LOCAL_MAP_LOOP;
         } LEAVE_LEVEL_MODE;
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index 0fbc657f0..8a45f7893 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -10,8 +10,6 @@
 #include <string>
 #include <vector>
 
-#include <mpi.h>
-
 #include <cctk.h>
 #include <cctk_Parameters.h>
 #include <util_ErrorCodes.h>
@@ -165,6 +163,8 @@ namespace Carpet {
   static void
   ensure_CartGrid3D_type ();
   static void
+  ensure_CartGrid3D_domain ();  // UNUSED
+  static void
   ensure_CartGrid3D_avoid_origin ();
   static void
   ensure_ReflectionSymmetry_avoid_origin (centering refcentering);
@@ -838,9 +838,9 @@ namespace Carpet {
     DECLARE_CCTK_PARAMETERS;
     
     // Calculate base extent
-    ivect const lb(0);
-    ivect const ub(sizes-1);
     ivect const str(1);
+    ivect const lb(0);
+    ivect const ub((sizes-1) * str);
     ibbox const baseext(lb, ub, str);
     vector <vector <ibbox> > baseexts (1);
     baseexts.AT(0).resize (1);
@@ -2250,6 +2250,28 @@ namespace Carpet {
   
   
   
+  // UNUSED:
+  // Ensure that CartGrid3D doesn't apply symmetries
+  void
+  ensure_CartGrid3D_domain ()
+  {
+    if (CCTK_IsThornActive ("CartGrid3D")) {
+      int type;
+      void const * ptr;
+      
+      ptr = CCTK_ParameterGet ("domain", "CartGrid3D", & type);
+      assert (ptr != 0);
+      assert (type == PARAMETER_KEYWORD);
+      char const * const domain
+        = * static_cast<char const * const *> (ptr);
+      if (not CCTK_EQUALS (domain, "full")) {
+        CCTK_WARN (0, "When Carpet::domain_from_coordbase = no, and when Carpet::max_refinement_levels > 1, then thorn CartGrid3D cannot provide symmetry boundaries");
+      }
+    }
+  }
+  
+  
+  
   // Ensure that CartGrid3D::avoid_origin = no
   void
   ensure_CartGrid3D_avoid_origin ()
diff --git a/Carpet/Carpet/src/carpet_public.h b/Carpet/Carpet/src/carpet_public.h
index 20a265848..b47d369c2 100644
--- a/Carpet/Carpet/src/carpet_public.h
+++ b/Carpet/Carpet/src/carpet_public.h
@@ -1,10 +1,13 @@
 #ifndef CARPET_PUBLIC_H
 #define CARPET_PUBLIC_H
 
-#include <mpi.h>
-
 #include <cctk.h>
 
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 
 /* Tell thorns that the Carpet routines exist */
diff --git a/Carpet/Carpet/src/functions.hh b/Carpet/Carpet/src/functions.hh
index 676abea57..3ae48a66b 100644
--- a/Carpet/Carpet/src/functions.hh
+++ b/Carpet/Carpet/src/functions.hh
@@ -1,14 +1,18 @@
 #ifndef FUNCTIONS_HH
 #define FUNCTIONS_HH
 
+#include <cctk.h>
+#include <cctk_Schedule.h>
+
 #include <map>
 #include <string>
 #include <vector>
 
-#include <mpi.h>
-
-#include <cctk.h>
-#include <cctk_Schedule.h>
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 #include <bbox.hh>
 #include <dh.hh>
diff --git a/Carpet/Carpet/src/helpers.cc b/Carpet/Carpet/src/helpers.cc
index c184c8c3f..852d86a8c 100644
--- a/Carpet/Carpet/src/helpers.cc
+++ b/Carpet/Carpet/src/helpers.cc
@@ -1,14 +1,18 @@
+#include <cctk.h>
+#include <cctk_FortranString.h>
+#include <cctk_Parameters.h>
+
 #include <cassert>
 #include <cstdarg>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 
-#include <mpi.h>
-
-#include <cctk.h>
-#include <cctk_FortranString.h>
-#include <cctk_Parameters.h>
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 #include <defs.hh>
 #include <dist.hh>
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index 61ac33cc2..661986a14 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -12,11 +12,15 @@
 #ifndef VARIABLES_HH
 #define VARIABLES_HH
 
-#include <vector>
+#include <cctk.h>
 
-#include <mpi.h>
+#include <vector>
 
-#include <cctk.h>
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
 
 #include <bbox.hh>
 #include <data.hh>