Import the recently announced changes:

Import the recently announced changes:
1. Carpet has now an infrastructure for multiple maps (aka "grid
patches").  Instead of a single grid hierarchy there can now be
several.  This is largely untested, because the remainder of Cactus
cannot handle multiple coordinate systems.

2. The order in which the schedule bins are called has changed.  As Ian
Hawke pointed out, the previous order during time evolution was
inconsistent.  The initial data ordering did not allow for recovering
and was not usable for progressively solving elliptic equations for
initial data.

3. Carpet now supports convergence levels.  The convergence level
specifies by how many factors of two the resolution in the parameter
file should be coarsened (or refined, if negative).  This should make
convergence tests and test runs much easier.  It is, in principle, also
possible to run several convergence levels at once.  This has not been
tested because the remainder of Cactus cannot handle multiple
resolutions.  This will be necessary for a multigrid solver, and also
for having a shadow hierarchy to determine where to refine adaptively.

4. Carpet works together with the new CoordBase domain specification
parameters.  Without these, using convergence levels will lead to very
strange results.

5. The "modes" have changed.  There are now:
        meta mode:      the whole simulation
        global mode:    one convergence level
        level mode:     one refinement level
        singlemap mode: one map on one refinement level
        local mode:     as previously
The whole mode handling has been cleaned up.

6. The regridding thorn has been cleaned up.

7. The kind of prolongation stencil is now determined in Carpet, i.e. at
a fairly hight level, instead of in CarpetLib.

8. The low-order prolongation operators have been made much more
efficient (as have previously the higher-order ones).

9. Assorted smaller changes.

For Carpet users, there should be no major incompatibilities.  The major
improvements are 3 and 4 combined.  Here is an example:

CoordBase::domainsize    = extent
CoordBase::spacing       = gridspacing
CoordBase::zero_origin_x = yes
CoordBase::zero_origin_y = yes
CoordBase::zero_origin_z = yes
CoordBase::xextent       = 20.0
CoordBase::yextent       = 20.0
CoordBase::zextent       = 20.0
CoordBase::dx            = 1.0
CoordBase::dy            = 1.0
CoordBase::dz            = 1.0

CoordBase::boundary_shiftout_x_lower = 1
CoordBase::boundary_shiftout_y_lower = 1
CoordBase::boundary_shiftout_z_lower = 1

Carpet::domain_from_coordbase   = yes
Carpet::convergence_level       = 0

grid::type         = coordbase
grid::domain       = octant
grid::avoid_origin = no

This gives you a grid that extends from the origin ("zero_origin") up to
20.0 with a grid spacing of 1.0.  Symmetry zones and boundary zones are
added automatically.  The "shiftout" says that there is no boundary
point on the origin.  The staggering parameters (not shown) default to
"no".  In order to change the resolution, only the convergence level
has to be adjusted.  Note that the old way of specifying the domain
extent still works.

For Carpet developers, one major change is the new mode handling.  As
described in 5, the looping macros (that loop over all refinement
levels, or all components) have changed.

darcs-hash:20040125135727-07bb3-51c9647c1b5080e7e180b52a1b81fa155cfd19e9.gz

1 files changed, 100 insertions, 62 deletions

diff --git a/Carpet/CarpetReduce/src/reduce.cc b/Carpet/CarpetReduce/src/reduce.cc
index 4a37ddbeb..bf3c69b7f 100644
--- a/Carpet/CarpetReduce/src/reduce.cc
+++ b/Carpet/CarpetReduce/src/reduce.cc
@@ -1,4 +1,4 @@
-// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.30 2003/11/12 17:29:30 schnetter Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.31 2004/01/25 14:57:30 schnetter Exp $
 
 #include <assert.h>
 #include <float.h>
@@ -23,7 +23,7 @@
 #include "reduce.hh"
 
 extern "C" {
-  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.30 2003/11/12 17:29:30 schnetter Exp $";
+  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.31 2004/01/25 14:57:30 schnetter Exp $";
   CCTK_FILEVERSION(Carpet_CarpetReduce_reduce_cc);
 }
 
@@ -50,25 +50,26 @@ namespace CarpetReduce {
   template<> inline complex<long double> mymax (const complex<long double> x, const complex<long double> y) { return complex<long double> (max(x.real(), y.real()), max(x.imag(), y.imag())); }
 #endif
   
-  template<class T> inline T mymin () { return mymin(numeric_limits<T>::min(),-numeric_limits<T>::max()); }
-  template<> inline complex<float> mymin () { return complex<float> (mymin<float>(),mymin<float>()); }
-  template<> inline complex<double> mymin () { return complex<double> (mymin<double>(),mymin<double>()); }
+  template<class T> inline T mymin (const T& dummy) { return mymin(numeric_limits<T>::min(),-numeric_limits<T>::max()); }
+  template<> inline complex<float> mymin (const complex<float>& dummy) { return complex<float> (mymin<float>(dummy.real()),mymin<float>(dummy.imag())); }
+  template<> inline complex<double> mymin (const complex<double>& dummy) { return complex<double> (mymin<double>(dummy.real()),mymin<double>(dummy.imag())); }
 #ifdef LDBL_MAX
-  template<> inline complex<long double> mymin () { return complex<long double> (mymin<long double>(),mymin<long double>()); }
+  template<> inline complex<long double> mymin (const complex<long double>& dummy) { return complex<long double> (mymin<long double>(dummy.real()),mymin<long double>(dummy.imag())); }
 #endif
   
-  template<class T> inline T mymax () { return mymax(numeric_limits<T>::max(),-numeric_limits<T>::min()); }
-  template<> inline complex<float> mymax () { return complex<float> (mymax<float>(),mymax<float>()); }
-  template<> inline complex<double> mymax () { return complex<double> (mymax<double>(),mymax<double>()); }
+  template<class T> inline T mymax (const T& dummy) { return mymax(numeric_limits<T>::max(),-numeric_limits<T>::min()); }
+  template<> inline complex<float> mymax (const complex<float>& dummy) { return complex<float> (mymax<float>(dummy.real()),mymax<float>(dummy.imag())); }
+  template<> inline complex<double> mymax (const complex<double>& dummy) { return complex<double> (mymax<double>(dummy.real()),mymax<double>(dummy.imag())); }
 #ifdef LDBL_MAX
-  template<> inline complex<long double> mymax () { return complex<long double> (mymax<long double>(),mymax<long double>()); }
+  template<> inline complex<long double> mymax (const complex<long double>& dummy) { return complex<long double> (mymax<long double>(dummy.real()),mymax<long double>(dummy.imag())); }
 #endif
   
   
   
   // Poor man's RTTI
-  enum ared { do_count, do_minimum, do_maximum, do_product, do_sum, do_sum_abs,
-	      do_sum_squared, do_average, do_norm1, do_norm2, do_norm_inf };
+  enum ared { do_count, do_origin, do_minimum, do_maximum, do_product, do_sum,
+              do_sum_abs, do_sum_squared, do_average, do_norm1, do_norm2,
+              do_norm_inf };
   
   
   
@@ -94,13 +95,26 @@ namespace CarpetReduce {
     MPI_Op mpi_op () const { return MPI_SUM; }
   };
   
+  struct origin : reduction {
+    origin () { }
+    ared thered () const { return do_origin; }
+    bool uses_cnt () const { return false; }
+    template<class T>
+    struct op {
+      static inline void initialise (T& accum) { accum = 0; }
+      static inline void reduce (T& accum, const T& val) { assert(0); }
+      static inline void finalise (T& accum, const T& cnt) { }
+    };
+    MPI_Op mpi_op () const { return MPI_SUM; }
+  };
+  
   struct minimum : reduction {
     minimum () { }
     ared thered () const { return do_minimum; }
     bool uses_cnt () const { return false; }
     template<class T>
     struct op {
-      static inline void initialise (T& accum) { accum = mymax<T>(); }
+      static inline void initialise (T& accum) { accum = mymax(accum); }
       static inline void reduce (T& accum, const T& val) { accum = mymin(accum, val); }
       static inline void finalise (T& accum, const T& cnt) { }
     };
@@ -113,7 +127,7 @@ namespace CarpetReduce {
     bool uses_cnt () const { return false; }
     template<class T>
     struct op {
-      static inline void initialise (T& accum) { accum = mymin<T>(); }
+      static inline void initialise (T& accum) { accum = mymin(accum); }
       static inline void reduce (T& accum, const T& val) { accum = mymax(accum, val); }
       static inline void finalise (T& accum, const T& cnt) { }
     };
@@ -307,6 +321,7 @@ namespace CarpetReduce {
       case N: {					\
 	switch (red->thered()) {		\
 	  INITIALISE(count,T);			\
+	  INITIALISE(origin,T);			\
 	  INITIALISE(minimum,T);		\
 	  INITIALISE(maximum,T);		\
 	  INITIALISE(product,T);		\
@@ -435,6 +450,7 @@ namespace CarpetReduce {
       case N: {					\
 	switch (red->thered()) {		\
 	  REDUCE(count,T);			\
+	  REDUCE(origin,T);			\
 	  REDUCE(minimum,T);			\
 	  REDUCE(maximum,T);			\
 	  REDUCE(product,T);			\
@@ -524,6 +540,7 @@ namespace CarpetReduce {
 	case N: {				\
 	  switch (red->thered()) {		\
 	    FINALISE(count,T);			\
+	    FINALISE(origin,T);			\
 	    FINALISE(minimum,T);		\
 	    FINALISE(maximum,T);		\
 	    FINALISE(product,T);		\
@@ -678,7 +695,10 @@ namespace CarpetReduce {
     }
     
     // global mode
-    if (! reduce_arrays && reflevel == -1) {
+    if (! reduce_arrays && is_meta_mode()) {
+      CCTK_WARN (0, "Grid function reductions are not possible in meta mode");
+    }
+    if (! reduce_arrays && is_global_mode()) {
       CCTK_WARN (0, "Grid function reduction operators in global mode are not yet implemented");
     }
     
@@ -690,53 +710,68 @@ namespace CarpetReduce {
     Initialise (cgh, proc, num_invars * num_outvals, &myoutvals[0], outtype,
                 &mycounts[0], red);
     
-    BEGIN_LOCAL_COMPONENT_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) {
-      
-      assert (grpdim<=dim);
-      int lsh[dim], bbox[2*dim], nghostzones[dim];
-      ierr = CCTK_GrouplshVI(cgh, grpdim, lsh, vi);
-      assert (!ierr);
-      ierr = CCTK_GroupbboxVI(cgh, 2*grpdim, bbox, vi);
-      assert (!ierr);
-      ierr = CCTK_GroupnghostzonesVI(cgh, grpdim, nghostzones, vi);
-      assert (!ierr);
-      for (int d=0; d<grpdim; ++d) {
-        assert (lsh[d]>=0);
-        assert (nghostzones[d]>=0 && 2*nghostzones[d]<=lsh[d]);
-      }
-      
-      vector<const void*> inarrays (num_invars);
-      for (int n=0; n<num_invars; ++n) {
-        inarrays[n] = CCTK_VarDataPtrI(cgh, 0, invars[n]);
-        assert (inarrays[n]);
-      }
-      
-      const int intype = CCTK_VarTypeI(vi);
-      for (int n=0; n<num_invars; ++n) {
-        assert (CCTK_VarTypeI(invars[n]) == intype);
-      }
-      
-      vect<int,dim> mylsh, mynghostzones;
-      vect<vect<int,2>,dim> mybbox;
-      for (int d=0; d<grpdim; ++d) {
-        mylsh[d] = lsh[d];
-        mybbox[d][0] = bbox[2*d  ];
-        mybbox[d][1] = bbox[2*d+1];
-        mynghostzones[d] = nghostzones[d];
-      }
-      for (int d=grpdim; d<dim; ++d) {
-        mylsh[d] = 1;
-        mybbox[d][0] = 0;
-        mybbox[d][1] = 0;
-        mynghostzones[d] = 0;
-      }
-      
-      Reduce (cgh, proc, &mylsh[0], &mybbox[0][0], &mynghostzones[0],
-              num_invars, &inarrays[0], intype,
-              num_invars * num_outvals, &myoutvals[0], outtype,
-              &mycounts[0], red);
-      
-    } END_LOCAL_COMPONENT_LOOP;
+    int const saved_reflevel = reflevel;
+    int const saved_map = Carpet::map;
+    int const saved_component = component;
+    
+    BEGIN_GLOBAL_MODE(cgh) {
+      BEGIN_REFLEVEL_LOOP(cgh) {
+        BEGIN_MAP_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) {
+          BEGIN_LOCAL_COMPONENT_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) {
+            if (reduce_arrays
+                || ((saved_reflevel==-1 || reflevel==saved_reflevel)
+                    && (saved_map==-1 || Carpet::map==saved_map)
+                    && (saved_component==-1 || component==saved_component))) {
+              
+              assert (grpdim<=dim);
+              int lsh[dim], bbox[2*dim], nghostzones[dim];
+              ierr = CCTK_GrouplshVI(cgh, grpdim, lsh, vi);
+              assert (!ierr);
+              ierr = CCTK_GroupbboxVI(cgh, 2*grpdim, bbox, vi);
+              assert (!ierr);
+              ierr = CCTK_GroupnghostzonesVI(cgh, grpdim, nghostzones, vi);
+              assert (!ierr);
+              for (int d=0; d<grpdim; ++d) {
+                assert (lsh[d]>=0);
+                assert (nghostzones[d]>=0 && 2*nghostzones[d]<=lsh[d]);
+              }
+              
+              vector<const void*> inarrays (num_invars);
+              for (int n=0; n<num_invars; ++n) {
+                inarrays[n] = CCTK_VarDataPtrI(cgh, 0, invars[n]);
+                assert (inarrays[n]);
+              }
+              
+              const int intype = CCTK_VarTypeI(vi);
+              for (int n=0; n<num_invars; ++n) {
+                assert (CCTK_VarTypeI(invars[n]) == intype);
+              }
+              
+              vect<int,dim> mylsh, mynghostzones;
+              vect<vect<int,2>,dim> mybbox;
+              for (int d=0; d<grpdim; ++d) {
+                mylsh[d] = lsh[d];
+                mybbox[d][0] = bbox[2*d  ];
+                mybbox[d][1] = bbox[2*d+1];
+                mynghostzones[d] = nghostzones[d];
+              }
+              for (int d=grpdim; d<dim; ++d) {
+                mylsh[d] = 1;
+                mybbox[d][0] = 0;
+                mybbox[d][1] = 0;
+                mynghostzones[d] = 0;
+              }
+              
+              Reduce (cgh, proc, &mylsh[0], &mybbox[0][0], &mynghostzones[0],
+                      num_invars, &inarrays[0], intype,
+                      num_invars * num_outvals, &myoutvals[0], outtype,
+                      &mycounts[0], red);
+              
+            }
+          } END_LOCAL_COMPONENT_LOOP;
+        } END_MAP_LOOP;
+      } END_REFLEVEL_LOOP;
+    } END_GLOBAL_MODE;
     
     Finalise (cgh, proc, num_invars * num_outvals, outvals, outtype,
               &myoutvals[0], &mycounts[0], red);
@@ -773,6 +808,7 @@ namespace CarpetReduce {
   }
   
   REDUCTION(count);
+  REDUCTION(origin);
   REDUCTION(minimum);
   REDUCTION(maximum);
   REDUCTION(product);
@@ -791,6 +827,7 @@ namespace CarpetReduce {
   void CarpetReduceStartup ()
   {
     CCTK_RegisterReductionOperator (count_GVs,       "count");
+    CCTK_RegisterReductionOperator (origin_GVs,      "origin");
     CCTK_RegisterReductionOperator (minimum_GVs,     "minimum");
     CCTK_RegisterReductionOperator (maximum_GVs,     "maximum");
     CCTK_RegisterReductionOperator (product_GVs,     "product");
@@ -803,6 +840,7 @@ namespace CarpetReduce {
     CCTK_RegisterReductionOperator (norm_inf_GVs,    "norm_inf");
     
     CCTK_RegisterReductionArrayOperator (count_arrays,       "count");
+    CCTK_RegisterReductionArrayOperator (origin_arrays,      "origin");
     CCTK_RegisterReductionArrayOperator (minimum_arrays,     "minimum");
     CCTK_RegisterReductionArrayOperator (maximum_arrays,     "maximum");
     CCTK_RegisterReductionArrayOperator (product_arrays,     "product");