Made WaveToyF77 run on multiple processors with CarpetSlab.

Made WaveToyF77 run on multiple processors with CarpetSlab.
Addes "bases" in gh and dh.  They contain the extent of a bounding box
enclosing all components on a level.

darcs-hash:20010307120057-f6438-829d3c60a80c7b13f850c0ad187257bb6e1b963f.gz

10 files changed, 267 insertions, 136 deletions

diff --git a/Carpet/CarpetLib/src/bbox.cc b/Carpet/CarpetLib/src/bbox.cc
index 7c24c7954..7ae17fe5d 100644
--- a/Carpet/CarpetLib/src/bbox.cc
+++ b/Carpet/CarpetLib/src/bbox.cc
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.cc,v 1.2 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.cc,v 1.3 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -54,6 +54,19 @@ bbox<T,D>::bbox (const vect<T,D>& lower, const vect<T,D>& upper,
   assert (all((upper-lower)%stride==0));
 }
 
+// Accessors
+template<class T, int D>
+T bbox<T,D>::size () const {
+  if (empty()) return 0;
+  return prod(shape());
+}
+
+template<class T, int D>
+T bbox<T,D>::num_points () const {
+  if (empty()) return 0;
+  return prod((shape()+stride()-1)/stride());
+}
+
 // Queries
 template<class T, int D>
 bool bbox<T,D>::contains (const vect<T,D>& x) const {
@@ -162,6 +175,42 @@ bbox<T,D> bbox<T,D>::contracted_for (const bbox& b) const {
   return bbox(lo,up,str);
 }
 
+// Set operations
+// Smallest bbox containing both boxes
+template<class T, int D>
+bbox<T,D> bbox<T,D>::operator* (const bbox& b) const {
+  if (empty()) return b;
+  if (b.empty()) return *this;
+  assert (aligned_with(b));
+  const vect<T,D> lo = min(lower(), b.lower());
+  const vect<T,D> up = max(upper(), b.upper());
+  const vect<T,D> str = min(stride(), b.stride());
+  return bbox(lo,up,str);
+}
+
+template<class T, int D>
+bbox<T,D>& bbox<T,D>::operator*= (const bbox& b) {
+  *this = *this * b;
+  return *this;
+}
+
+// Largest bbox inside both boxes
+template<class T, int D>
+bbox<T,D> bbox<T,D>::operator+ (const bbox& b) const {
+  if (empty() || b.empty()) return bbox();
+  assert (aligned_with(b));
+  const vect<T,D> lo = max(lower(), b.lower());
+  const vect<T,D> up = min(upper(), b.upper());
+  const vect<T,D> str = min(stride(), b.stride());
+  return bbox(lo,up,str);
+}
+
+template<class T, int D>
+bbox<T,D>& bbox<T,D>::operator+= (const bbox& b) {
+  *this = *this + b;
+  return *this;
+}
+
 // Iterators
 template<class T, int D>
 bbox<T,D>::iterator::iterator (const bbox& box, const vect<T,D>& pos)
diff --git a/Carpet/CarpetLib/src/bbox.hh b/Carpet/CarpetLib/src/bbox.hh
index 5cc6fa1ff..6787b9240 100644
--- a/Carpet/CarpetLib/src/bbox.hh
+++ b/Carpet/CarpetLib/src/bbox.hh
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.hh,v 1.2 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.hh,v 1.3 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -42,7 +42,7 @@ template<class T, int D>
 class bbox {
   
   // Fields
-  vect<T,D> _lower, _upper, _stride;// bounds are inclusive
+  vect<T,D> _lower, _upper, _stride; // bounds are inclusive
   
 public:
   
@@ -63,15 +63,8 @@ public:
     return any(lower()>upper());
   }
   
-  T size () const {
-    if (empty()) return 0;
-    return prod(shape());
-  }
-  
-  T num_points () const {
-    if (empty()) return 0;
-    return prod((shape()+stride()-1)/stride());
-  }
+  T size () const;
+  T num_points () const;
   
   // Queries
   bool contains (const vect<T,D>& x) const;
@@ -102,6 +95,14 @@ public:
   // Find the largest b-compatible box inside *this
   bbox contracted_for (const bbox& b) const;
   
+  // Set operations
+  // Smallest bbox containing both boxes
+  bbox operator* (const bbox& b) const;
+  bbox& operator*= (const bbox& b);
+  // Largest bbox inside both boxes
+  bbox operator+ (const bbox& b) const;
+  bbox& operator+= (const bbox& b);
+  
   // Iterators
   class iterator {
   protected:
diff --git a/Carpet/CarpetLib/src/defs.hh b/Carpet/CarpetLib/src/defs.hh
index 741678370..bc2547d87 100644
--- a/Carpet/CarpetLib/src/defs.hh
+++ b/Carpet/CarpetLib/src/defs.hh
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.hh,v 1.1 2001/03/01 13:40:10 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.hh,v 1.2 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -42,6 +42,7 @@
 // A general type
 enum centering { vertex_centered, cell_centered };
 
+// Useful helper
 template<class T>
 inline T square (const T& x) { return x*x; }
 
diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index 623dd1b51..5dc7ba0e7 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -6,7 +6,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.2 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.3 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -30,7 +30,6 @@
 #  include "dh.hh"
 #endif
 
-#define DEBUG_DIST DIST_NODEBUG
 #undef DEBUG_OUTPUT
 
 
@@ -41,7 +40,7 @@ dh<D>::dh (gh<D>& h, const ivect& lghosts, const ivect& ughosts)
   : h(h), lghosts(lghosts), ughosts(ughosts)
 {
   assert (all(lghosts>=0 && ughosts>=0));
-  DEBUG_DIST;
+  CHECKPOINT;
   h.add(this);
   recompose();
 }
@@ -49,14 +48,14 @@ dh<D>::dh (gh<D>& h, const ivect& lghosts, const ivect& ughosts)
 // Destructors
 template<int D>
 dh<D>::~dh () {
-  DEBUG_DIST;
+  CHECKPOINT;
   h.remove(this);
 }
 
 // Modifiers
 template<int D>
 void dh<D>::recompose () {
-  DEBUG_DIST;
+  CHECKPOINT;
   
   boxes.clear();
   
@@ -209,6 +208,23 @@ void dh<D>::recompose () {
     } // for c
   } // for rl
   
+  bases.resize(h.reflevels());
+  for (int rl=0; rl<h.reflevels(); ++rl) {
+    if (h.components(rl)==0) {
+      bases[rl].resize(0);
+    } else {
+      bases[rl].resize(h.mglevels(rl,0));
+      for (int ml=0; ml<h.mglevels(rl,0); ++ml) {
+	bases[rl][ml].exterior = ibbox();
+	bases[rl][ml].interior = ibbox();
+	for (int c=0; c<h.components(rl); ++c) {
+	  bases[rl][ml].exterior *= boxes[rl][c][ml].exterior;
+	  bases[rl][ml].interior *= boxes[rl][c][ml].interior;
+	}
+      }
+    }
+  }
+  
 #ifdef DEBUG_OUTPUT
   for (int rl=0; rl<h.reflevels(); ++rl) {
     for (int c=0; c<h.components(rl); ++c) {
@@ -234,6 +250,17 @@ void dh<D>::recompose () {
       }
     }
   }
+  for (int rl=0; rl<h.reflevels(); ++rl) {
+    if (h.components(rl)>0) {
+      for (int ml=0; ml<h.mglevels(rl,c); ++ml) {
+      	cout << endl;
+      	cout << "dh bases:" << endl;
+      	cout << "rl=" << rl << " ml=" << ml << endl;
+      	cout << "exterior=" << bases[rl][ml].exterior << endl;
+      	cout << "interior=" << bases[rl][ml].interior << endl;
+      }
+    }
+  }
 #endif
   
   for (list<generic_gf<D>*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
@@ -244,13 +271,13 @@ void dh<D>::recompose () {
 // Grid function management
 template<int D>
 void dh<D>::add (generic_gf<D>* f) {
-  DEBUG_DIST;
+  CHECKPOINT;
   gfs.push_back(f);
 }
 
 template<int D>
 void dh<D>::remove (generic_gf<D>* f) {
-  DEBUG_DIST;
+  CHECKPOINT;
   gfs.remove(f);
 }
 
diff --git a/Carpet/CarpetLib/src/dh.hh b/Carpet/CarpetLib/src/dh.hh
index 0d3c4c716..8960e60d1 100644
--- a/Carpet/CarpetLib/src/dh.hh
+++ b/Carpet/CarpetLib/src/dh.hh
@@ -6,7 +6,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.hh,v 1.1 2001/03/01 13:40:10 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.hh,v 1.2 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -77,10 +77,18 @@ class dh {
     iblistvect recv_ref_bnd_coarse;
   };
   
+  struct dbases {
+    ibbox exterior;		// whole region (including boundaries)
+    ibbox interior;		// interior (without boundaries)
+  };
+  
   typedef vector<dboxes> mboxes; // ... for each multigrid level
   typedef vector<mboxes> cboxes; // ... for each component
   typedef vector<cboxes> rboxes; // ... for each refinement level
   
+  typedef vector<dbases> mbases; // ... for each multigrid level
+  typedef vector<mbases> rbases; // ... for each refinement level
+  
 public:				// should be readonly
   
   // Fields
@@ -88,6 +96,7 @@ public:				// should be readonly
   ivect lghosts, ughosts;	// ghost zones
   
   rboxes boxes;
+  rbases bases;
   
   list<generic_gf<D>*> gfs;
   
diff --git a/Carpet/CarpetLib/src/dist.cc b/Carpet/CarpetLib/src/dist.cc
index e6c1e026a..9aa613764 100644
--- a/Carpet/CarpetLib/src/dist.cc
+++ b/Carpet/CarpetLib/src/dist.cc
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dist.cc,v 1.2 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dist.cc,v 1.3 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -22,6 +22,9 @@
 
 #include <mpi.h>
 
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
 #include "defs.hh"
 
 #if !defined(TMPL_IMPLICIT) || !defined(DIST_HH)
@@ -30,22 +33,48 @@
 
 
 
-MPI_Comm dist::comm;
-
-
-
-void dist::init (int& argc, char**& argv) {
-  MPI_Init (&argc, &argv);
-  pseudoinit();
-}
-
-void dist::pseudoinit () {
-  comm = MPI_COMM_WORLD;
-}
-
-void dist::finalize () {
-  MPI_Finalize ();
-}
+namespace dist {
+  
+  MPI_Comm comm;
+  
+  MPI_Datatype mpi_complex_float;
+  MPI_Datatype mpi_complex_double;
+  MPI_Datatype mpi_complex_long_double;
+  
+  void init (int& argc, char**& argv) {
+    MPI_Init (&argc, &argv);
+    pseudoinit();
+  }
+  
+  void pseudoinit () {
+    comm = MPI_COMM_WORLD;
+    
+    MPI_Type_contiguous (2, MPI_FLOAT, &mpi_complex_float);
+    MPI_Type_commit (&mpi_complex_float);
+    MPI_Type_contiguous (2, MPI_DOUBLE, &mpi_complex_double);
+    MPI_Type_commit (&mpi_complex_double);
+    MPI_Type_contiguous (2, MPI_LONG_DOUBLE, &mpi_complex_long_double);
+    MPI_Type_commit (&mpi_complex_long_double);
+  }
+  
+  void finalize () {
+    MPI_Finalize ();
+  }
+  
+  void checkpoint (const char* file, int line) {
+    DECLARE_CCTK_PARAMETERS;
+    if (verbose) {
+      int rank;
+      MPI_Comm_rank (comm, &rank);
+      printf ("CHECKPOINT: processor %d, file %s, line %d\n",
+	      rank, file, line);
+    }
+    if (barriers) {
+      MPI_Barrier (comm);
+    }
+  }
+  
+} // namespace dist
 
 
 
diff --git a/Carpet/CarpetLib/src/dist.hh b/Carpet/CarpetLib/src/dist.hh
index e62bb94da..dac83076f 100644
--- a/Carpet/CarpetLib/src/dist.hh
+++ b/Carpet/CarpetLib/src/dist.hh
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dist.hh,v 1.2 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dist.hh,v 1.3 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -25,111 +25,111 @@
 #include <cstdio>
 #include <cstdlib>
 
+#include <complex>
+
 #include <mpi.h>
 
 #include "defs.hh"
 
 
 
-// A checkpoint for debugging purposes
-#define DIST_VERBOSE						\
-do {								\
-  int rank;							\
-  MPI_Comm_rank (dist::comm, &rank);				\
-  printf ("CHECKPOINT: processor %d, file %s, line %d\n",	\
-	  rank, __FILE__, __LINE__);				\
-} while(0)
-
-// A barrier for debugging purposes
-#define DIST_BARRIER				\
-do {						\
-  MPI_Barrier(dist::comm);			\
-} while(0)
-
-// Both of the above
-#define DIST_VERBOSE_BARRIER			\
-do {						\
-  DIST_VERBOSE;					\
-  DIST_BARRIER;					\
-} while(0)
-
-// Do nothing
-#define DIST_NODEBUG do {} while(0)
-
-
-
-struct dist {
+namespace dist {
+  
+  const int tag = 1;
+  
+  extern MPI_Comm comm;
+  
+  extern MPI_Datatype mpi_complex_float;
+  extern MPI_Datatype mpi_complex_double;
+  extern MPI_Datatype mpi_complex_long_double;
+  
+  void init (int& argc, char**& argv);
+  void pseudoinit ();
+  void finalize ();
   
-  static const int tag = 1;
+  // Debugging output
+#define CHECKPOINT dist::checkpoint(__FILE__, __LINE__)
+  void checkpoint (const char* file, int line);
   
-  static MPI_Comm comm;
   
-  static void init (int& argc, char**& argv);
-  static void pseudoinit ();
-  static void finalize ();
   
+  // Datatype helpers
+  
+  // This generic routine is only declared and not defined.  Only
+  // specialised versions have definitions.  Type errors will be
+  // caught by the compiler.
   template<class T>
-  static MPI_Datatype datatype (const T& dummy);
+  MPI_Datatype datatype (const T& dummy);
   
-};
-
-
-
-template<class T>
-inline MPI_Datatype dist::datatype (const T& dummy)
-{ abort(); return -1; }
-
-template<>
-inline MPI_Datatype dist::datatype (const char& dummy)
-{ return MPI_CHAR; }
-
-template<>
-inline MPI_Datatype dist::datatype (const signed char& dummy)
-{ return MPI_UNSIGNED_CHAR; }
+  template<class T>
+  inline MPI_Datatype datatype (const T& dummy)
+  { abort(); return -1; }
   
-template<>
-inline MPI_Datatype dist::datatype (const unsigned char& dummy)
-{ return MPI_BYTE; }
-
-template<>
-inline MPI_Datatype dist::datatype (const short& dummy)
-{ return MPI_SHORT; }
-
-template<>
-inline MPI_Datatype dist::datatype (const unsigned short& dummy)
-{ return MPI_UNSIGNED_SHORT; }
-
-template<>
-inline MPI_Datatype dist::datatype (const int& dummy)
-{ return MPI_INT; }
-
-template<>
-inline MPI_Datatype dist::datatype (const unsigned int& dummy)
-{ return MPI_UNSIGNED; }
-
-template<>
-inline MPI_Datatype dist::datatype (const long& dummy)
-{ return MPI_LONG; }
-
-template<>
-inline MPI_Datatype dist::datatype (const unsigned long& dummy)
-{ return MPI_UNSIGNED_LONG; }
+  template<>
+  inline MPI_Datatype datatype (const char& dummy)
+  { return MPI_CHAR; }
   
-template<>
-inline MPI_Datatype dist::datatype (const long long& dummy)
-{ return MPI_LONG_LONG_INT; }
-
-template<>
-inline MPI_Datatype dist::datatype (const float& dummy)
-{ return MPI_FLOAT; }
-
-template<>
-inline MPI_Datatype dist::datatype (const double& dummy)
-{ return MPI_DOUBLE; }
-
-template<>
-inline MPI_Datatype dist::datatype (const long double& dummy)
-{ return MPI_LONG_DOUBLE; }
+  template<>
+  inline MPI_Datatype datatype (const signed char& dummy)
+  { return MPI_UNSIGNED_CHAR; }
+  
+  template<>
+  inline MPI_Datatype datatype (const unsigned char& dummy)
+  { return MPI_BYTE; }
+  
+  template<>
+  inline MPI_Datatype datatype (const short& dummy)
+  { return MPI_SHORT; }
+  
+  template<>
+  inline MPI_Datatype datatype (const unsigned short& dummy)
+  { return MPI_UNSIGNED_SHORT; }
+  
+  template<>
+  inline MPI_Datatype datatype (const int& dummy)
+  { return MPI_INT; }
+  
+  template<>
+  inline MPI_Datatype datatype (const unsigned int& dummy)
+  { return MPI_UNSIGNED; }
+  
+  template<>
+  inline MPI_Datatype datatype (const long& dummy)
+  { return MPI_LONG; }
+  
+  template<>
+  inline MPI_Datatype datatype (const unsigned long& dummy)
+  { return MPI_UNSIGNED_LONG; }
+  
+  template<>
+  inline MPI_Datatype datatype (const long long& dummy)
+  { return MPI_LONG_LONG_INT; }
+  
+  template<>
+  inline MPI_Datatype datatype (const float& dummy)
+  { return MPI_FLOAT; }
+  
+  template<>
+  inline MPI_Datatype datatype (const double& dummy)
+  { return MPI_DOUBLE; }
+  
+  template<>
+  inline MPI_Datatype datatype (const long double& dummy)
+  { return MPI_LONG_DOUBLE; }
+  
+  template<>
+  inline MPI_Datatype datatype (const complex<float>& dummy)
+  { return mpi_complex_float; }
+  
+  template<>
+  inline MPI_Datatype datatype (const complex<double>& dummy)
+  { return mpi_complex_double; }
+  
+  template<>
+  inline MPI_Datatype datatype (const complex<long double>& dummy)
+  { return mpi_complex_long_double; }
+  
+} // namespace dist
 
 
 
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index ee80c6d91..36889de97 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.3 2001/03/05 21:48:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.4 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -30,8 +30,6 @@
 #  include "gdata.hh"
 #endif
 
-#define DEBUG_DIST DIST_NODEBUG
-
 
 
 // Constructors
@@ -57,7 +55,7 @@ void generic_data<D>::write_ascii (const string name, const int time,
   const
 {
   assert (_has_storage);
-  DEBUG_DIST;
+  CHECKPOINT;
   
   if (_proc==0) {
     // output on processor 0
diff --git a/Carpet/CarpetLib/src/gh.cc b/Carpet/CarpetLib/src/gh.cc
index 5002f5199..13a674931 100644
--- a/Carpet/CarpetLib/src/gh.cc
+++ b/Carpet/CarpetLib/src/gh.cc
@@ -7,7 +7,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.cc,v 1.1 2001/03/01 13:40:10 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.cc,v 1.2 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -124,6 +124,22 @@ void gh<D>::recompose (const rexts& exts, const rprocs& procs) {
     assert (all.size() == s);
   }
   
+  // Calculate base extents of all levels
+  bases.resize(reflevels());
+  for (int rl=0; rl<reflevels(); ++rl) {
+    if (components(rl)==0) {
+      bases[rl].resize(0);
+    } else {
+      bases[rl].resize(mglevels(rl,0));
+      for (int ml=0; ml<mglevels(rl,0); ++ml) {
+	bases[rl][ml] = ibbox();
+	for (int c=0; c<components(rl); ++c) {
+	  bases[rl][ml] *= extents[rl][c][ml];
+	}
+      }
+    }
+  }
+  
   // Recompose the other hierarchies
   
   for (list<th<D>*>::iterator t=ths.begin(); t!=ths.end(); ++t) {
diff --git a/Carpet/CarpetLib/src/gh.hh b/Carpet/CarpetLib/src/gh.hh
index 1d185bee5..ff90a9476 100644
--- a/Carpet/CarpetLib/src/gh.hh
+++ b/Carpet/CarpetLib/src/gh.hh
@@ -7,7 +7,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.hh,v 1.1 2001/03/01 13:40:10 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.hh,v 1.2 2001/03/07 13:00:57 eschnett Exp $
 
  ***************************************************************************/
 
@@ -72,6 +72,7 @@ public:				// should be readonly
   centering mgcent;		// default (vertex or cell centered)
   
   ibbox baseextent;		// bounds (inclusive) of base level
+  vector<vector<ibbox> > bases; // [rl][ml]
   
   rexts extents;		// bounds of all grids
   rprocs processors;		// processor numbers of all grids