Changed the way output is handled. Now output is per refinement level

Changed the way output is handled.  Now output is per refinement level
only.  A single cctk_iteration is now a time step on the finest grid,
instead of a time step on the coarsest grid.  The analysis stage
should now work on the finer grid.

darcs-hash:20010316203212-f6438-36f1809e816f4b3ae73a1a837c0fcd8d81616120.gz

16 files changed, 699 insertions, 649 deletions

diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl
index 13eb6b396..f1da3ec03 100644
--- a/Carpet/Carpet/param.ccl
+++ b/Carpet/Carpet/param.ccl
@@ -1,5 +1,5 @@
 # Parameter definitions for thorn Carpet
-# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/param.ccl,v 1.3 2001/03/10 20:54:59 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/param.ccl,v 1.4 2001/03/16 21:32:12 eschnett Exp $
 
 shares: Cactus
 
@@ -73,6 +73,12 @@ CCTK_INT multigrid_factor "Multigrid factor"
 
 
 
+BOOLEAN early_timelevel_cycling "Cycle time levels before PRESTEP instead of before the POSTSTEP"
+{
+} "no"
+
+
+
 BOOLEAN verbose "Display info on the screen"
 {
 } "no"
diff --git a/Carpet/Carpet/src/carpet.cc b/Carpet/Carpet/src/carpet.cc
index 7d564e954..1fdf773db 100644
--- a/Carpet/Carpet/src/carpet.cc
+++ b/Carpet/Carpet/src/carpet.cc
@@ -1,8 +1,9 @@
-// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Attic/carpet.cc,v 1.12 2001/03/15 09:59:36 eschnett Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Attic/carpet.cc,v 1.13 2001/03/16 21:32:17 eschnett Exp $
 
-/* It is assumed that the number of components of all arrays is equal
-   to the number of components of the grid functions, and that their
-   distribution onto the processors is the same.  */
+// It is assumed that the number of components of all arrays is equal
+// to the number of components of the grid functions, and that their
+// distribution onto the processors is the same, and that all
+// processors own the same number of components.
 
 #include <algorithm>
 #include <cassert>
@@ -32,16 +33,12 @@
 
 #include "carpet.hh"
 
-static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Attic/carpet.cc,v 1.12 2001/03/15 09:59:36 eschnett Exp $";
+static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Attic/carpet.cc,v 1.13 2001/03/16 21:32:17 eschnett Exp $";
 
 
 
 namespace Carpet {
   
-  static void RecursiveInitialise (cGH* cgh);
-  static void RecursiveEvolve (cGH* cgh);
-  static void RecursiveShutdown (cGH* cgh);
-  
   static void Recompose (cGH* cgh);
   static void CycleTimeLevels (cGH* cgh);
   static void Restrict (cGH* cgh);
@@ -51,38 +48,48 @@ namespace Carpet {
   
   
   
-  // handle from CCTK_RegisterGHExtension
+  // Handle from CCTK_RegisterGHExtension
   int GHExtension;
   
-  // time step on base grid
-  CCTK_REAL base_delta_time;
+  // Maximum number of refinement levels
+  int maxreflevels;
   
-  // active time level
+  // Refinement factor on finest grid
+  int maxreflevelfact;
+  
+  // Active time level
   int activetimelevel;		// 0 for current, 1 for next
   
-  // current position on the grid hierarchy
+  // Current iteration per refinement level
+  vector<int> iteration;
+  
+  // Current position on the grid hierarchy
   int mglevel;
   int reflevel;
   int component;
   
-  // current refinement factor
-  int reflevelfactor;
+  // refinement factor of current level: pow(refinement_factor, reflevel)
+  int reflevelfact;
+  
+  // Time step on base grid
+  CCTK_REAL base_delta_time;
   
   
   
-  // data for scalars
-  vector<vector<vector<void*> > > scdata;// [group][var][ti]
+  // Data for scalars
+  vector<vector<vector<void*> > > scdata; // [group][var][ti]
   
-  // data for arrays
+  // Data for arrays
+  // TODO: have replicated arrays
   vector<arrdesc> arrdata;	// [group]
   
-  // the grid hierarchy
+  // The grid hierarchy
   gh<dim>* hh;
   th<dim>* tt;
   dh<dim>* dd;
   int gfsize[dim];
   
-  // data for grid functions
+  // Data for grid functions
   vector<gfdesc> gfdata;	// [group]
   
   
@@ -106,7 +113,7 @@ namespace Carpet {
     CCTK_OverloadBarrier (Barrier);
     CCTK_OverloadExit (Exit);
     CCTK_OverloadAbort (Abort);
-    CCTK_OverloadMyProc (myProc);
+    CCTK_OverloadMyProc (MyProc);
     CCTK_OverloadnProcs (nProcs);
     CCTK_OverloadArrayGroupSizeB (ArrayGroupSizeB);
     CCTK_OverloadQueryGroupStorageB (QueryGroupStorageB);
@@ -120,13 +127,17 @@ namespace Carpet {
   {
     DECLARE_CCTK_PARAMETERS;
     
-    // not sure what to do with that
+    // Not sure what to do with that
     assert (convLevel==0);
     
     dist::pseudoinit();
     Checkpoint ("starting SetupGH...");
     
-    // ghost zones
+    // Refinement information
+    maxreflevels = max_refinement_levels;
+    maxreflevelfact = (int)floor(pow(refinement_factor, maxreflevels-1) + 0.5);
+    
+    // Ghost zones
     vect<int,dim> lghosts, ughosts;
     if (ghost_size == -1) {
       lghosts = vect<int,dim>(ghost_size_x, ghost_size_y, ghost_size_z);
@@ -136,9 +147,8 @@ namespace Carpet {
       ughosts = vect<int,dim>(ghost_size, ghost_size, ghost_size);
     }
     
-    // grid size
-    const int stride
-      = (int)floor(pow(refinement_factor, max_refinement_levels-1) + 0.5);
+    // Grid size
+    const int stride = maxreflevelfact;
     vect<int,dim> npoints;
     if (global_nsize == -1) {
       npoints = vect<int,dim>(global_nx, global_ny, global_nz);
@@ -152,26 +162,26 @@ namespace Carpet {
     
     const bbox<int,dim> baseext(lb, ub, str);
     
-    // allocate grid hierarchy
+    // Allocate grid hierarchy
     hh = new gh<dim>(refinement_factor, vertex_centered,
 		     multigrid_factor, vertex_centered,
 		     baseext);
     
-    // allocate time hierarchy
-    tt = new th<dim>
-      (*hh, (int)floor(pow(refinement_factor, max_refinement_levels-1) + 0.5));
+    // Allocate time hierarchy
+    tt = new th<dim>(*hh, maxreflevelfact);
     
-    // allocate data hierarchy
+    // Allocate data hierarchy
     dd = new dh<dim>(*hh, lghosts, ughosts);
     
-    // allocate space for groups
+    // Allocate space for groups
     scdata.resize(CCTK_NumGroups());
     arrdata.resize(CCTK_NumGroups());
     gfdata.resize(CCTK_NumGroups());
     
-    // allocate space for variables in group
-    // (don't enable storage yet)
+    // Allocate space for variables in group (but don't enable storage
+    // yet)
     for (int group=0; group<CCTK_NumGroups(); ++group) {
+      
       switch (CCTK_GroupTypeI(group)) {
 	
       case CCTK_SCALAR: {
@@ -231,28 +241,38 @@ namespace Carpet {
       }
     }
     
-    // active time level
-    activetimelevel = 0;
+    // Initialise cgh
+    for (int d=0; d<dim; ++d) {
+      cgh->cctk_nghostzones[d] = dd->lghosts[d];
+    }
     
-    // current position
+    // Initialise current position
     reflevel  = 0;
     mglevel   = 0;
     component = -1;
     
-    // initialise cgh
-    cgh->cctk_convlevel = mglevel;
-    
     // Recompose grid hierarchy
     Recompose (cgh); 
     
-    // enace current position
-    enact_reflevel (cgh);
+    // Initialise time step on coarse grid
+    base_delta_time = 0;
+    
+    // Active time level
+    activetimelevel = 0;
+    
+    // Current iteration
+    iteration.resize(maxreflevels, 0);
+    
+    // Set current position (this time for real)
+    set_reflevel  (cgh, 0);
+    set_mglevel   (cgh, 0);
+    set_component (cgh, -1);
     
     Checkpoint ("done with SetupGH.");
     
     // We register only once, ergo we get only one handle, ergo there
     // is only one grid hierarchy for us.  We store that statically,
-    // so there is no need to pass it to Cactus.
+    // so there is no need to pass anything to Cactus.
     return 0;
   }
   
@@ -284,65 +304,54 @@ namespace Carpet {
     CCTK_ScheduleTraverse ("CCTK_PARAMCHECK", cgh, CallFunction);
     CCTKi_FinaliseParamWarn();
     
-    // Initialise all levels recursively
-    RecursiveInitialise (cgh);
+    BEGIN_REFLEVEL_LOOP(cgh) {
+      
+      // Set up the grid
+      CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
+      if (reflevel==0) {
+	base_delta_time = cgh->cctk_delta_time;
+      } else {
+// 	assert (abs(cgh->cctk_delta_time - base_delta_time / reflevelfactor)
+// 		< 1e-6 * base_delta_time);
+	// This fixes a bug in CactusBase/Time
+	cgh->cctk_delta_time = base_delta_time / reflevelfact;
+      }
+      
+      // Set up the initial data
+      CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction);
+      CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction);
+      
+      // Recover
+      CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction);
+      CCTK_ScheduleTraverse ("CCTK_CPINITIAL", cgh, CallFunction);
+      
+    } END_REFLEVEL_LOOP(cgh);
+    
+    BEGIN_REVERSE_REFLEVEL_LOOP(cgh) {
+      
+      // Restrict
+      Restrict (cgh);
+      
+    } END_REVERSE_REFLEVEL_LOOP(cgh);
     
-    // Output
-    CCTK_OutputGH (cgh);
+    BEGIN_REVERSE_REFLEVEL_LOOP(cgh) {
+      
+      // Poststep
+      CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
+      
+      // Analysis
+      CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
+      
+      // Output
+      CCTK_OutputGH (cgh);
+      
+    } END_REVERSE_REFLEVEL_LOOP(cgh);
     
     Checkpoint ("done with Initialise.");
     
     return 0;
   }
   
-  static void RecursiveInitialise (cGH* cgh)
-  {
-    DECLARE_CCTK_PARAMETERS;
-    
-    Checkpoint ("%*sstarting RecursiveInitialise on level %d...",
-		2*reflevel, "", reflevel);
-    
-    // Set up the grid
-    CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
-    if (reflevel==0) {
-      base_delta_time = cgh->cctk_delta_time;
-    } else {
-//       assert (abs(cgh->cctk_delta_time - base_delta_time / reflevelfactor)
-// 	      < 1e-6 * base_delta_time);
-      // This fixes a bug in CactusBase/Time
-      cgh->cctk_delta_time = base_delta_time / reflevelfactor;
-    }
-    
-    // Set up the initial data
-    CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction);
-    CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction);
-    
-    // Recover
-    CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction);
-    CCTK_ScheduleTraverse ("CCTK_CPINITIAL", cgh, CallFunction);
-    
-    // Recurse
-    if (reflevel < hh->reflevels()-1) {
-      ++reflevel;
-      enact_reflevel (cgh);
-      RecursiveInitialise (cgh);
-      --reflevel;
-      enact_reflevel (cgh);
-    }
-    
-    // Restrict
-    Restrict (cgh);
-    
-    // Poststep
-    CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-    
-    // Analysis
-    CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
-    
-    Checkpoint ("%*sdone with RecursiveInitialise on level %d...",
-		2*reflevel, "", reflevel);
-  }
-  
   
   
   int Evolve (tFleshConfig* fc)
@@ -364,103 +373,80 @@ namespace Carpet {
       
       Checkpoint ("Evolving iteration %d...", cgh->cctk_iteration);
       
-      RecursiveEvolve (cgh);
-      
-      // Output
-      CCTK_OutputGH (cgh);
-    }
-    
-    Checkpoint ("done with Evolve.");
-    
-    return 0;
-  }
-  
-  static void RecursiveEvolve (cGH* cgh)
-  {
-    DECLARE_CCTK_PARAMETERS;
-    
-    Checkpoint ("%*sstarting RecursiveEvolve on level %d...",
-		2*reflevel, "", reflevel);
-    
-    // Move activity to next time level
-    assert (activetimelevel == 0);
-    activetimelevel = 1;
-    
-    // Prestep
-    CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
-    
-    // Evolve
-    CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
-    
-    // Recurse
-    if (reflevel < hh->reflevels()-1) {
-      
-      const CCTK_REAL old_delta_time = cgh->cctk_delta_time;
-      const CCTK_REAL old_time = cgh->cctk_time;
-      const int oldactivetimelevel = activetimelevel;
-      
-      ++reflevel;
-      enact_reflevel (cgh);
-      activetimelevel = 0;
-      
-      for (int i=0; i<hh->reffact; ++i) {
-	RecursiveEvolve (cgh);
-	assert (abs(cgh->cctk_time - (old_time + (i+1) * cgh->cctk_delta_time))
-		< 1e-6 * old_delta_time);
-      }
-      
-      --reflevel;
-      enact_reflevel (cgh);
-      activetimelevel = oldactivetimelevel;
+      BEGIN_REFLEVEL_LOOP(cgh) {
+	if ((cgh->cctk_iteration-1) % (maxreflevelfact/reflevelfact) == 0) {
+	  
+	  // Prestep
+	  CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
+	  
+	  // Move activity to next time level
+	  assert (activetimelevel == 0);
+	  ++activetimelevel;
+	  
+	  // Evolve
+	  CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
+	  
+	  // Move activity back to current time level
+	  --activetimelevel;
+	  assert (activetimelevel == 0);
+	  
+	}
+      } END_REFLEVEL_LOOP(cgh);
       
-      assert (abs(cgh->cctk_delta_time - old_delta_time)
-	      < 1e-6 * old_delta_time);
-      assert (abs(cgh->cctk_time - (old_time + old_delta_time))
-	      < 1e-6 * old_delta_time);
+      // Advance time
+      cgh->cctk_time += base_delta_time / maxreflevelfact;
       
-    } else {
+      BEGIN_REVERSE_REFLEVEL_LOOP(cgh) {
+	if (cgh->cctk_iteration % (maxreflevelfact/reflevelfact) == 0) {
+	  
+	  // Cycle time levels
+	  CycleTimeLevels (cgh);
+	  
+	  // Advance level times
+	  tt->advance_time (reflevel, mglevel);
+	  for (int group=0; group<CCTK_NumGroups(); ++group) {
+	    switch (CCTK_GroupTypeI(group)) {
+	    case CCTK_SCALAR:
+	      break;
+	    case CCTK_ARRAY:
+	      arrdata[group].tt->advance_time (reflevel, mglevel);
+	      break;
+	    case CCTK_GF:
+	      break;
+	    default:
+	      abort();
+	    }
+	  }
+	  
+	  // Restrict
+	  Restrict (cgh);
+	  
+	}
+      } END_REVERSE_REFLEVEL_LOOP(cgh);
       
-      cgh->cctk_time += cgh->cctk_delta_time;
+      BEGIN_REVERSE_REFLEVEL_LOOP(cgh) {
+	if (cgh->cctk_iteration % (maxreflevelfact/reflevelfact) == 0) {
+	  
+	  // Poststep
+	  CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
+	  
+	  // Checkpoint
+	  CCTK_ScheduleTraverse ("CCTK_CHECKPOINT", cgh, CallFunction);
+	  
+	  // Analysis
+	  CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
+	  
+	  // Output
+	  CCTK_OutputGH (cgh);
+	  
+	}
+      } END_REVERSE_REFLEVEL_LOOP(cgh);
       
-    }
-    
-    // Advance level times
-    tt->advance_time (reflevel, mglevel);
-    for (int group=0; group<CCTK_NumGroups(); ++group) {
-      switch (CCTK_GroupTypeI(group)) {
-      case CCTK_SCALAR:
-	break;
-      case CCTK_ARRAY:
-	arrdata[group].tt->advance_time (reflevel, mglevel);
-	break;
-      case CCTK_GF:
-	break;
-      default:
-	abort();
-      }
-    }
-    
-    // Cycle time levels
-    CycleTimeLevels (cgh);
+    } // main loop
     
-    // Move activity back to current time level
-    assert (activetimelevel == 1);
-    activetimelevel = 0;
-    
-    // Restrict
-    Restrict (cgh);
-    
-    // Poststep
-    CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-    
-    // Checkpoint
-    CCTK_ScheduleTraverse ("CCTK_CHECKPOINT", cgh, CallFunction);
-    
-    // Analysis
-    CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
+    Checkpoint ("done with Evolve.");
     
-    Checkpoint ("%*sdone with RecursiveEvolve on level %d...",
-		2*reflevel, "", reflevel);
+    return 0;
   }
   
   
@@ -474,7 +460,15 @@ namespace Carpet {
     const int convlev = 0;
     cGH* cgh = fc->GH[convlev];
     
-    RecursiveShutdown (cgh);
+    // Terminate
+    BEGIN_REFLEVEL_LOOP(cgh) {
+      CCTK_ScheduleTraverse ("CCTK_TERMINATE", cgh, CallFunction);
+    } END_REFLEVEL_LOOP(cgh);
+    
+    // Shutdown
+    BEGIN_REFLEVEL_LOOP(cgh) {
+      CCTK_ScheduleTraverse ("CCTK_SHUTDOWN", cgh, CallFunction);
+    } END_REFLEVEL_LOOP(cgh);
     
     CCTK_PRINTSEPARATOR;
     printf ("Done.\n");
@@ -487,61 +481,11 @@ namespace Carpet {
     return 0;
   }
   
-  static void RecursiveShutdown (cGH* cgh)
-  {
-    DECLARE_CCTK_PARAMETERS;
-    
-    Checkpoint ("%*sstarting RecursiveShutdown on level %d...",
-		2*reflevel, "", reflevel);
-    
-    // Recurse
-    if (reflevel < hh->reflevels()-1) {
-      ++reflevel;
-      enact_reflevel (cgh);
-      RecursiveShutdown (cgh);
-      --reflevel;
-      enact_reflevel (cgh);
-    }
-    
-    // Terminate
-    CCTK_ScheduleTraverse ("CCTK_TERMINATE", cgh, CallFunction);
-    
-    // Shutdown
-    CCTK_ScheduleTraverse ("CCTK_SHUTDOWN", cgh, CallFunction);
-    
-    Checkpoint ("%*sdone with RecursiveShutdown on level %d...",
-		2*reflevel, "", reflevel);
-  }
-  
-  
-  
-#if 0
-  int ScheduleTraverse (cGH* cgh, const char* rfrPoint)
-  {
-    // traverse all functions on all refinement levels on all
-    // multigrid levels
-    
-    abort();
-    
-    assert (mglevel==-1);
-    for (mglevel=0; mglevel<hh->mglevels(0,0); ++mglevel) {
-      assert (reflevel==-1);
-      for (reflevel=0; reflevel<hh->reflevels(); ++reflevel) {
-	CCTK_ScheduleTraverse (rfrPoint, cgh, CallFunction);
-      }
-      reflevel = -1;
-    }
-    mglevel = -1;
-    
-    return 0;
-  }
-#endif
-  
   
   
   int CallFunction (void* function, cFunctionData* attribute, void* data)
   {
-    // traverse one function on all components of one refinement level
+    // Traverse one function on all components of one refinement level
     // of one multigrid level
     
     assert (mglevel>=0);
@@ -551,227 +495,28 @@ namespace Carpet {
     
     cGH* cgh = (cGH*)data;
     
-    // set Cactus parameters
-    for (int d=0; d<dim; ++d) {
-      cgh->cctk_nghostzones[d] = dd->lghosts[d];
-    }
-    
     if (attribute->global) {
-      // global operation: call once per refinement level
-      
-      assert (component==-1);
-      
-      // set Cactus parameters to pseudo values
-      for (int d=0; d<dim; ++d) {
-	cgh->cctk_lsh[d]      = 0xdeadbeef;
-// 	cgh->cctk_gsh[d]      = 0xdeadbeef;
-	cgh->cctk_bbox[2*d  ] = 0xdeadbeef;
-	cgh->cctk_bbox[2*d+1] = 0xdeadbeef;
-	cgh->cctk_lbnd[d]     = 0xdeadbeef;
-	cgh->cctk_ubnd[d]     = 0xdeadbeef;
-	for (int stg=0; stg<CCTK_NSTAGGER; ++stg) {
-	  cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = 0xdeadbeef;
-	}
-      }
-      
-      // set local grid function and array sizes
-      for (int d=0; d<dim; ++d) {
-	gfsize[d] = 0xdeadbeef;
-      }
-      for (int group=0; group<CCTK_NumGroups(); ++group) {
-	const int n0 = CCTK_FirstVarIndexI(group);
-	switch (CCTK_GroupTypeFromVarI(n0)) {
-	case CCTK_SCALAR:
-	  break;
-	case CCTK_ARRAY:
-	  for (int d=0; d<dim; ++d) {
-	    arrdata[group].size[d] = 0xdeadbeef;
-	  }
-	  break;
-	case CCTK_GF:
-	  break;
-	default:
-	  abort();
-	}
-      }
-      
-      // set Cactus pointers to data
-      for (int n=0; n<CCTK_NumVars(); ++n) {
-	for (int ti=0; ti<CCTK_NumTimeLevelsFromVarI(n); ++ti) {
-	  
-	  const int group = CCTK_GroupIndexFromVarI(n);
-	  assert (group>=0);
-	  const int var   = n - CCTK_FirstVarIndexI(group);
-	  assert (var>=0);
-	  
-	  if (CCTK_QueryGroupStorageI(cgh, group)) {
-	    // group has storage
-	    
-	    switch (CCTK_GroupTypeFromVarI(n)) {
-	      
-	    case CCTK_SCALAR:
-	      // scalar variables can be accessed
-	      assert (group<(int)scdata.size());
-	      assert (var<(int)scdata[group].size());
-	      assert (ti<(int)scdata[group][var].size());
-	      cgh->data[n][ti] = scdata[group][var][ti];
-	      break;
-	      
-	    case CCTK_ARRAY:
-	    case CCTK_GF:
-	      // arrays and grid functions cannot be accessed
-	      cgh->data[n][ti] = 0;
-	      break;
-	      
-	    default:
-	      abort();
-	    }
-	    
-	  } else {
-	    
-	    // group has no storage
-	    cgh->data[n][ti] = 0;
-	    
-	  }
-	  
-	} // for ti
-      }	// for n
+      // Global operation: call once per refinement level
       
-      // traverse
       const int res = CCTK_CallFunction (function, attribute, data);
       assert (res==0);
       
     } else {
-      // local operation: call once per component
+      // Local operation: call once per component
       
-      assert (component==-1);
-      for (component=0; component<hh->components(reflevel); ++component) {
-	
-	// this requires that all processors have the same number of
+      BEGIN_COMPONENT_LOOP(cgh) {
+	// This requires that all processors have the same number of
 	// local components
  	if (hh->is_local(reflevel, component)) {
 	  
-	  // set Cactus parameters
-	  for (int d=0; d<dim; ++d) {
-	    const bbox<int,dim>& ext
-	      = dd->boxes[reflevel][component][mglevel].exterior;
-	    const bbox<int,dim>& base = hh->baseextent;
-	    cgh->cctk_lsh[d] = (ext.shape() / ext.stride())[d];
-// 	    cgh->cctk_gsh[d]
-// 	      = ((base.shape() / base.stride() + dd->lghosts + dd->ughosts)[d]
-// 		 * cgh->cctk_levfac[d]);
-	    cgh->cctk_lbnd[d] = (ext.lower() / ext.stride())[d];
-	    cgh->cctk_ubnd[d] = (ext.upper() / ext.stride())[d];
-	    // no outer boundaries on the finer grids
-	    cgh->cctk_bbox[2*d  ]
-	      = reflevel==0 && cgh->cctk_lbnd[d] == 0;
-	    cgh->cctk_bbox[2*d+1]
-	      = reflevel==0 && cgh->cctk_ubnd[d] == cgh->cctk_gsh[d]-1;
-#if 0
-	    // do allow outer boundaries on the finer grids
-	    // (but this is generally inconsistent -- c. f. periodicity)
- 	    cgh->cctk_bbox[2*d  ] = (ext.lower() < base.lower())[d];
- 	    cgh->cctk_bbox[2*d+1] = (ext.upper() > base.upper())[d];
-#endif
-	    for (int stg=0; stg<CCTK_NSTAGGER; ++stg) {
-	      // TODO: support staggering
-	      cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = cgh->cctk_lsh[d];
-	    }
-	  }
-	  
-	  // set local grid function and array sizes
-	  const bbox<int,dim> ext =
-	    dd->boxes[reflevel][component][mglevel].exterior;
-	  for (int d=0; d<dim; ++d) {
-	    gfsize[d] = ext.shape()[d] / ext.stride()[d];
-	  }
-	  
-	  for (int group=0; group<CCTK_NumGroups(); ++group) {
-	    const int n0 = CCTK_FirstVarIndexI(group);
-	    switch (CCTK_GroupTypeFromVarI(n0)) {
-	    case CCTK_SCALAR:
-	      break;
-	    case CCTK_ARRAY: {
-	      const bbox<int,dim> ext =
-		arrdata[group].dd->
-		boxes[reflevel][component][mglevel].exterior;
-	      for (int d=0; d<dim; ++d) {
-		arrdata[group].size[d] = ext.shape()[d] / ext.stride()[d];
-	      }
-	      break;
-	    }
-	    case CCTK_GF:
-	      break;
-	    default:
-	      abort();
-	    }
-	  }
-	  
-	  // set Cactus pointers to data
-	  for (int n=0; n<CCTK_NumVars(); ++n) {
-	    for (int ti=0; ti<CCTK_NumTimeLevelsFromVarI(n); ++ti) {
-	      
-	      const int group = CCTK_GroupIndexFromVarI(n);
-	      assert (group>=0);
-	      const int var   = n - CCTK_FirstVarIndexI(group);
-	      assert (var>=0);
-	      const int tmin  = min(0, 2 - CCTK_NumTimeLevelsFromVarI(n));
-	      const int tl    = tmin + ti;
-	      
-	      if (CCTK_QueryGroupStorageI(cgh, group)) {
-		// group has storage
-		
-		switch (CCTK_GroupTypeFromVarI(n)) {
-		  
-		case CCTK_SCALAR:
-		  assert (group<(int)scdata.size());
-		  assert (var<(int)scdata[group].size());
-		  assert (ti<(int)scdata[group][var].size());
-		  cgh->data[n][ti] = scdata[group][var][ti];
-		  break;
-		  
-		case CCTK_ARRAY:
-		  assert (group<(int)arrdata.size());
-		  assert (var<(int)arrdata[group].data.size());
-		  cgh->data[n][ti]
-		    = ((*arrdata[group].data[var])
-		       (tl, reflevel, component, mglevel)
-		       ->storage());
-		  break;
-		  
-		case CCTK_GF:
-		  assert (group<(int)gfdata.size());
-		  assert (var<(int)gfdata[group].data.size());
-		  cgh->data[n][ti]
-		    = ((*gfdata[group].data[var])
-		       (tl, reflevel, component, mglevel)
-		       ->storage());
-		  break;
-		  
-		default:
-		  abort();
-		}
-		assert (cgh->data[n][ti]);
-		
-	      } else {
-		
-		// group has no storage
-		cgh->data[n][ti] = 0;
-		
-	      }	// if ! has storage
-	      
-	    } // for ti
-	  } // for n
-	  
 	  const int res = CCTK_CallFunction (function, attribute, data);
 	  assert (res==0);
 	  
 	} // if is_local
 	
-      }	// for component
-      component = -1;
+      }	END_COMPONENT_LOOP(cgh);
       
-    } // if local operation
+    }
     
 //     Checkpoint ("%*sdone with CallFunction.", 2*reflevel, "");
     
@@ -785,7 +530,7 @@ namespace Carpet {
   {
     assert (component == -1);
     
-    Checkpoint ("SyncGroup %s", groupname);
+    Checkpoint ("%*sSyncGroup %s", 2*reflevel, "", groupname);
     
     const int group = CCTK_GroupIndex(groupname);
     assert (group>=0 && group<CCTK_NumGroups());
@@ -847,7 +592,15 @@ namespace Carpet {
   
   int EnableGroupStorage (cGH* cgh, const char* groupname)
   {
-    Checkpoint ("EnableGroupStorage %s", groupname);
+    Checkpoint ("%*sEnableGroupStorage %s", 2*reflevel, "", groupname);
+    
+    // TODO: Enabling storage for one refinement level has to enable
+    // it for all other refinement levels as well.  Disabling must
+    // wait until all refinement levels have been disabled.
+    
+    // TODO: Invent a mode "reflevel==-1" that is global, i. e. has
+    // effect for all refinement levels.  This mode is used during
+    // INITIAL, and en-/disabling storage in it is also global.
     
     const int group = CCTK_GroupIndex(groupname);
     assert (group>=0 && group<CCTK_NumGroups());
@@ -950,7 +703,7 @@ namespace Carpet {
   
   int DisableGroupStorage (cGH* cgh, const char* groupname)
   {
-    Checkpoint ("DisableGroupStorage %s", groupname);
+    Checkpoint ("%*sDisableGroupStorage %s", 2*reflevel, "", groupname);
     
     const int group = CCTK_GroupIndex(groupname);
     assert (group>=0 && group<CCTK_NumGroups());
@@ -1113,6 +866,7 @@ namespace Carpet {
       }
     }
     hh->recompose(bbsss, pss);
+    // TODO: recompose arrays too
   }
   
   
@@ -1131,6 +885,7 @@ namespace Carpet {
 	    assert (var<(int)scdata[group].size());
 	    const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n));
 	    for (int ti=0; ti<CCTK_NumTimeLevelsFromVarI(n)-1; ++ti) {
+	      // TODO: Which refinement level to use?
 	      memcpy (scdata[group][var][ti], scdata[group][var][ti+1], sz);
 	    }
 	    break;
@@ -1247,7 +1002,7 @@ namespace Carpet {
   
   
   
-  int myProc (cGH* cgh)
+  int MyProc (cGH* cgh)
   {
     int rank;
     MPI_Comm_rank (dist::comm, &rank);
@@ -1371,22 +1126,237 @@ namespace Carpet {
   
   
   
-  void enact_reflevel (cGH* cgh)
+  void set_reflevel (cGH* cgh, const int rl)
   {
     // Check
-    assert (reflevel>=0 && reflevel<hh->reflevels());
-    assert (mglevel == 0);
+    assert (rl>=0 && rl<hh->reflevels());
     assert (component == -1);
     
     // Change
+    reflevel = rl;
     const bbox<int,dim>& base = hh->baseextent;
-    reflevelfactor = (int)floor(pow(hh->reffact, reflevel)+0.5);
-    cgh->cctk_delta_time = base_delta_time / reflevelfactor;
+    reflevelfact = (int)floor(pow(hh->reffact, reflevel)+0.5);
+    cgh->cctk_delta_time = base_delta_time / reflevelfact;
     for (int d=0; d<dim; ++d) {
       cgh->cctk_gsh[d]
-	= ((base.shape() / base.stride() + dd->lghosts + dd->ughosts)[d]
-	   * reflevelfactor);
-      cgh->cctk_levfac[d] = reflevelfactor;
+	= ((base.shape() / base.stride()
+	    + dd->lghosts + dd->ughosts)[d] - 1) * reflevelfact + 1;
+      cgh->cctk_levfac[d] = reflevelfact;
+    }
+  }
+  
+  
+  
+  void set_mglevel (cGH* cgh, const int ml)
+  {
+    assert (ml==0);
+    assert (component==-1);
+    mglevel = ml;
+    cgh->cctk_convlevel = mglevel;
+  }
+  
+  
+  
+  void set_component (cGH* cgh, const int c)
+  {
+    assert (c==-1 || (c>=0 && c<hh->components(reflevel)));
+    component = c;
+    
+    if (component == -1) {
+      // Global mode -- no component is active
+      
+      // Set Cactus parameters to pseudo values
+      for (int d=0; d<dim; ++d) {
+	cgh->cctk_lsh[d]      = 0xdeadbeef;
+	cgh->cctk_bbox[2*d  ] = 0xdeadbeef;
+	cgh->cctk_bbox[2*d+1] = 0xdeadbeef;
+	cgh->cctk_lbnd[d]     = 0xdeadbeef;
+	cgh->cctk_ubnd[d]     = 0xdeadbeef;
+	for (int stg=0; stg<CCTK_NSTAGGER; ++stg) {
+	  cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = 0xdeadbeef;
+	}
+      }
+      
+      // Set local grid function and array sizes
+      for (int d=0; d<dim; ++d) {
+	gfsize[d] = 0xdeadbeef;
+      }
+      for (int group=0; group<CCTK_NumGroups(); ++group) {
+	const int n0 = CCTK_FirstVarIndexI(group);
+	switch (CCTK_GroupTypeFromVarI(n0)) {
+	case CCTK_SCALAR:
+	  break;
+	case CCTK_ARRAY:
+	  for (int d=0; d<dim; ++d) {
+	    arrdata[group].size[d] = 0xdeadbeef;
+	  }
+	  break;
+	case CCTK_GF:
+	  break;
+	default:
+	  abort();
+	}
+      }
+      
+      // Set Cactus pointers to data
+      for (int n=0; n<CCTK_NumVars(); ++n) {
+	for (int ti=0; ti<CCTK_NumTimeLevelsFromVarI(n); ++ti) {
+	  
+	  const int group = CCTK_GroupIndexFromVarI(n);
+	  assert (group>=0);
+	  const int var   = n - CCTK_FirstVarIndexI(group);
+	  assert (var>=0);
+	  
+	  if (CCTK_QueryGroupStorageI(cgh, group)) {
+	    // Group has storage
+	    
+	    switch (CCTK_GroupTypeFromVarI(n)) {
+	      
+	    case CCTK_SCALAR:
+	      // Scalar variables can be accessed
+	      assert (group<(int)scdata.size());
+	      assert (var<(int)scdata[group].size());
+	      assert (ti<(int)scdata[group][var].size());
+	      cgh->data[n][ti] = scdata[group][var][ti];
+	      break;
+	      
+	    case CCTK_ARRAY:
+	    case CCTK_GF:
+	      // Arrays and grid functions cannot be accessed
+	      cgh->data[n][ti] = 0;
+	      break;
+	      
+	    default:
+	      abort();
+	    }
+	    
+	  } else {
+	    // Group has no storage
+	    
+	    cgh->data[n][ti] = 0;
+	    
+	  }
+	  
+	} // for ti
+      }	// for n
+      
+    } else {
+      // Local mode -- a component is active
+      
+      // Set Cactus parameters
+      for (int d=0; d<dim; ++d) {
+	const bbox<int,dim>& ext
+	  = dd->boxes[reflevel][component][mglevel].exterior;
+	cgh->cctk_lsh[d] = (ext.shape() / ext.stride())[d];
+	cgh->cctk_lbnd[d] = (ext.lower() / ext.stride())[d];
+	cgh->cctk_ubnd[d] = (ext.upper() / ext.stride())[d];
+	assert (cgh->cctk_lsh[d]>=0 && cgh->cctk_lsh[d]<=cgh->cctk_gsh[d]);
+	assert (cgh->cctk_lbnd[d]>=0 && cgh->cctk_ubnd[d]<cgh->cctk_gsh[d]);
+	assert (cgh->cctk_lbnd[d]<=cgh->cctk_ubnd[d]+1);
+	// No outer boundaries on the finer grids
+	cgh->cctk_bbox[2*d  ]
+	  = reflevel==0 && cgh->cctk_lbnd[d] == 0;
+	cgh->cctk_bbox[2*d+1]
+	  = reflevel==0 && cgh->cctk_ubnd[d] == cgh->cctk_gsh[d]-1;
+#if 0
+	// Do allow outer boundaries on the finer grids (but this is
+	// generally inconsistent -- c. f. periodicity)
+	const bbox<int,dim>& base = hh->baseextent;
+	cgh->cctk_bbox[2*d  ] = (ext.lower() < base.lower())[d];
+	cgh->cctk_bbox[2*d+1] = (ext.upper() > base.upper())[d];
+#endif
+	for (int stg=0; stg<CCTK_NSTAGGER; ++stg) {
+	  // TODO: support staggering
+	  cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = cgh->cctk_lsh[d];
+	}
+      }
+      
+      // Set local grid function and array sizes
+      const bbox<int,dim> ext =
+	dd->boxes[reflevel][component][mglevel].exterior;
+      for (int d=0; d<dim; ++d) {
+	gfsize[d] = ext.shape()[d] / ext.stride()[d];
+      }
+      
+      for (int group=0; group<CCTK_NumGroups(); ++group) {
+	const int n0 = CCTK_FirstVarIndexI(group);
+	switch (CCTK_GroupTypeFromVarI(n0)) {
+	case CCTK_SCALAR:
+	  break;
+	case CCTK_ARRAY: {
+	  const bbox<int,dim> ext =
+	    arrdata[group].dd->
+	    boxes[reflevel][component][mglevel].exterior;
+	  for (int d=0; d<dim; ++d) {
+	    arrdata[group].size[d] = ext.shape()[d] / ext.stride()[d];
+	  }
+	  break;
+	}
+	case CCTK_GF:
+	  break;
+	default:
+	  abort();
+	}
+      }
+      
+      // Set Cactus pointers to data
+      for (int n=0; n<CCTK_NumVars(); ++n) {
+	for (int ti=0; ti<CCTK_NumTimeLevelsFromVarI(n); ++ti) {
+	  
+	  const int group = CCTK_GroupIndexFromVarI(n);
+	  assert (group>=0);
+	  const int var   = n - CCTK_FirstVarIndexI(group);
+	  assert (var>=0);
+	  const int tmin  = min(0, 2 - CCTK_NumTimeLevelsFromVarI(n));
+	  const int tl    = tmin + ti;
+	  
+	  if (CCTK_QueryGroupStorageI(cgh, group)) {
+	    // Group has storage
+	    
+	    switch (CCTK_GroupTypeFromVarI(n)) {
+	      
+	    case CCTK_SCALAR:
+	      assert (group<(int)scdata.size());
+	      assert (var<(int)scdata[group].size());
+	      assert (ti<(int)scdata[group][var].size());
+	      cgh->data[n][ti] = scdata[group][var][ti];
+	      break;
+	      
+	    case CCTK_ARRAY:
+	      assert (group<(int)arrdata.size());
+	      assert (var<(int)arrdata[group].data.size());
+	      cgh->data[n][ti]
+		= ((*arrdata[group].data[var])
+		   (tl, reflevel, component, mglevel)->storage());
+	      break;
+	      
+	    case CCTK_GF:
+	      assert (group<(int)gfdata.size());
+	      assert (var<(int)gfdata[group].data.size());
+	      cgh->data[n][ti]
+		= ((*gfdata[group].data[var])
+		   (tl, reflevel, component, mglevel)->storage());
+	      break;
+	      
+	    default:
+	      abort();
+	    }
+	    if (hh->is_local(reflevel,component)) {
+	      assert (cgh->data[n][ti]);
+	    } else {
+	      assert (! cgh->data[n][ti]);
+	    }
+	    
+	  } else {
+	    // Group has no storage
+	    
+	    cgh->data[n][ti] = 0;
+	    
+	  } // if ! has storage
+	  
+	} // for ti
+      } // for n
+      
     }
   }
   
diff --git a/Carpet/Carpet/src/carpet.hh b/Carpet/Carpet/src/carpet.hh
index df114d39f..c2afaa994 100644
--- a/Carpet/Carpet/src/carpet.hh
+++ b/Carpet/Carpet/src/carpet.hh
@@ -1,4 +1,4 @@
-// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet.hh,v 1.4 2001/03/12 16:54:18 eschnett Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet.hh,v 1.5 2001/03/16 21:32:17 eschnett Exp $
 
 #include <vector>
 
@@ -18,29 +18,38 @@ namespace Carpet {
   
   
   
-  // handle from CCTK_RegisterGHExtension
+  // Handle from CCTK_RegisterGHExtension
   extern int GHExtension;
   
-  // time step on base grid
-  extern CCTK_REAL base_delta_time;
+  // Maximum number of refinement levels
+  extern int maxreflevels;
+  
+  // Refinement factor on finest grid
+  extern int maxreflevelfact;
   
-  // active time level
+  // Active time level
   extern int activetimelevel;	// 0 for current, 1 for next
   
-  // current position on the grid hierarchy
-  extern int mglevel;
+  // Current iteration per refinement level
+  extern vector<int> iteration;
+  
+  // Current position on the grid hierarchy
   extern int reflevel;
+  extern int mglevel;
   extern int component;
   
-  // current refinement factor
-  extern int reflevelfactor;
+  // Current refinement factor
+  extern int reflevelfact;
+  
+  // Time step on base grid
+  extern CCTK_REAL base_delta_time;
   
   
   
-  // data for scalars
-  extern vector<vector<vector<void*> > > scdata;// [group][var][tl]
+  // Data for scalars
+  extern vector<vector<vector<void*> > > scdata; // [group][var][tl]
   
-  // data for arrays
+  // Data for arrays
   struct arrdesc {
     gh<dim>* hh;
     th<dim>* tt;
@@ -50,9 +59,9 @@ namespace Carpet {
   };
   extern vector<arrdesc> arrdata; // [group]
   
-  // data for grid functions
+  // Data for grid functions
   
-  // the grid hierarchy
+  // The grid hierarchy
   extern gh<dim>* hh;
   extern th<dim>* tt;
   extern dh<dim>* dd;
@@ -65,37 +74,102 @@ namespace Carpet {
   
   
   
-  // scheduled functions
+  // Scheduled functions
   extern "C" {
     int CarpetStartup();
   }
   
-  // registered functions
-  void* SetupGH (tFleshConfig *fc, int convLevel, cGH *cgh);
+  // Registered functions
+  void* SetupGH (tFleshConfig* fc, int convLevel, cGH* cgh);
   
-  int Initialise (tFleshConfig *config);
-  int Evolve (tFleshConfig *config);
-  int Shutdown (tFleshConfig *config);
-  int CallFunction (void *function, cFunctionData *attribute, void *data);
+  int Initialise (tFleshConfig* config);
+  int Evolve (tFleshConfig* config);
+  int Shutdown (tFleshConfig* config);
+  int CallFunction (void* function, cFunctionData* attribute, void* data);
   
-  int SyncGroup (cGH *cgh, const char *groupname);
-  int EnableGroupStorage (cGH *cgh, const char *groupname);
-  int DisableGroupStorage (cGH *cgh, const char *groupname); 
-  int EnableGroupComm (cGH *cgh, const char *groupname);
-  int DisableGroupComm (cGH *cgh, const char *groupname);
-  int Barrier (cGH *cgh);
-  int Exit (cGH *cgh, int retval);
-  int Abort (cGH *cgh, int retval);
-  int myProc (cGH *cgh);
-  int nProcs (cGH *cgh);
-  const int* ArrayGroupSizeB (cGH *cgh, int dir, int group,
-			      const char *groupname);
-  int QueryGroupStorageB (cGH *cgh, int group, const char *groupname);
+  int SyncGroup (cGH* cgh, const char* groupname);
+  int EnableGroupStorage (cGH* cgh, const char* groupname);
+  int DisableGroupStorage (cGH* cgh, const char* groupname); 
+  int EnableGroupComm (cGH* cgh, const char* groupname);
+  int DisableGroupComm (cGH* cgh, const char* groupname);
+  int Barrier (cGH* cgh);
+  int Exit (cGH* cgh, int retval);
+  int Abort (cGH* cgh, int retval);
+  int MyProc (cGH* cgh);
+  int nProcs (cGH* cgh);
+  const int* ArrayGroupSizeB (cGH* cgh, int dir, int group,
+			      const char* groupname);
+  int QueryGroupStorageB (cGH* cgh, int group, const char* groupname);
   
   
   
   // Helper functions
-  void enact_reflevel (cGH *cgh);
+  void set_reflevel (cGH* cgh, const int rl);
+  void set_mglevel (cGH* cgh, const int ml);
+  void set_component (cGH* cgh, const int c);
+  
+  
+  
+  // Refinement level iterator
+  
+#define BEGIN_REFLEVEL_LOOP(cgh)		\
+  do {						\
+    int _rl;					\
+    assert (reflevel==0);			\
+    for (;;) {					\
+      {
+#define END_REFLEVEL_LOOP(cgh)			\
+      }						\
+      if (reflevel==maxreflevels-1) break;	\
+      set_reflevel ((cgh), reflevel+1);		\
+    }						\
+    set_reflevel ((cgh), 0);			\
+    assert (reflevel==0);			\
+    _rl = 0;					\
+  } while (0)
+  
+  
+  
+  // Reverse refinement level iterator
+  
+#define BEGIN_REVERSE_REFLEVEL_LOOP(cgh)	\
+  do {						\
+    int _rrl;					\
+    assert (reflevel==0);			\
+    set_reflevel ((cgh), maxreflevels-1);	\
+    for (;;) {					\
+      {
+#define END_REVERSE_REFLEVEL_LOOP(cgh)		\
+      }						\
+      if (reflevel==0) break;			\
+      set_reflevel ((cgh), reflevel-1);		\
+    }						\
+    assert (reflevel==0);			\
+    _rrl = 0;					\
+  } while (0)
+  
+  
+  
+  // Component iterator
+  
+#define BEGIN_COMPONENT_LOOP(cgh)		\
+  do {						\
+    int _cl;					\
+    assert (component==-1);			\
+    set_component ((cgh), 0);			\
+    for (;;) {					\
+      {
+#define END_COMPONENT_LOOP(cgh)				\
+      }							\
+      if (component==hh->components(reflevel)-1) break;	\
+      set_component ((cgh), component+1);		\
+    }							\
+    set_component ((cgh), -1);				\
+    assert (component==-1);				\
+    _cl = 0;						\
+  } while (0)
+  
+  
   
   extern "C" {
     MPI_Comm CarpetMPICommunicator();
diff --git a/Carpet/CarpetIOASCII/src/ioascii.cc b/Carpet/CarpetIOASCII/src/ioascii.cc
index 9226656f5..6df502412 100644
--- a/Carpet/CarpetIOASCII/src/ioascii.cc
+++ b/Carpet/CarpetIOASCII/src/ioascii.cc
@@ -1,4 +1,5 @@
 #include <cassert>
+#include <climits>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -19,7 +20,7 @@
 
 #include "ioascii.hh"
 
-static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.cc,v 1.7 2001/03/15 23:28:33 eschnett Exp $";
+static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.cc,v 1.8 2001/03/16 21:32:17 eschnett Exp $";
 
 
 
@@ -48,7 +49,7 @@ int CarpetIOASCIIStartup()
 template<int outdim> int CarpetIOASCII<outdim>::GHExtension;
 template<int outdim> int CarpetIOASCII<outdim>::IOMethod;
 template<int outdim> vector<bool> CarpetIOASCII<outdim>::do_truncate;
-template<int outdim> vector<int> CarpetIOASCII<outdim>::last_output;
+template<int outdim> vector<vector<int> > CarpetIOASCII<outdim>::last_output;
 
 
 
@@ -78,8 +79,8 @@ int CarpetIOASCII<outdim>::Startup()
 
 
 template<int outdim>
-void* CarpetIOASCII<outdim>::SetupGH (tFleshConfig* const fc,
-				const int convLevel, cGH* const cgh)
+void* CarpetIOASCII<outdim>
+::SetupGH (tFleshConfig* const fc, const int convLevel, cGH* const cgh)
 {
   DECLARE_CCTK_PARAMETERS;
   
@@ -87,17 +88,22 @@ void* CarpetIOASCII<outdim>::SetupGH (tFleshConfig* const fc,
   do_truncate.resize(CCTK_NumVars(), ! IOUtil_RestartFromRecovery(cgh));
   
   // No iterations have yet been output
-  last_output.resize(CCTK_NumVars(), -1);
+  last_output.resize(maxreflevels);
+  for (int rl=0; rl<maxreflevels; ++rl) {
+    last_output[rl].resize(CCTK_NumVars(), INT_MIN);
+  }
   
   // We register only once, ergo we get only one handle.  We store
-  // that statically, so there is no need to pass it to Cactus.
+  // that statically, so there is no need to pass anything to Cactus.
   return 0;
 }
 
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::OutputGH (cGH* const cgh) {
+int CarpetIOASCII<outdim>
+::OutputGH (cGH* const cgh)
+{
   for (int vindex=0; vindex<CCTK_NumVars(); ++vindex) {
     if (TimeToOutput(cgh, vindex)) {
       TriggerOutput(cgh, vindex);
@@ -109,8 +115,10 @@ int CarpetIOASCII<outdim>::OutputGH (cGH* const cgh) {
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::OutputVarAs (cGH* const cgh, const char* const varname,
-				  const char* const alias) {
+int CarpetIOASCII<outdim>
+::OutputVarAs (cGH* const cgh,
+	       const char* const varname, const char* const alias)
+{
   DECLARE_CCTK_PARAMETERS;
   
   const int n = CCTK_VarIndex(varname);
@@ -299,59 +307,39 @@ int CarpetIOASCII<outdim>::OutputVarAs (cGH* const cgh, const char* const varnam
 	    abort();
 	  }
 	  
-	  // Traverse all components on all refinement levels
-	  assert (mglevel>=0);
-	  assert (reflevel==0);
-	  for (reflevel=0; reflevel<hh->reflevels(); ++reflevel) {
-	    enact_reflevel (cgh);
+	  // Traverse all components on this refinement and multigrid
+	  // level
+	  BEGIN_COMPONENT_LOOP(cgh) {
 	    
-	    assert (component==-1);
-	    for (component=0; component<hh->components(reflevel);
-		 ++component) {
-	      
-	      generic_gf<dim>* ff = 0;
-	      
-	      switch (CCTK_GroupTypeI(group)) {
-		
-	      case CCTK_ARRAY:
-		assert (var < (int)arrdata[group].data.size());
-		ff = arrdata[group].data[var];
-		break;
-		
-	      case CCTK_GF:
-		assert (var < (int)gfdata[group].data.size());
-		ff = gfdata[group].data[var];
-		break;
-		
-	      default:
-		abort();
-	      }
+	    generic_gf<dim>* ff = 0;
+	    
+	    switch (CCTK_GroupTypeI(group)) {
 	      
-	      const generic_data<dim>* const data
-		= (*ff) (tl, reflevel, component, mglevel);
-	      const vect<int,dim> offset1 = (offset * data->extent().stride()
-					     * vect<int,dim>(reflevelfactor));
+	    case CCTK_ARRAY:
+	      assert (var < (int)arrdata[group].data.size());
+	      ff = arrdata[group].data[var];
+	      break;
 	      
-	      data->write_ascii (filename, cgh->cctk_iteration, offset1, dirs,
-				 tl, reflevel, component, mglevel);
+	    case CCTK_GF:
+	      assert (var < (int)gfdata[group].data.size());
+	      ff = gfdata[group].data[var];
+	      break;
 	      
-	    } // Loop over components
-	    component = -1;
-	    
-	    // Append EOL after every complete set of components
-	    if (CCTK_MyProc(cgh)==0) {
-	      ofstream file(filename, ios::app);
-	      assert (file.good());
-	      file << endl;
-	      file.close();
-	      assert (file.good());
+	    default:
+	      abort();
 	    }
 	    
-	  } // Loop over refinement levels
-	  reflevel = 0;
-	  enact_reflevel (cgh);
+	    const generic_data<dim>* const data
+	      = (*ff) (tl, reflevel, component, mglevel);
+	    const bbox<int,dim> ext = data->extent();
+	    const vect<int,dim> offset1 = offset * ext.stride();
+	    
+	    data->write_ascii (filename, cgh->cctk_iteration, offset1, dirs,
+			       tl, reflevel, component, mglevel);
+	    
+	  } END_COMPONENT_LOOP(cgh);
 	  
-	  // Append EOL after every complete set of refinement levels
+	  // Append EOL after every complete set of components
 	  if (CCTK_MyProc(cgh)==0) {
 	    ofstream file(filename, ios::app);
 	    assert (file.good());
@@ -392,10 +380,12 @@ int CarpetIOASCII<outdim>::OutputVarAs (cGH* const cgh, const char* const varnam
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::TimeToOutput (cGH* const cgh, const int vindex) {
+int CarpetIOASCII<outdim>
+::TimeToOutput (cGH* const cgh, const int vindex)
+{
   DECLARE_CCTK_PARAMETERS;
   
-  assert (vindex>=0 && vindex<(int)last_output.size());
+  assert (vindex>=0 && vindex<CCTK_NumVars());
   
   const int myoutevery = GetIntParameter("out%dD_every", out_every);
   
@@ -414,7 +404,7 @@ int CarpetIOASCII<outdim>::TimeToOutput (cGH* const cgh, const int vindex) {
     return 0;
   }
   
-  if (last_output[vindex] == cgh->cctk_iteration) {
+  if (last_output[reflevel][vindex] == cgh->cctk_iteration) {
     // Has already been output during this iteration
     char* varname = CCTK_FullName(vindex);
     CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
@@ -426,7 +416,7 @@ int CarpetIOASCII<outdim>::TimeToOutput (cGH* const cgh, const int vindex) {
     return 0;
   }
   
-  assert (last_output[vindex] < cgh->cctk_iteration);
+  assert (last_output[reflevel][vindex] < cgh->cctk_iteration);
   
   // Should be output during this iteration
   return 1;
@@ -435,14 +425,16 @@ int CarpetIOASCII<outdim>::TimeToOutput (cGH* const cgh, const int vindex) {
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::TriggerOutput (cGH* const cgh, const int vindex) {
+int CarpetIOASCII<outdim>
+::TriggerOutput (cGH* const cgh, const int vindex)
+{
   assert (vindex>=0 && vindex<CCTK_NumVars());
   
   char* varname = CCTK_FullName(vindex);
   const int retval = OutputVarAs (cgh, varname, CCTK_VarName(vindex));
   free (varname);
   
-  last_output[vindex] = cgh->cctk_iteration;
+  last_output[reflevel][vindex] = cgh->cctk_iteration;
   
   return retval;
 }
@@ -450,10 +442,11 @@ int CarpetIOASCII<outdim>::TriggerOutput (cGH* const cgh, const int vindex) {
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::GetGridOffset (cGH* cgh, int dir,
-				    const char* itempl, const char* iglobal,
-				    const char* ctempl, const char* cglobal,
-				    CCTK_REAL cfallback)
+int CarpetIOASCII<outdim>
+::GetGridOffset (cGH* cgh, int dir,
+		 const char* itempl, const char* iglobal,
+		 const char* ctempl, const char* cglobal,
+		 const CCTK_REAL cfallback)
 {
   /* First choice: explicit coordinate */
   char cparam[1000];
@@ -510,7 +503,8 @@ int CarpetIOASCII<outdim>::GetGridOffset (cGH* cgh, int dir,
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::CoordToOffset (cGH* cgh, int dir, CCTK_REAL coord)
+int CarpetIOASCII<outdim>
+::CoordToOffset (cGH* cgh, int dir, CCTK_REAL coord)
 {
   CCTK_REAL lower, upper;
   CCTK_CoordRange (cgh, &lower, &upper, dir, 0, "cart3d");
@@ -527,8 +521,9 @@ int CarpetIOASCII<outdim>::CoordToOffset (cGH* cgh, int dir, CCTK_REAL coord)
 
 
 template<int outdim>
-const char* CarpetIOASCII<outdim>::GetStringParameter
-(const char* const parametertemplate, const char* const fallback)
+const char* CarpetIOASCII<outdim>
+::GetStringParameter (const char* const parametertemplate,
+		      const char* const fallback)
 {
   char parametername[1000];
   sprintf (parametername, parametertemplate, outdim);
@@ -548,8 +543,8 @@ const char* CarpetIOASCII<outdim>::GetStringParameter
 
 
 template<int outdim>
-int CarpetIOASCII<outdim>::GetIntParameter (const char* const parametertemplate,
-				      int fallback)
+int CarpetIOASCII<outdim>
+::GetIntParameter (const char* const parametertemplate, const int fallback)
 {
   char parametername[1000];
   sprintf (parametername, parametertemplate, outdim);
diff --git a/Carpet/CarpetIOASCII/src/ioascii.hh b/Carpet/CarpetIOASCII/src/ioascii.hh
index f56555c5a..061e4c18c 100644
--- a/Carpet/CarpetIOASCII/src/ioascii.hh
+++ b/Carpet/CarpetIOASCII/src/ioascii.hh
@@ -1,4 +1,4 @@
-// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.hh,v 1.3 2001/03/15 23:28:33 eschnett Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.hh,v 1.4 2001/03/16 21:32:17 eschnett Exp $
 
 #include <vector>
 
@@ -30,8 +30,9 @@ struct CarpetIOASCII {
   // Do truncate the output files for a variable
   static vector<bool> do_truncate;
   
-  // Last iteration on which a variable was output (-1 for none)
-  static vector<int> last_output;
+  // Last iteration on which a refinement level of a variable was
+  // output (INT_MIN for none)
+  static vector<vector<int> > last_output; // [rl][var]
   
   
   
diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index ba3cb79df..573e9b426 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -6,7 +6,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.7 2001/03/13 17:40:38 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.8 2001/03/16 21:32:17 eschnett Exp $
 
  ***************************************************************************/
 
@@ -162,14 +162,14 @@ void dh<D>::recompose () {
       	if (rl<h.reflevels()-1) {
       	  for (int cc=0; cc<h.components(rl+1); ++cc) {
       	    const ibbox intrf = boxes[rl+1][cc][ml].interior;
-      	    const ibbox extrf = boxes[rl+1][cc][ml].exterior;
+// 	    const ibbox extrf = boxes[rl+1][cc][ml].exterior;
       	    // Restriction (interior)
       	    {
       	      // (the restriction may fill the interior of the of the
-      	      // coarse grid, and may use the exterior of the fine
+      	      // coarse grid, and may use the interior of the fine
       	      // grid, and the bbox must be as large as possible)
-      	      const ibbox recv = extrf.contracted_for(intr) & intr;
-      	      const ibbox send = recv.expanded_for(extrf);
+      	      const ibbox recv = intrf.contracted_for(intr) & intr;
+      	      const ibbox send = recv.expanded_for(intrf);
       	      boxes[rl+1][cc][ml].send_ref_coarse[c ].push_back(send);
       	      boxes[rl  ][c ][ml].recv_ref_fine  [cc].push_back(recv);
       	    }
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index fa852b881..40a6338cf 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -5,7 +5,7 @@
     copyright            : (C) 2000 by Erik Schnetter
     email                : schnetter@astro.psu.edu
 
-    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.6 2001/03/12 16:54:25 eschnett Exp $
+    $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.7 2001/03/16 21:32:17 eschnett Exp $
 
  ***************************************************************************/
 
@@ -58,7 +58,7 @@ void generic_data<D>::write_ascii (const string name, const int time,
   assert (_has_storage);
   CHECKPOINT;
   
-  if (_proc==0) {
+  if (proc()==0) {
     // output on processor 0
     
     int rank;
@@ -118,8 +118,8 @@ void generic_data<D>::write_ascii (const string name, const int time,
     // copy to processor 0 and output there
     
     generic_data* const tmp = make_typed();
-    tmp->allocate(_extent, 0);
-    tmp->copy_from (this, _extent);
+    tmp->allocate(extent(), 0);
+    tmp->copy_from (this, extent());
     tmp->write_ascii (name, time, org, dirs, tl, rl, c, ml);
     delete tmp;
     
diff --git a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc
index 5f87c41a9..e0494da20 100644
--- a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc
+++ b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc
@@ -1,4 +1,5 @@
 #include <cassert>
+#include <climits>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -27,7 +28,7 @@
 
 #include "ioflexio.hh"
 
-static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc,v 1.1 2001/03/15 23:28:50 eschnett Exp $";
+static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc,v 1.2 2001/03/16 21:32:17 eschnett Exp $";
 
 
 
@@ -41,7 +42,7 @@ namespace CarpetIOFlexIO {
   int GHExtension;
   int IOMethod;
   vector<bool> do_truncate;
-  vector<int> last_output;
+  vector<vector<int> > last_output;
   
   
   
@@ -81,10 +82,14 @@ namespace CarpetIOFlexIO {
     do_truncate.resize(CCTK_NumVars(), ! IOUtil_RestartFromRecovery(cgh));
     
     // No iterations have yet been output
-    last_output.resize(CCTK_NumVars(), -1);
+    last_output.resize(maxreflevels);
+    for (int rl=0; rl<maxreflevels; ++rl) {
+      last_output[rl].resize(CCTK_NumVars(), INT_MIN);
+    }
     
     // We register only once, ergo we get only one handle.  We store
-    // that statically, so there is no need to pass it to Cactus.
+    // that statically, so there is no need to pass anything to
+    // Cactus.
     return 0;
   }
   
@@ -194,22 +199,22 @@ namespace CarpetIOFlexIO {
 	
 	// Set datatype
 	assert (CCTK_VarTypeI(n) == CCTK_VARIABLE_REAL8);
+	// TODO: Set datatype correctly
 	amrwriter->setType (IObase::Float64);
 	
 	// Set coordinate information
-	assert (reflevel==0);
-	double lower[dim], upper[dim];
+	CCTK_REAL lower[dim], upper[dim];
 	double origin[dim], delta[dim], timestep;
 	for (int d=0; d<dim; ++d) {
 	  const int ierr = CCTK_CoordRange
 	    (cgh, &lower[d], &upper[d], d+1, 0, "cart3d");
 	  assert (ierr==0);
 	  origin[d] = lower[d];
-	  delta[d] = cgh->cctk_delta_space[d];
+	  delta[d] = cgh->cctk_delta_space[d] / reflevelfact;
 	}
-	timestep = cgh->cctk_delta_time;
+	timestep = cgh->cctk_delta_time / reflevelfact;
 	amrwriter->setTopLevelParameters
-	  (dim, origin, delta, timestep, hh->reflevels());
+	  (dim, origin, delta, timestep, maxreflevels);
 	
 	// Set refinement information
 	int interlevel_timerefinement, interlevel_spacerefinement[dim];
@@ -219,66 +224,59 @@ namespace CarpetIOFlexIO {
 	}
 	amrwriter->setRefinement
 	  (interlevel_timerefinement, interlevel_spacerefinement);
-      }
-      
-      // Traverse all components on all refinement levels
-      assert (mglevel>=0);
-      assert (reflevel==0);
-      for (reflevel=0; reflevel<hh->reflevels(); ++reflevel) {
-	enact_reflevel (cgh);
 	
+	// Set level
 	amrwriter->setLevel(reflevel);
+	
+	// Set current time
 	amrwriter->setTime(cgh->cctk_iteration);
+      }
+      
+      // Traverse all components on this refinement and multigrid
+      // level
+      BEGIN_COMPONENT_LOOP(cgh) {
 	
-	assert (component==-1);
-	for (component=0; component<hh->components(reflevel); ++component) {
-	  
-	  generic_gf<dim>* ff = 0;
-	  
-	  switch (CCTK_GroupTypeI(group)) {
-	    
-	  case CCTK_ARRAY:
-	    assert (var < (int)arrdata[group].data.size());
-	    ff = arrdata[group].data[var];
-	    break;
-	    
-	  case CCTK_GF:
-	    assert (var < (int)gfdata[group].data.size());
-	    ff = gfdata[group].data[var];
-	    break;
-	    
-	  default:
-	    abort();
-	  }
+	generic_gf<dim>* ff = 0;
+	
+	switch (CCTK_GroupTypeI(group)) {
 	  
-	  const generic_data<dim>* const data
-	    = (*ff) (tl, reflevel, component, mglevel);
+	case CCTK_ARRAY:
+	  assert (var < (int)arrdata[group].data.size());
+	  ff = arrdata[group].data[var];
+	  break;
 	  
-	  // Make temporary copy on processor 0
-	  const bbox<int,dim> ext = data->extent();
-	  generic_data<dim>* tmp = data->make_typed ();
-	  tmp->allocate (ext, 0);
-	  tmp->copy_from (data, ext);
+	case CCTK_GF:
+	  assert (var < (int)gfdata[group].data.size());
+	  ff = gfdata[group].data[var];
+	  break;
 	  
-	  // Write data
-	  if (CCTK_MyProc(cgh)==0) {
-	    int origin[dim], dims[dim];
-	    for (int d=0; d<dim; ++d) {
-	      origin[d] = (ext.lower() / ext.stride())[d];
-	      dims[d]   = ((ext.upper() - ext.lower()) / ext.stride() + 1)[d];
-	    }
-	    amrwriter->write (origin, dims, (void*)data->storage());
+	default:
+	  abort();
+	}
+	
+	const generic_data<dim>* const data
+	  = (*ff) (tl, reflevel, component, mglevel);
+	
+	// Make temporary copy on processor 0
+	const bbox<int,dim> ext = data->extent();
+	generic_data<dim>* tmp = data->make_typed ();
+	tmp->allocate (ext, 0);
+	tmp->copy_from (data, ext);
+	
+	// Write data
+	if (CCTK_MyProc(cgh)==0) {
+	  int origin[dim], dims[dim];
+	  for (int d=0; d<dim; ++d) {
+	    origin[d] = (ext.lower() / ext.stride())[d];
+	    dims[d]   = ((ext.upper() - ext.lower()) / ext.stride() + 1)[d];
 	  }
-	  
-	  // Delete temporary copy
-	  delete tmp;
-	  
-	} // Loop over components
-	component = -1;
+	  amrwriter->write (origin, dims, (void*)data->storage());
+	}
+	
+	// Delete temporary copy
+	delete tmp;
 	
-      } // Loop over refinement levels
-      reflevel = 0;
-      enact_reflevel (cgh);
+      } END_COMPONENT_LOOP(cgh);
       
       // Close the file
       if (CCTK_MyProc(cgh)==0) {
@@ -305,7 +303,7 @@ namespace CarpetIOFlexIO {
   int TimeToOutput (cGH* const cgh, const int vindex) {
     DECLARE_CCTK_PARAMETERS;
     
-    assert (vindex>=0 && vindex<(int)last_output.size());
+    assert (vindex>=0 && vindex<(int)last_output[reflevel].size());
     
     const int myoutevery = GetIntParameter("out3D_every", out_every);
     
@@ -319,12 +317,12 @@ namespace CarpetIOFlexIO {
       return 0;
     }
     
-    if (! CheckForVariable(cgh, GetStringParameter("out3D_vars", ""), vindex)) {
+    if (! CheckForVariable(cgh, GetStringParameter("out3D_vars",""), vindex)) {
       // This variable should not be output
       return 0;
     }
     
-    if (last_output[vindex] == cgh->cctk_iteration) {
+    if (last_output[reflevel][vindex] == cgh->cctk_iteration) {
       // Has already been output during this iteration
       char* varname = CCTK_FullName(vindex);
       CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING,
@@ -336,7 +334,7 @@ namespace CarpetIOFlexIO {
       return 0;
     }
     
-    assert (last_output[vindex] < cgh->cctk_iteration);
+    assert (last_output[reflevel][vindex] < cgh->cctk_iteration);
     
     // Should be output during this iteration
     return 1;
@@ -351,7 +349,7 @@ namespace CarpetIOFlexIO {
     const int retval = OutputVarAs (cgh, varname, CCTK_VarName(vindex));
     free (varname);
     
-    last_output[vindex] = cgh->cctk_iteration;
+    last_output[reflevel][vindex] = cgh->cctk_iteration;
     
     return retval;
   }
diff --git a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh
index 0b43a3eeb..a40550028 100644
--- a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh
+++ b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh
@@ -1,4 +1,4 @@
-// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh,v 1.1 2001/03/15 23:28:50 eschnett Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.hh,v 1.2 2001/03/16 21:32:17 eschnett Exp $
 
 #include <vector>
 
@@ -10,7 +10,7 @@ namespace CarpetIOFlexIO {
   extern int GHExtension;
   extern int IOMethod;
   extern vector<bool> do_truncate;
-  extern vector<int> last_output;
+  extern vector<vector<int> > last_output;
   
   // Scheduled functions
   extern "C" {
diff --git a/CarpetExtra/IDScalarWave/src/InitialData.F77 b/CarpetExtra/IDScalarWave/src/InitialData.F77
index e0c2398d8..63fadc2b7 100644
--- a/CarpetExtra/IDScalarWave/src/InitialData.F77
+++ b/CarpetExtra/IDScalarWave/src/InitialData.F77
@@ -40,7 +40,7 @@ c     -*-Fortran-*-
       INTEGER    i,j,k
       CCTK_REAL  dt,omega, pi
       
-c      print '("IDScalarWave_InitialData")'
+c     call CCTK_INFO ("IDScalarWave_InitialData")
       
       pi = 4.d0*atan(1.d0)
 
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par
index e8584d3ed..7954e1fe0 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par,v 1.1 2001/03/14 14:57:38 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -22,6 +22,7 @@ driver::global_nz		= 61
 grid::type			= byspacing
 grid::dxyz			= 0.3
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic"
 
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par
index d21cf84ac..86e6b91c6 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par,v 1.1 2001/03/15 09:45:05 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -22,6 +22,7 @@ driver::global_nz		= 31
 grid::type			= byspacing
 grid::dxyz			= 0.6
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic_coarse"
 
@@ -31,7 +32,7 @@ IOBasic::outinfo_every		= 1 # 10
 #IOBasic::outScalar_every	= 2
 #IOBasic::outScalar_vars		= "wavetoy::phi"
 
-IOASCII::out1D_every		= 1 # 2
+IOASCII::out1D_every		= 2 # 1 # 2
 IOASCII::out1D_vars		= "wavetoy::phi grid::coordinates"
 
 WaveToyF77::bound		= radiation
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par
index 3910fd4a0..e16bc1bbc 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par,v 1.1 2001/03/15 09:45:06 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl2.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -24,6 +24,7 @@ Carpet::max_refinement_levels	= 2
 grid::type			= byspacing
 grid::dxyz			= 0.6
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic_coarse_rl2"
 
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par
index 4f3d63665..6167a146d 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par,v 1.1 2001/03/15 09:45:06 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_coarse_rl3.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -24,6 +24,7 @@ Carpet::max_refinement_levels	= 3
 grid::type			= byspacing
 grid::dxyz			= 0.6
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic_coarse_rl3"
 
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par
index fbdd6063a..f5ab34e5d 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par,v 1.1 2001/03/14 14:57:38 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl2.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -24,6 +24,7 @@ Carpet::max_refinement_levels	= 2
 grid::type			= byspacing
 grid::dxyz			= 0.3
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic_rl2"
 
diff --git a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par
index e58353630..9a089ab13 100644
--- a/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par
+++ b/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par
@@ -7,7 +7,7 @@
 #   @enddesc
 # @@*/
 #
-# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par,v 1.1 2001/03/14 14:57:38 eschnett Exp $
+# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyF77/par/wavetoyf77_periodic_rl3.par,v 1.2 2001/03/16 21:32:18 eschnett Exp $
 
 ActiveThorns = "Boundary IOBasic IOUtil Time Cart3d Carpet CarpetIOASCII CarpetLib CarpetSlab IDScalarWave WaveToyF77"
 
@@ -24,6 +24,7 @@ Carpet::max_refinement_levels	= 3
 grid::type			= byspacing
 grid::dxyz			= 0.3
 grid::mode			= periodic
+grid::avoid_origin		= no
 
 IO::outdir			= "wavetoyf77_periodic_rl3"