Re-organise time level handling

Store the current Cactus time (and not a fake Carpet time) in the th
"time hiearchy".  This removes the now redundant "leveltimes" data
structure in Carpet.

Add past time levels to th, so that it can store the time for past
time levels instead of assuming the time step size is constant.  This
allows changing the time step size during evolution.

Share the time hierarchy between all maps, instead of having one time
hierarchy per map.

Simplify the time level cycling and time stepping code used during
evolution.

Improve structure of the code that loops over time levels for certain
schedule bins.  Introduce a new Carpet variable "timelevel", similar
to "reflevel".

This also makes it possible to avoid time interpolation for the past
time levels during regridding.  The past time levels of the fine grid
then remain aligned (in time) with the past time levels of the coarse
grid.  This is controlled by a new parameter
"time_interpolation_during_regridding", which defaults to "yes" for
backwards compatibility.

Simplify the three time level initialisation.  Instead of initialising
all three time levels by taking altogether three time steps (forwards
and backwards), initialise only one past time level by taking one time
step backwards.  The remaining time level is initialised during the
first time step of the evolution, which begins by cycling time levels,
which drops the non-initialised last time level anyway.

Update Carpet and the mode handling correspondingly.

Update the CarpetIOHDF5 checkpoint format correspondingly.

Update CarpetInterp, CarpetReduce, and CarpetRegrid2 correspondingly.

Update CarpetJacobi and CarpetMG correspondingly.

13 files changed, 349 insertions, 566 deletions

diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index 64f2f0480..c01a59fa3 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -22,7 +22,6 @@ namespace Carpet {
 
 
   static void ProlongateGroupBoundaries (const cGH* cctkGH,
-                                         CCTK_REAL initial_time,
                                          const vector<int>& groups);
 
 
@@ -137,7 +136,7 @@ namespace Carpet {
       if (local_do_prolongate) {
         static Timer timer ("Evolve::Prolongate");
         timer.start();
-        ProlongateGroupBoundaries (cctkGH, cctk_initial_time, goodgroups);
+        ProlongateGroupBoundaries (cctkGH, goodgroups);
         timer.stop();
       }
       
@@ -156,11 +155,9 @@ namespace Carpet {
 
   // Prolongate the boundaries of all CCTK_GF groups in the given set
   static void ProlongateGroupBoundaries (const cGH* cctkGH,
-                                         const CCTK_REAL initial_time,
                                          const vector<int>& groups)
   {
     DECLARE_CCTK_PARAMETERS;
-    const int tl = 0;
     
     if (reflevel == 0) return;
     
@@ -169,8 +166,7 @@ namespace Carpet {
     assert (groups.size() > 0);
 
     // use the current time here (which may be modified by the user)
-    const CCTK_REAL time =
-      (cctkGH->cctk_time - initial_time) / delta_time;
+    const CCTK_REAL time = cctkGH->cctk_time;
     
     for (comm_state state; not state.done(); state.step()) {
       for (int group = 0; group < (int)groups.size(); ++group) {
@@ -180,11 +176,16 @@ namespace Carpet {
           continue;
         }
         assert (reflevel>=0 and reflevel<reflevels);
+        const int active_tl =
+          groupdata.AT(g).activetimelevels.AT(mglevel).AT(reflevel);
+        assert (active_tl>=0);
+        const int tl = active_tl > 1 ? timelevel : 0;
         
         for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
           for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
             ggf *const gv = arrdata.AT(g).AT(m).data.AT(v);
-            gv->ref_bnd_prolongate_all (state, tl, reflevel, mglevel, time);
+            gv->ref_bnd_prolongate_all
+              (state, tl, reflevel, mglevel, time);
           }
         }
       }
@@ -196,7 +197,6 @@ namespace Carpet {
   void SyncGroups (const cGH* cctkGH, const vector<int>& groups)
   {
     DECLARE_CCTK_PARAMETERS;
-    const int tl = 0;
 
     Checkpoint ("SyncGroups");
 
@@ -208,6 +208,10 @@ namespace Carpet {
         const int grouptype = CCTK_GroupTypeI (g);
         const int ml = grouptype == CCTK_GF ? mglevel : 0;
         const int rl = grouptype == CCTK_GF ? reflevel : 0;
+        const int active_tl =
+          groupdata.AT(g).activetimelevels.AT(mglevel).AT(reflevel);
+        assert (active_tl>=0);
+        const int tl = active_tl > 1 ? timelevel : 0;
         for (int m = 0; m < (int)arrdata.AT(g).size(); ++m) {
           for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
             arrdesc& array = arrdata.AT(g).AT(m);
diff --git a/Carpet/Carpet/src/Cycle.cc b/Carpet/Carpet/src/Cycle.cc
index acf4d19db..582a966e5 100644
--- a/Carpet/Carpet/src/Cycle.cc
+++ b/Carpet/Carpet/src/Cycle.cc
@@ -17,17 +17,21 @@ namespace Carpet {
   
   
   
-  void CycleTimeLevels (const cGH* cgh)
+  void CycleTimeLevels (cGH* const cctkGH)
   {
     DECLARE_CCTK_PARAMETERS;
     
     Checkpoint ("CycleTimeLevels");
     assert (is_level_mode());
     
+    assert (timelevel == 0);
+    tt->advance_time (mglevel, reflevel);
+    cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel);
+    
     for (int group=0; group<CCTK_NumGroups(); ++group) {
-      if (CCTK_QueryGroupStorageI(cgh, group)) {
+      if (CCTK_QueryGroupStorageI(cctkGH, group)) {
         
-        int const activetimelevels = CCTK_ActiveTimeLevelsGI (cgh, group);
+        int const activetimelevels = CCTK_ActiveTimeLevelsGI (cctkGH, group);
         if (activetimelevels > 1) {
           if (activetimelevels < prolongation_order_time+1) {
             char * const groupname = CCTK_GroupName (group);
@@ -62,7 +66,7 @@ namespace Carpet {
 	      {
                 int const varindex = firstvarindex + var;
 		for (int tl=0; tl<numtimelevels; ++tl) {
-		  cgh->data[varindex][tl]
+		  cctkGH->data[varindex][tl]
 		    = (tl < groupdata.AT(group).info.activetimelevels
 		       ? ((*arrdata.AT(group).AT(0).data.AT(var))
 			  (tl, 0, 0, 0)->storage())
@@ -82,17 +86,91 @@ namespace Carpet {
   
   
   
-  void FlipTimeLevels (const cGH* cgh)
+  void UncycleTimeLevels (cGH* const cctkGH)
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    Checkpoint ("UncycleTimeLevels");
+    assert (is_level_mode());
+    
+    assert (timelevel == 0);
+    tt->retreat_time (mglevel, reflevel);
+    cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel);
+    
+    for (int group=0; group<CCTK_NumGroups(); ++group) {
+      if (CCTK_QueryGroupStorageI(cctkGH, group)) {
+        
+        int const activetimelevels = CCTK_ActiveTimeLevelsGI (cctkGH, group);
+        if (activetimelevels > 1) {
+          if (activetimelevels < prolongation_order_time+1) {
+            char * const groupname = CCTK_GroupName (group);
+            CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                        "Group \"%s\" has %d only active time levels.  Groups with more than one active time level need at least %d active time levels for prolongation_order_time=%d",
+                        groupname,
+                        activetimelevels, int(prolongation_order_time+1),
+                        int(prolongation_order_time));
+            free (groupname);
+          }
+        }
+        
+        switch (CCTK_GroupTypeI(group)) {
+          
+        case CCTK_GF:
+          assert (reflevel>=0 and reflevel<reflevels);
+	  for (int m=0; m<(int)arrdata.AT(group).size(); ++m) {
+	    for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) {
+              arrdata.AT(group).AT(m).data.AT(var)->
+                uncycle_all (reflevel, mglevel);
+            }
+          }
+          break;
+          
+        case CCTK_SCALAR:
+        case CCTK_ARRAY:
+          if (do_global_mode) {
+	    int const numtimelevels = CCTK_NumTimeLevelsI (group);
+	    int const firstvarindex = CCTK_FirstVarIndexI (group);
+            for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) {
+              arrdata.AT(group).AT(0).data.AT(var)->uncycle_all (0, mglevel);
+	      {
+                int const varindex = firstvarindex + var;
+		for (int tl=0; tl<numtimelevels; ++tl) {
+		  cctkGH->data[varindex][tl]
+		    = (tl < groupdata.AT(group).info.activetimelevels
+		       ? ((*arrdata.AT(group).AT(0).data.AT(var))
+			  (tl, 0, 0, 0)->storage())
+		       : NULL);
+		}
+	      }
+            }
+          }
+          break;
+          
+        default:
+          assert (0);
+        } // switch grouptype
+      } // if storage
+    } // for group
+  }
+  
+  
+  
+  void FlipTimeLevels (cGH* const cctkGH)
   {
     DECLARE_CCTK_PARAMETERS;
     
     Checkpoint ("FlipTimeLevels");
     assert (is_level_mode());
     
+    assert (timelevel == 0);
+    tt->flip_timelevels (mglevel, reflevel);
+    cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel);
+    cctkGH->cctk_delta_time *= -1;
+    
     for (int group=0; group<CCTK_NumGroups(); ++group) {
-      if (CCTK_QueryGroupStorageI(cgh, group)) {
+      if (CCTK_QueryGroupStorageI(cctkGH, group)) {
         
-        int const activetimelevels = CCTK_ActiveTimeLevelsGI (cgh, group);
+        int const activetimelevels = CCTK_ActiveTimeLevelsGI (cctkGH, group);
         if (activetimelevels > 1) {
           if (activetimelevels < prolongation_order_time+1) {
             char * const groupname = CCTK_GroupName (group);
@@ -127,7 +205,7 @@ namespace Carpet {
 	      {
                 int const varindex = firstvarindex + var;
 		for (int tl=0; tl<numtimelevels; ++tl) {
-		  cgh->data[varindex][tl]
+		  cctkGH->data[varindex][tl]
 		    = (tl < groupdata.AT(group).info.activetimelevels
 		       ? ((*arrdata.AT(group).AT(0).data.AT(var))
 			  (tl, 0, 0, 0)->storage())
@@ -147,13 +225,13 @@ namespace Carpet {
   
   
   
-  void FillTimeLevels (const cGH* cgh)
+  void FillTimeLevels (const cGH* const cctkGH)
   {
     Checkpoint ("FillTimeLevels");
     assert (is_level_mode());
     
     for (int group=0; group<CCTK_NumGroups(); ++group) {
-      if (CCTK_QueryGroupStorageI(cgh, group)) {
+      if (CCTK_QueryGroupStorageI(cctkGH, group)) {
         switch (CCTK_GroupTypeI(group)) {
           
         case CCTK_GF:
diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc
index baadd19bd..3ed5ae848 100644
--- a/Carpet/Carpet/src/Evolve.cc
+++ b/Carpet/Carpet/src/Evolve.cc
@@ -95,7 +95,9 @@ namespace Carpet {
             int const do_every =
               ipow (mgfact, ml) * (maxtimereflevelfact / timereffacts.AT(rl));
             if (cctkGH->cctk_iteration % do_every == 0) {
-              assert (abs (leveltimes.AT(ml).AT(rl) - global_time) <=
+              // assert (abs (leveltimes.AT(ml).AT(rl) - global_time) <=
+              //         eps * global_time);
+              assert (abs (tt->get_time(ml,rl,0) - global_time) <=
                       eps * global_time);
             }
           }
@@ -296,57 +298,19 @@ namespace Carpet {
                 do_global_mode = do_late_global_mode;
                 do_meta_mode = do_late_meta_mode;
                 
-                Waypoint ("Postregrid at iteration %d time %g%s%s",
-                          cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-                          (do_global_mode ? " (global)" : ""),
-                          (do_meta_mode ? " (meta)" : ""));
-                
-                int const num_tl = prolongation_order_time+1;
-                
-                bool const old_do_allow_past_timelevels =
-                  do_allow_past_timelevels;
-                do_allow_past_timelevels = false;
-                
-                // Rewind times
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                for (int tl=0; tl<num_tl; ++tl) {
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                }
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
-                cctkGH->cctk_time -=
-                  num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
-                
-                for (int tl=0; tl<num_tl; ++tl) {
+                BEGIN_TIMELEVEL_LOOP (cctkGH) {
                   
-                  // Advance times
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                  cctkGH->cctk_time +=
-                    cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
+                  Waypoint ("Postregrid at iteration %d time %g timelevel %d%s%s",
+                            cctkGH->cctk_iteration,
+                            (double)cctkGH->cctk_time,
+                            timelevel,
+                            (do_global_mode ? " (global)" : ""),
+                            (do_meta_mode ? " (meta)" : ""));
                   
                   // Postregrid
                   ScheduleTraverse (where, "CCTK_POSTREGRID", cctkGH);
                   
-                } // for tl
-                cctkGH->cctk_time = old_cctk_time;
-                
-                do_allow_past_timelevels = old_do_allow_past_timelevels;
+                } END_TIMELEVEL_LOOP;
                 
                 EndTimingLevel (cctkGH);
               } LEAVE_LEVEL_MODE;
@@ -413,43 +377,14 @@ namespace Carpet {
               }
               
               // Advance times
-              cctkGH->cctk_time
-                = (global_time
-                   - delta_time / maxtimereflevelfact
-                   + delta_time * mglevelfact / timereflevelfact);
-              CCTK_REAL const carpet_time = cctkGH->cctk_time / delta_time;
-              for (int m=0; m<maps; ++m) {
-                vtt.AT(m)->advance_time (reflevel, mglevel);
-                if (not adaptive_stepsize) {
-#if 0
-                  // We must not perform this check, since the
-                  // relative accuracy of incrementally adding to the
-                  // current time cannot be good enough.  Just setting
-                  // the time (see below) is fine.
-                  CCTK_REAL const eps = 1.0e-12;
-                  static_assert (abs(0.1) > 0,
-                                 "Function CarpetLib::abs has wrong signature");
-                  CCTK_REAL const level_time =
-                    vtt.AT(m)->get_time (reflevel, mglevel);
-                  if (not (abs (level_time - carpet_time) <=
-                           eps * max (carpet_time, 1.0))) {
-                    int const oldprecision = cerr.precision();
-                    cerr.precision (17);
-                    cerr << "ml: " << ml << endl
-                         << "rl: " << rl << endl
-                         << "m: " << m << endl
-                         << "level_time: " << level_time << endl
-                         << "carpet_time: " << carpet_time << endl
-                         << "(level_time - carpet_time): " << (level_time - carpet_time) << endl;
-                    cerr.precision (oldprecision);
-                  }
-                  assert (abs (level_time - carpet_time) <=
-                          eps * max (carpet_time, 1.0));
-#endif
-                  vtt.AT(m)->set_time (reflevel, mglevel, carpet_time);
-                }
-              }
               CycleTimeLevels (cctkGH);
+              if (not adaptive_stepsize) {
+                cctkGH->cctk_time
+                  = (global_time
+                     - delta_time / maxtimereflevelfact
+                     + delta_time * mglevelfact / timereflevelfact);
+                tt->set_time (mglevel, reflevel, timelevel, cctkGH->cctk_time);
+              }
               
               Waypoint ("Evolution I at iteration %d time %g%s%s%s",
                         cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -633,7 +568,7 @@ namespace Carpet {
       streamsize const oldprecision = cout.precision();
       cout.precision (17);
       cout << "   global_time: " << global_time << endl
-           << "   leveltimes: " << leveltimes << endl
+        // << "   leveltimes: " << leveltimes << endl
            << "   delta_time: " << delta_time << endl;
       cout.precision (oldprecision);
     }
diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc
index f81c0d368..1ff189dd7 100644
--- a/Carpet/Carpet/src/Initialise.cc
+++ b/Carpet/Carpet/src/Initialise.cc
@@ -64,8 +64,14 @@ namespace Carpet {
     global_time = cctk_initial_time;
     delta_time = 1.0;
     for (int ml = 0; ml < mglevel; ++ ml) {
-      assert (leveltimes.AT(ml).size() == 1);
-      leveltimes.AT(ml).AT(0) = global_time;
+      // assert (leveltimes.AT(ml).size() == 1);
+      // leveltimes.AT(ml).AT(0) = global_time;
+      for (int rl = 0; rl < reflevels; ++ rl) {
+        CCTK_REAL const dt = delta_time / timereffacts.AT(rl);
+        for (int tl = 0; tl < tt->timelevels; ++ tl) {
+          tt->set_time (ml, rl, tl, global_time - tl * dt);
+        }
+      }
     }
     
     cctkGH->cctk_iteration = 0;
@@ -99,6 +105,10 @@ namespace Carpet {
       
       CallRecoverVariables (cctkGH);
       CallPostRecoverVariables (cctkGH);
+      // TODO: We should probably restrict here.
+      // CallRestrict (cctkGH);
+      // TODO: Should we also execute another bin after this?
+      // CallPostRestrictRecover (cctkGH);
       print_internal_data ();
       
     } else {
@@ -263,47 +273,14 @@ namespace Carpet {
           do_global_mode = do_early_global_mode; // on first iteration, coarsest grid
           do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid
           
-          Waypoint ("Recovering II at iteration %d time %g%s%s",
-                    cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-                    (do_global_mode ? " (global)" : ""),
-                    (do_meta_mode ? " (meta)" : ""));
-          
-          int const num_tl = prolongation_order_time+1;
-          
-          bool const old_do_allow_past_timelevels = do_allow_past_timelevels;
-          do_allow_past_timelevels = false;
-          
-          // Rewind times
-          for (int m=0; m<maps; ++m) {
-            CCTK_REAL const old_delta =
-              vtt.AT(m)->get_delta (reflevel, mglevel);
-            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-          }
-          FlipTimeLevels (cctkGH);
-          for (int tl=0; tl<num_tl; ++tl) {
-            for (int m=0; m<maps; ++m) {
-              vtt.AT(m)->advance_time (reflevel, mglevel);
-            }
-            CycleTimeLevels (cctkGH);
-          }
-          for (int m=0; m<maps; ++m) {
-            CCTK_REAL const old_delta =
-              vtt.AT(m)->get_delta (reflevel, mglevel);
-            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-          }
-          FlipTimeLevels (cctkGH);
-          CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
-          cctkGH->cctk_time -=
-            num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
-          
-          for (int tl=0; tl<num_tl; ++tl) {
+          BEGIN_TIMELEVEL_LOOP (cctkGH) {
             
-            // Advance times
-            for (int m=0; m<maps; ++m) {
-              vtt.AT(m)->advance_time (reflevel, mglevel);
-            }
-            CycleTimeLevels (cctkGH);
-            cctkGH->cctk_time += cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
+            Waypoint ("Recovering II at iteration %d time %g timelevel %d%s%s",
+                      cctkGH->cctk_iteration,
+                      (double)cctkGH->cctk_time,
+                      timelevel,
+                      (do_global_mode ? " (global)" : ""),
+                      (do_meta_mode ? " (meta)" : ""));
             
             // Post recover variables
             ScheduleTraverse (where, "CCTK_POST_RECOVER_VARIABLES", cctkGH);
@@ -311,10 +288,7 @@ namespace Carpet {
             // Checking
             PoisonCheck (cctkGH, currenttime);
             
-          } // for tl
-          cctkGH->cctk_time = old_cctk_time;
-          
-          do_allow_past_timelevels = old_do_allow_past_timelevels;
+          } END_TIMELEVEL_LOOP;
           
           CheckChecksums (cctkGH, allbutcurrenttime);
           
@@ -384,61 +358,36 @@ namespace Carpet {
           // Set up the grids
           ScheduleTraverse (where, "CCTK_BASEGRID", cctkGH);
           
-          int const num_tl =
-            init_each_timelevel ? prolongation_order_time+1 : 1;
-          
-          bool const old_do_allow_past_timelevels = do_allow_past_timelevels;
-          do_allow_past_timelevels =
-            not CCTK_EQUALS (initial_data_setup_method, "init_single_level");
-          
-          // Rewind times
-          for (int m=0; m<maps; ++m) {
-            CCTK_REAL const old_delta =
-              vtt.AT(m)->get_delta (reflevel, mglevel);
-            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-          }
-          FlipTimeLevels (cctkGH);
-          for (int tl=0; tl<num_tl; ++tl) {
-            for (int m=0; m<maps; ++m) {
-              vtt.AT(m)->advance_time (reflevel, mglevel);
-            }
-            CycleTimeLevels (cctkGH);
-          }
-          for (int m=0; m<maps; ++m) {
-            CCTK_REAL const old_delta =
-              vtt.AT(m)->get_delta (reflevel, mglevel);
-            vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-          }
-          FlipTimeLevels (cctkGH);
-          CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
-          cctkGH->cctk_time -=
-            num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
-          
-          for (int tl=0; tl<num_tl; ++tl) {
+          if (init_each_timelevel) {
             
-            // Advance times
-            for (int m=0; m<maps; ++m) {
-              vtt.AT(m)->advance_time (reflevel, mglevel);
-            }
-            CycleTimeLevels (cctkGH);
-            cctkGH->cctk_time += cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
+            BEGIN_TIMELEVEL_LOOP (cctkGH) {
+              
+              // Set up the initial data
+              ScheduleTraverse (where, "CCTK_INITIAL", cctkGH);
+              ScheduleTraverse (where, "CCTK_POSTINITIAL", cctkGH);
+              
+            } END_TIMELEVEL_LOOP;
+            
+          } else {              // not init_each_timelevel
+            
+            assert (do_allow_past_timelevels);
+            do_allow_past_timelevels =
+              not CCTK_EQUALS (initial_data_setup_method, "init_single_level");
             
             // Set up the initial data
             ScheduleTraverse (where, "CCTK_INITIAL", cctkGH);
             ScheduleTraverse (where, "CCTK_POSTINITIAL", cctkGH);
             
-            if (init_fill_timelevels) {
-              assert (tl==0);
-              FillTimeLevels (cctkGH);
-            }
-            
-            // Checking
-            PoisonCheck (cctkGH, currenttime);
+            do_allow_past_timelevels = true;
             
-          } // for tl
-          cctkGH->cctk_time = old_cctk_time;
+          } // not init_each_timelevel
           
-          do_allow_past_timelevels = old_do_allow_past_timelevels;
+          if (init_fill_timelevels) {
+            FillTimeLevels (cctkGH);
+          }
+          
+          // Checking
+          PoisonCheck (cctkGH, currenttime);
           
           if (regrid_during_initialisation and mglevel==0) {
             // Regrid after initialising each level
@@ -875,57 +824,19 @@ namespace Carpet {
                 do_global_mode = do_late_global_mode;
                 do_meta_mode = do_late_meta_mode;
                 
-                Waypoint ("Postregrid at iteration %d time %g%s%s",
-                          cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-                          (do_global_mode ? " (global)" : ""),
-                          (do_meta_mode ? " (meta)" : ""));
-                
-                int const num_tl = prolongation_order_time + 1;
-                
-                bool const old_do_allow_past_timelevels =
-                  do_allow_past_timelevels;
-                do_allow_past_timelevels = false;
-                
-                // Rewind times
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                for (int tl=0; tl<num_tl; ++tl) {
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                }
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
-                cctkGH->cctk_time -=
-                  num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
+                BEGIN_TIMELEVEL_LOOP (cctkGH) {
                 
-                for (int tl=0; tl<num_tl; ++tl) {
-                  
-                  // Advance times
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                  cctkGH->cctk_time +=
-                    cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
+                  Waypoint ("Postregrid at iteration %d time %g timelevel %d%s%s",
+                            cctkGH->cctk_iteration,
+                            (double)cctkGH->cctk_time,
+                            timelevel,
+                            (do_global_mode ? " (global)" : ""),
+                            (do_meta_mode ? " (meta)" : ""));
                   
                   // Postregrid
                   ScheduleTraverse (where, "CCTK_POSTREGRID", cctkGH);
                   
-                } // for tl
-                cctkGH->cctk_time = old_cctk_time;
-                
-                do_allow_past_timelevels = old_do_allow_past_timelevels;
+                } END_TIMELEVEL_LOOP;
                 
                 EndTimingLevel (cctkGH);
               } LEAVE_LEVEL_MODE;
@@ -1079,53 +990,26 @@ namespace Carpet {
                           (do_global_mode ? " (global)" : ""),
                           (do_meta_mode ? " (meta)" : ""));
                 
-                int const num_tl =
-                  init_each_timelevel ? prolongation_order_time+1 : 1;
-                
-                bool const old_do_allow_past_timelevels =
-                  do_allow_past_timelevels;
-                do_allow_past_timelevels = false;
-                
-                // Rewind times
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                for (int tl=0; tl<num_tl; ++tl) {
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                }
-                for (int m=0; m<maps; ++m) {
-                  CCTK_REAL const old_delta =
-                    vtt.AT(m)->get_delta (reflevel, mglevel);
-                  vtt.AT(m)->set_delta (reflevel, mglevel, - old_delta);
-                }
-                FlipTimeLevels (cctkGH);
-                CCTK_REAL const old_cctk_time = cctkGH->cctk_time;
-                cctkGH->cctk_time -=
-                  num_tl * (cctkGH->cctk_delta_time / cctkGH->cctk_timefac);
-                
-                for (int tl=0; tl<num_tl; ++tl) {
+                if (init_each_timelevel) {
                   
-                  // Advance times
-                  for (int m=0; m<maps; ++m) {
-                    vtt.AT(m)->advance_time (reflevel, mglevel);
-                  }
-                  CycleTimeLevels (cctkGH);
-                  cctkGH->cctk_time +=
-                    cctkGH->cctk_delta_time / cctkGH->cctk_timefac;
+                  BEGIN_TIMELEVEL_LOOP (cctkGH) {
+                    
+                    // Postregrid
+                    ScheduleTraverse (where, "CCTK_POSTREGRIDINITIAL", cctkGH);
+                    
+                  } END_TIMELEVEL_LOOP;
+                  
+                } else {        // not init_each_timelevel
+                  
+                  assert (do_allow_past_timelevels);
+                  do_allow_past_timelevels = false;
                   
                   // Postregrid
                   ScheduleTraverse (where, "CCTK_POSTREGRIDINITIAL", cctkGH);
                   
-                } // for tl
-                cctkGH->cctk_time = old_cctk_time;
-                
-                do_allow_past_timelevels = old_do_allow_past_timelevels;
+                  do_allow_past_timelevels = true;
+                  
+                } // not init_each_timelevel
                 
                 EndTimingLevel (cctkGH);
               } LEAVE_LEVEL_MODE;
@@ -1156,274 +1040,107 @@ namespace Carpet {
   
   
   
-  // Use Scott Hawley's algorithm to get two extra timelevels of data
-  
-  static void initialise_3tl_advance_time (cGH * const cctkGH);
-  static void initialise_3tl_evolve_Ia (cGH * const cctkGH);
   static void initialise_3tl_flip_timelevels (cGH * const cctkGH);
-  static void initialise_3tl_evolve_Ib (cGH * const cctkGH);
-  static void initialise_3tl_evolve_IIb (cGH * const cctkGH);
-  static void initialise_3tl_advance_time_2 (cGH * const cctkGH);
-  static void initialise_3tl_evolve_Ic (cGH * const cctkGH);
-  static void initialise_3tl_reset_time (cGH * const cctkGH);
+  static void initialise_3tl_evolve (cGH * const cctkGH);
+  static void initialise_3tl_recycle (cGH * const cctkGH);
+  
+  
   
   void
   Initialise3tl (cGH * const cctkGH)
   {
     DECLARE_CCTK_PARAMETERS;
     
-    Waypoint ("Initialising three timelevels");
+    Waypoint ("Initialising three timelevels:");
     
-    // TODO: ensure that there are 3 timelevels
-    if (not (prolongation_order_time == 2)) {
-      CCTK_WARN (CCTK_WARN_ABORT,
-                 "The 3 timelevel initialisation scheme (init_3_timelevels=yes) requires 3 timelevels and a temporal prolongation order of 2 (prolongation_order_time=2)");
-    }
-    assert (prolongation_order_time == 2);
+    initialise_3tl_flip_timelevels (cctkGH);
+    initialise_3tl_evolve (cctkGH);
+    // TODO: May want to restrict here if possible (i.e. if the time
+    // refinement factor is one)
+    initialise_3tl_recycle (cctkGH);
+    initialise_3tl_flip_timelevels (cctkGH);
     
-    for (int rl=0; rl<int(timereffacts.size()); ++rl) {
-      if (not (timereffacts.AT(rl) == ipow (2, rl))) {
-        CCTK_WARN (CCTK_WARN_ABORT,
-                   "The 3 timelevel initialisation scheme (init_3_timelevels=yes) requires temporal refinement factors of 2 for all refinement levels (time_refinement_factors[rl]=pow(2,rl))");
-      }
-      assert (timereffacts.AT(rl) == ipow (2, rl));
-    }
+    Waypoint ("Finished initialising three timelevels");
+  }
+  
+  void
+  initialise_3tl_flip_timelevels (cGH * const cctkGH)
+  {
+    Waypoint ("Initialise3TL::Flip");
+    
+    delta_time *= -1;
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
-      BEGIN_REFLEVEL_LOOP(cctkGH) {
-        BeginTimingLevel (cctkGH);
+      BEGIN_REFLEVEL_LOOP (cctkGH) {
         
-        do_early_global_mode = reflevel==0;
-        do_late_global_mode = reflevel==reflevels-1;
-        do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
-        do_late_meta_mode = do_late_global_mode and mglevel==0;
-        do_global_mode = do_early_global_mode;
-        do_meta_mode = do_early_meta_mode;
+        FlipTimeLevels (cctkGH);
         
-        initialise_3tl_advance_time (cctkGH);
-        initialise_3tl_evolve_Ia (cctkGH);
-        initialise_3tl_flip_timelevels (cctkGH);
-        initialise_3tl_evolve_Ib (cctkGH);
-        initialise_3tl_flip_timelevels (cctkGH);
-        
-        EndTimingLevel (cctkGH);
       } END_REFLEVEL_LOOP;
     } END_MGLEVEL_LOOP;
-    
-    Waypoint ("Hourglass structure in place");
-    
-    initialise_3tl_flip_timelevels (cctkGH);
+  }
+  
+  void
+  initialise_3tl_evolve (cGH * const cctkGH)
+  {
+    char const * const where = "Initialise3TL::Evolve";
+    static Timer timer (where);
+    timer.start();
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
-      BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) {
+      BEGIN_REFLEVEL_LOOP(cctkGH) {
         BeginTimingLevel (cctkGH);
         
-        do_early_global_mode = reflevel==reflevels-1;
-        do_late_global_mode = reflevel==0;
+        do_early_global_mode = reflevel==0;
+        do_late_global_mode = reflevel==reflevels-1;
         do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
         do_late_meta_mode = do_late_global_mode and mglevel==0;
         do_global_mode = do_early_global_mode;
         do_meta_mode = do_early_meta_mode;
         
-        initialise_3tl_evolve_IIb (cctkGH);
-        initialise_3tl_advance_time_2 (cctkGH);
-        initialise_3tl_evolve_Ic (cctkGH);
+        Waypoint ("Initialisation 3TL evolution",
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
         
-        EndTimingLevel (cctkGH);
-      } END_REVERSE_REFLEVEL_LOOP;
-    } END_MGLEVEL_LOOP;
-    
-    initialise_3tl_flip_timelevels (cctkGH);
-    
-    BEGIN_MGLEVEL_LOOP(cctkGH) {
-      BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) {
-        BeginTimingLevel (cctkGH);
+        CycleTimeLevels (cctkGH);
         
-        do_early_global_mode = reflevel==reflevels-1;
-        do_late_global_mode = reflevel==0;
-        do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
-        do_late_meta_mode = do_late_global_mode and mglevel==0;
-        do_global_mode = do_early_global_mode;
-        do_meta_mode = do_early_meta_mode;
+        CalculateChecksums (cctkGH, allbutcurrenttime);
+        Poison (cctkGH, currenttimebutnotifonly);
+        
+        // Evolve
+        ScheduleTraverse (where, "CCTK_PRESTEP", cctkGH);
+        ScheduleTraverse (where, "CCTK_EVOL", cctkGH);
         
-        initialise_3tl_reset_time (cctkGH);
+        PoisonCheck (cctkGH, currenttime);
         
         EndTimingLevel (cctkGH);
-      } END_REVERSE_REFLEVEL_LOOP;
+      } END_REFLEVEL_LOOP;
     } END_MGLEVEL_LOOP;
     
-    Waypoint ("Finished initialising three timelevels");
-  }
-  
-  
-  
-  void
-  initialise_3tl_advance_time (cGH * const cctkGH)
-  {
-    Waypoint ("Advancing time");
-    
-    cctkGH->cctk_time
-      = global_time + delta_time * mglevelfact / timereflevelfact;
-    for (int m=0; m<maps; ++m) {
-      vtt.AT(m)->advance_time (reflevel, mglevel);
-    }
-    
-    CycleTimeLevels (cctkGH);
-  }
-  
-  void
-  initialise_3tl_evolve_Ia (cGH * const cctkGH)
-  {
-    char const * const where = "Initialise3TL::EvolveIa";
-    static Timer timer (where);
-    timer.start();
-    
-    Waypoint ("Initialisation 3TL evolution I (a) (forwards) at iteration"
-              " %d time %g%s%s",
-              cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-              (do_global_mode ? " (global)" : ""),
-              (do_meta_mode ? " (meta)" : ""));
-    
-    CalculateChecksums (cctkGH, allbutcurrenttime);
-    Poison (cctkGH, currenttimebutnotifonly);
-    
-    // Evolve forward
-    ScheduleTraverse (where, "CCTK_PRESTEP", cctkGH);
-    ScheduleTraverse (where, "CCTK_EVOL", cctkGH);
-    
-    PoisonCheck (cctkGH, currenttime);
-    
-    timer.stop();
-  }
-  
-  void
-  initialise_3tl_flip_timelevels (cGH * const cctkGH)
-  {
-    Waypoint ("Flipping timelevels");
-    
-    BEGIN_META_MODE(cctkGH) {
-      
-      delta_time *= -1;
-      
-      BEGIN_MGLEVEL_LOOP(cctkGH) {
-        BEGIN_REFLEVEL_LOOP (cctkGH) {
-          
-          cctkGH->cctk_time
-            = global_time + delta_time * mglevelfact / timereflevelfact;
-          
-          FlipTimeLevels (cctkGH);
-          
-        } END_REFLEVEL_LOOP;
-      } END_MGLEVEL_LOOP;
-    } END_META_MODE;
-  }
-  
-  void
-  initialise_3tl_evolve_Ib (cGH * const cctkGH)
-  {
-    char const * const where = "Initialise3TL::EvolveIb";
-    static Timer timer (where);
-    timer.start();
-    
-    Waypoint ("Initialisation 3TL evolution I (b) (backwards) at iteration"
-              " %d time %g%s%s",
-              cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-              (do_global_mode ? " (global)" : ""),
-              (do_meta_mode ? " (meta)" : ""));
-    
-    // Checking
-    CalculateChecksums (cctkGH, allbutcurrenttime);
-    Poison (cctkGH, currenttimebutnotifonly);
-    
-    // Evolve backward
-    ScheduleTraverse (where, "CCTK_PRESTEP", cctkGH);
-    ScheduleTraverse (where, "CCTK_EVOL", cctkGH);
-    
-    // Checking
-    PoisonCheck (cctkGH, alltimes);
-    
     timer.stop();
   }
   
-  // Evolve backwards one more timestep
-  // Starting with the finest level and proceeding to the coarsest
   void
-  initialise_3tl_evolve_IIb (cGH * const cctkGH)
+  initialise_3tl_recycle (cGH * const cctkGH)
   {
-    char const * const where = "Initialise3TL::EvolveIIb";
+    char const * const where = "Initialise3TL::Recycle";
     static Timer timer (where);
     timer.start();
     
-    Waypoint ("Initialisation 3TL evolution II (b) (backwards) at iteration"
-              " %d time %g%s%s",
-              cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-              (do_global_mode ? " (global)" : ""),
-              (do_meta_mode ? " (meta)" : ""));
-    
-    Restrict (cctkGH);
-    
-    if (reflevel < reflevels-1) {
-      ScheduleTraverse (where, "CCTK_POSTRESTRICT", cctkGH);
-    }
-    
-    ScheduleTraverse (where, "CCTK_POSTSTEP", cctkGH);
-    
-    PoisonCheck (cctkGH, alltimes);
-    
-    timer.stop();
-  }
-  
-  void
-  initialise_3tl_advance_time_2 (cGH * const cctkGH)
-  {
-    Waypoint ("Advancing time");
-    
-    cctkGH->cctk_time
-      = global_time + 2 * delta_time * mglevelfact / timereflevelfact;
-    for (int m=0; m<maps; ++m) {
-      vtt.AT(m)->advance_time (reflevel, mglevel);
-    }
-    
-    CycleTimeLevels (cctkGH);
-  }
-  
-  void
-  initialise_3tl_evolve_Ic (cGH * const cctkGH)
-  {
-    char const * const where = "Initialise3TL::EvolveIc";
-    static Timer timer (where);
-    timer.start();
-    
-    Waypoint ("Initialisation 3TL evolution I (c) (backwards) at iteration"
-              " %d time %g%s%s",
-              cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
-              (do_global_mode ? " (global)" : ""),
-              (do_meta_mode ? " (meta)" : ""));
-    
-    CalculateChecksums (cctkGH, allbutcurrenttime);
-    Poison (cctkGH, currenttimebutnotifonly);
-    
-    // Evolve backward
-    ScheduleTraverse (where, "CCTK_PRESTEP", cctkGH);
-    ScheduleTraverse (where, "CCTK_EVOL", cctkGH);
-    ScheduleTraverse (where, "CCTK_POSTSTEP", cctkGH);
-    
-    PoisonCheck (cctkGH, alltimes);
+    BEGIN_MGLEVEL_LOOP(cctkGH) {
+      BEGIN_REFLEVEL_LOOP(cctkGH) {
+        BeginTimingLevel (cctkGH);
+        
+        Waypoint ("Initialisation 3TL recycling");
+        
+        UncycleTimeLevels (cctkGH);
+        
+        EndTimingLevel (cctkGH);
+      } END_REFLEVEL_LOOP;
+    } END_MGLEVEL_LOOP;
     
     timer.stop();
   }
   
-  void
-  initialise_3tl_reset_time (cGH * const cctkGH)
-  {
-    Waypoint ("Resetting time");
-    
-    cctkGH->cctk_time = global_time;
-    for (int m=0; m<maps; ++m) {
-      vtt.AT(m)->set_time (reflevel, mglevel, 0);
-    }
-  }
-  
   
   
   //////////////////////////////////////////////////////////////////////////////
@@ -1442,7 +1159,7 @@ namespace Carpet {
       streamsize const oldprecision = cout.precision();
       cout.precision (17);
       cout << "   global_time: " << global_time << endl
-           << "   leveltimes: " << leveltimes << endl
+        // << "   leveltimes: " << leveltimes << endl
            << "   delta_time: " << delta_time << endl;
       cout.precision (oldprecision);
     }
diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc
index 598d48a14..94b1f90f7 100644
--- a/Carpet/Carpet/src/Recompose.cc
+++ b/Carpet/Carpet/src/Recompose.cc
@@ -356,10 +356,10 @@ namespace Carpet {
     }
     
     // Set new level times
-    for (int ml=0; ml<mglevels; ++ml) {
-      leveltimes.AT(ml).resize
-        (reflevels, leveltimes.AT(ml).AT(oldreflevels-1));
-    }
+    // for (int ml=0; ml<mglevels; ++ml) {
+    //   leveltimes.AT(ml).resize
+    //     (reflevels, leveltimes.AT(ml).AT(oldreflevels-1));
+    // }
     
     ++ regridding_epoch;
     
diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc
index bad055641..8ede1ebcd 100644
--- a/Carpet/Carpet/src/Restrict.cc
+++ b/Carpet/Carpet/src/Restrict.cc
@@ -62,28 +62,16 @@ namespace Carpet {
   static void RestrictGroups (const cGH* cctkGH, const vector<int>& groups) {
     DECLARE_CCTK_PARAMETERS;
 
-    const int tl = 0;
-
     for (comm_state state; not state.done(); state.step()) {
       for (int group = 0; group < (int)groups.size(); ++group) {
         const int g = groups.AT(group);
+        const int active_tl = CCTK_ActiveTimeLevelsGI (cctkGH, g);
+        assert (active_tl>=0);
+        const int tl = active_tl > 1 ? timelevel : 0;
         for (int m=0; m<(int)arrdata.AT(g).size(); ++m) {
-
-          // use background time here (which may not be modified
-          // by the user)
-          const CCTK_REAL time = vtt.AT(m)->time (tl, reflevel, mglevel);
-
-          const CCTK_REAL time1 = vtt.AT(m)->time (0, reflevel, mglevel);
-          const CCTK_REAL time2 =
-            (cctkGH->cctk_time - cctk_initial_time) / delta_time;
-          const CCTK_REAL time0 =
-            abs(time1) + abs(time2) + abs(cctkGH->cctk_delta_time);
-          const CCTK_REAL eps = 1.0e-12;
-          assert (abs(time1 - time2) <= eps * time0);
-
           for (int v = 0; v < (int)arrdata.AT(g).AT(m).data.size(); ++v) {
             ggf *const gv = arrdata.AT(g).AT(m).data.AT(v);
-            gv->ref_restrict_all (state, tl, reflevel, mglevel, time);
+            gv->ref_restrict_all (state, tl, reflevel, mglevel);
           }
         }
       } // loop over groups
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index af5f13422..4961cec3b 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -54,8 +54,7 @@ namespace Carpet {
   allocate_data_hierarchy (cGH const * cctkGH,
                            int m);
   static void
-  allocate_time_hierarchy (cGH const * cctkGH,
-                           int m);
+  allocate_time_hierarchy (cGH const * cctkGH);
   static void
   setup_grid_hierarchy (cGH const * cctkGH);
   static void
@@ -272,6 +271,9 @@ namespace Carpet {
     cctkGH->cctk_mode = CCTK_MODE_META;
 #endif
     
+    timelevels = prolongation_order_time + 1;
+    timelevel  = 0;
+    
     // Say hello
     Waypoint ("Setting up the grid hierarchy");
     
@@ -448,9 +450,10 @@ namespace Carpet {
       // Allocate hierarchies
       allocate_grid_hierarchy (cctkGH, m);
       allocate_data_hierarchy (cctkGH, m);
-      allocate_time_hierarchy (cctkGH, m);
       
     } // for m
+    
+    allocate_time_hierarchy (cctkGH);
   }
   
   
@@ -601,15 +604,14 @@ namespace Carpet {
   
   
   void
-  allocate_time_hierarchy (cGH const * const cctkGH,
-                           int const m)
+  allocate_time_hierarchy (cGH const * const cctkGH)
   {
     DECLARE_CCTK_PARAMETERS;
     
-    vtt.resize (maps);
-    vtt.AT(m) = new th (* vhh.AT(m),
-                        timereffacts,
-                        1.0);
+    // We are using the gh of the first map.  This works because all
+    // maps have the same number of refinement levels.
+    tt = new th (* vhh.AT(0), timelevels, timereffacts,
+                 time_interpolation_during_regridding);
   }
   
   
@@ -774,7 +776,7 @@ namespace Carpet {
         for (int m=0; m<maps; ++m) {
           arrdata.AT(group).AT(m).hh = vhh.AT(m);
           arrdata.AT(group).AT(m).dd = vdd.AT(m);
-          arrdata.AT(group).AT(m).tt = vtt.AT(m);
+          arrdata.AT(group).AT(m).tt = tt;
         }
         
         break;
@@ -865,9 +867,9 @@ namespace Carpet {
       new dh (*arrdata.AT(group).AT(m).hh,
               ghosts, buffers, my_prolongation_orders_space);
     
-    CCTK_REAL const basedelta = 1.0;
     arrdata.AT(group).AT(m).tt =
-      new th (*arrdata.AT(group).AT(m).hh, grouptimereffacts, basedelta);
+      new th (*arrdata.AT(group).AT(m).hh, timelevels, grouptimereffacts,
+              time_interpolation_during_regridding);
   }
   
   
@@ -1053,11 +1055,11 @@ namespace Carpet {
   void
   set_state (cGH * const cctkGH)
   {
-    // Allocate level times
-    leveltimes.resize (mglevels);
-    for (int ml=0; ml<mglevels; ++ml) {
-      leveltimes.AT(ml).resize (1);
-    }
+    // // Allocate level times
+    // leveltimes.resize (mglevels);
+    // for (int ml=0; ml<mglevels; ++ml) {
+    //   leveltimes.AT(ml).resize (1);
+    // }
     
     // Allocate orgin and spacings
     origin_space.resize (maps);
diff --git a/Carpet/Carpet/src/Timing.cc b/Carpet/Carpet/src/Timing.cc
index 51ecc45e8..43c230588 100644
--- a/Carpet/Carpet/src/Timing.cc
+++ b/Carpet/Carpet/src/Timing.cc
@@ -585,9 +585,9 @@ namespace Carpet {
          << "   Grid hierarchy:" << eol;
     for (int m = 0; m < Carpet::maps; ++ m) {
       cout << "   gh[" << m << "]: " << PRINTMEM(*vhh.AT(m)) << eol
-           << "   dh[" << m << "]: " << PRINTMEM(*vdd.AT(m)) << eol
-           << "   th[" << m << "]: " << PRINTMEM(*vtt.AT(m)) << eol;
+           << "   dh[" << m << "]: " << PRINTMEM(*vdd.AT(m)) << eol;
     }
+    cout << "   th: " << PRINTMEM(*tt) << eol;
 #if 0
     for (int g = 0; g < (int)arrdata.size(); ++ g) {
       if (CCTK_GroupTypeI(g) != CCTK_GF) {
diff --git a/Carpet/Carpet/src/carpet.hh b/Carpet/Carpet/src/carpet.hh
index bbcb4296a..3ca8535fe 100644
--- a/Carpet/Carpet/src/carpet.hh
+++ b/Carpet/Carpet/src/carpet.hh
@@ -38,8 +38,9 @@ namespace Carpet {
   // Other functions
   bool Regrid (cGH const * cctkGH, bool force_recompose, bool do_init);
   
-  void CycleTimeLevels (const cGH* cgh);
-  void FlipTimeLevels (const cGH* cgh);
+  void CycleTimeLevels (cGH* cgh);
+  void UncycleTimeLevels (cGH* cgh);
+  void FlipTimeLevels (cGH* cgh);
   void FillTimeLevels (const cGH* cgh);
   void SyncGroups (const cGH* cgh, const vector<int>& groups);
   int  SyncProlongateGroups (const cGH* cgh, const vector<int>& groups);
diff --git a/Carpet/Carpet/src/modes.cc b/Carpet/Carpet/src/modes.cc
index 68db7beaa..038815393 100644
--- a/Carpet/Carpet/src/modes.cc
+++ b/Carpet/Carpet/src/modes.cc
@@ -330,12 +330,17 @@ namespace Carpet {
     }
     
     // Set current time
-    assert (mglevel>=0 and mglevel<(int)leveltimes.size());
-    assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
+    // assert (mglevel>=0 and mglevel<(int)leveltimes.size());
+    // assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
+    // if (not adaptive_stepsize) {
+    //   cctkGH->cctk_time = leveltimes.AT(mglevel).AT(reflevel);
+    // } else {
+    //   leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
+    // }
     if (not adaptive_stepsize) {
-      cctkGH->cctk_time = leveltimes.AT(mglevel).AT(reflevel);
+      cctkGH->cctk_time = tt->get_time(mglevel, reflevel, timelevel);
     } else {
-      leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
+      tt->set_time (mglevel, reflevel, timelevel, cctkGH->cctk_time);
     }
     
     assert (is_level_mode());
@@ -354,9 +359,15 @@ namespace Carpet {
     CCTK_INT const deadbeef = get_deadbeef();
     
     // Save and unset current time
-    assert (mglevel>=0 and mglevel<(int)leveltimes.size());
-    assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
-    leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
+    // assert (mglevel>=0 and mglevel<(int)leveltimes.size());
+    // assert (reflevel>=0 and reflevel<(int)leveltimes.AT(mglevel).size());
+    // leveltimes.AT(mglevel).AT(reflevel) = cctkGH->cctk_time;
+    // if (not adaptive_stepsize) {
+    //   cctkGH->cctk_time = global_time;
+    // } else {
+    //   global_time = cctkGH->cctk_time;
+    // }
+    tt->set_time (mglevel, reflevel, timelevel, cctkGH->cctk_time);
     if (not adaptive_stepsize) {
       cctkGH->cctk_time = global_time;
     } else {
@@ -606,10 +617,30 @@ namespace Carpet {
               assert (firstvar>=0);
               const int max_tl = CCTK_MaxTimeLevelsGI (group);
               assert (max_tl>=0);
-              const int active_tl = CCTK_ActiveTimeLevelsGI (cctkGH, group);
+              const int active_tl =
+                groupdata.AT(group).activetimelevels.AT(mglevel).AT(reflevel);
               assert (active_tl>=0 and active_tl<=max_tl);
-              const int available_tl =
-                do_allow_past_timelevels ? active_tl : min (1, active_tl);
+              int available_tl;
+              int tl_offset;
+              if (active_tl == 0) {
+                // gropu has no storage
+                available_tl = active_tl;
+                tl_offset = 0;
+              } else if (do_allow_past_timelevels) {
+                // regular case; all timelevels are accessible
+                available_tl = active_tl;
+                tl_offset = 0;
+              } else {
+                // only one timelevel is accessible
+                available_tl = 1;
+                if (timelevel < active_tl) {
+                  // timelevel "timelevel" exists
+                  tl_offset = timelevel;
+                } else {
+                  // timelevel "timelevel" does not exist
+                  tl_offset = active_tl - 1;
+                }
+              }
               
               // assert (vhh.AT(map)->is_local(reflevel,component));
               
@@ -620,7 +651,7 @@ namespace Carpet {
                 for (int tl=0; tl<max_tl; ++tl) {
                   if (ff and tl<available_tl) {
                     gdata * const data =
-                      (*ff) (tl, reflevel, local_component, mglevel);
+                      (*ff) (tl_offset+tl, reflevel, local_component, mglevel);
                     assert (data);
                     cctkGH->data[firstvar+var][tl] = data->storage();
                   } else {
diff --git a/Carpet/Carpet/src/modes.hh b/Carpet/Carpet/src/modes.hh
index c2631d952..868c54172 100644
--- a/Carpet/Carpet/src/modes.hh
+++ b/Carpet/Carpet/src/modes.hh
@@ -247,6 +247,25 @@ namespace Carpet {
     assert (local_component_loop_);             \
     local_component_loop_ = false;              \
   } while (false)
+
+#define BEGIN_TIMELEVEL_LOOP(cctkGH)                                    \
+  do {                                                                  \
+    bool timelevel_loop_ = true;                                        \
+    assert (do_allow_past_timelevels);                                  \
+    do_allow_past_timelevels = false;                                   \
+    assert (timelevel == 0);                                            \
+    for (timelevel = 0; timelevel < timelevels; ++ timelevel) {         \
+      cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel);  \
+      {
+#define END_TIMELEVEL_LOOP                                              \
+      }                                                                 \
+    }                                                                   \
+    assert (timelevel_loop_);                                           \
+    timelevel_loop_ = false;                                            \
+    timelevel = 0;                                                      \
+    cctkGH->cctk_time = tt->get_time (mglevel, reflevel, timelevel);    \
+    do_allow_past_timelevels = true;                                    \
+  } while (false)
   
   
   
diff --git a/Carpet/Carpet/src/variables.cc b/Carpet/Carpet/src/variables.cc
index 0198a6018..c9d82cb16 100644
--- a/Carpet/Carpet/src/variables.cc
+++ b/Carpet/Carpet/src/variables.cc
@@ -42,6 +42,9 @@ namespace Carpet {
   // Maps
   int maps;
   
+  // Timelevels
+  int timelevels;
+  
   
   
   // Current position on the grid hierarchy
@@ -51,6 +54,7 @@ namespace Carpet {
   int map;
   int component;
   int local_component;
+  int timelevel;
   
   // Current refinement factors
   int timereflevelfact;
@@ -68,7 +72,7 @@ namespace Carpet {
   
   // Times and spaces on the refinement levels
   CCTK_REAL global_time;
-  vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
+  // vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
   CCTK_REAL delta_time;
   
   vector<vector<vect<CCTK_REAL,dim> > > origin_space; // [map][mglevel]
@@ -109,7 +113,7 @@ namespace Carpet {
   // The grid hierarchy
   vector<gh*> vhh;              // [map]
   vector<dh*> vdd;              // [map]
-  vector<th*> vtt;              // [map]
+  th* tt;
   int regridding_epoch;
   
   // Data for the groups
@@ -122,6 +126,6 @@ namespace Carpet {
   
   // MPI Communicators
   MPI_Comm comm_universe = MPI_COMM_NULL;
-  MPI_Comm comm_world = MPI_COMM_NULL;
+  MPI_Comm comm_world    = MPI_COMM_NULL;
   
 } // namespace Carpet
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index 2746ec295..61ac33cc2 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -72,6 +72,9 @@ namespace Carpet {
   // Maps
   extern int maps;
   
+  // Timelevels
+  extern int timelevels;
+  
   
   
   // Current position on the grid hierarchy
@@ -81,6 +84,7 @@ namespace Carpet {
   extern int map;
   extern int component;
   extern int local_component;   // -1 for non-local
+  extern int timelevel;
   
   // Current refinement factors
   extern int timereflevelfact;
@@ -98,7 +102,7 @@ namespace Carpet {
   
   // Times and spaces on the refinement levels
   extern CCTK_REAL global_time;
-  extern vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
+  // extern vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel]
   extern CCTK_REAL delta_time;
   
   extern vector<vector<vect<CCTK_REAL,dim> > > origin_space; // [map][mglevel]
@@ -146,7 +150,7 @@ namespace Carpet {
   // The grid hierarchy
   extern vector<gh*> vhh;       // [map]
   extern vector<dh*> vdd;       // [map]
-  extern vector<th*> vtt;       // [map]
+  extern th* tt;
   extern int regridding_epoch;
   
   // Data for the groups