Carpet: Add new modes "global_early" and "global_late"

Add new modes meta_early, meta_late, global_early, and global_late, which
execute like meta mode or global mode, but execut either as early or as
late as possible in the schedule.

7 files changed, 181 insertions, 63 deletions

diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl
index 94b4c7e4e..f311c6d96 100644
--- a/Carpet/Carpet/param.ccl
+++ b/Carpet/Carpet/param.ccl
@@ -499,9 +499,3 @@ BOOLEAN init_3_timelevels "Set up 3 timelevels of initial data" STEERABLE=always
 BOOLEAN adaptive_stepsize "Allow adaptive timestep sizes"
 {
 } "no"
-
-
-
-BOOLEAN global_mode_on_finest_grid "Run global mode routines on the finest grin" STEERABLE=always
-{
-} "no"
diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc
index ea64fa115..f5c79dbf1 100644
--- a/Carpet/Carpet/src/CallFunction.cc
+++ b/Carpet/Carpet/src/CallFunction.cc
@@ -46,7 +46,11 @@ namespace Carpet {
     cGH * cctkGH = static_cast<cGH *> (data);
     
     assert (int (not not attribute->meta) +
+            int (not not attribute->meta_early) +
+            int (not not attribute->meta_late) +
             int (not not attribute->global) +
+            int (not not attribute->global_early) +
+            int (not not attribute->global_late) +
             int (not not attribute->level) +
             int (not not attribute->singlemap) +
             int (not not attribute->local)
@@ -58,10 +62,16 @@ namespace Carpet {
             not not attribute->loop_local
             <= 1);
     
-    if (attribute->meta or is_meta_mode()) {
+    if (attribute->meta or attribute->meta_early or attribute->meta_late or
+        is_meta_mode())
+    {
       // Convtest operation
       
-      if (do_meta_mode) {
+      if ((attribute->meta and do_meta_mode) or
+          (attribute->meta_early and do_early_meta_mode) or
+          (attribute->meta_late and do_late_meta_mode) or
+          is_meta_mode())
+      {
         if (attribute->loop_local) {
           BEGIN_META_MODE(cctkGH) {
             BEGIN_MGLEVEL_LOOP(cctkGH) {
@@ -136,12 +146,16 @@ namespace Carpet {
         }
       }
       
-    } else if (attribute->global or is_global_mode()) {
+    } else if (attribute->global or attribute->global_early or
+               attribute->global_late or is_global_mode())
+    {
       // Global operation: call once
       
-      assert (not attribute->loop_meta);
-      
-      if (do_global_mode) {
+      if ((attribute->global and do_global_mode) or
+          (attribute->global_early and do_early_global_mode) or
+          (attribute->global_late and do_late_global_mode) or
+          is_global_mode())
+      {
         if (attribute->loop_local) {
           BEGIN_GLOBAL_MODE(cctkGH) {
             BEGIN_REFLEVEL_LOOP(cctkGH) {
@@ -198,9 +212,6 @@ namespace Carpet {
     } else if (attribute->level) {
       // Level operation: call once per refinement level
       
-      assert (not attribute->loop_meta);
-      assert (not attribute->loop_global);
-      
       if (attribute->loop_local) {
         BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
           BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
@@ -227,10 +238,6 @@ namespace Carpet {
     } else if (attribute->singlemap) {
       // Single map operation: call once per refinement level and map
       
-      assert (not attribute->loop_meta);
-      assert (not attribute->loop_global);
-      assert (not attribute->loop_level);
-      
       if (attribute->loop_local) {
         BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
           BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
@@ -253,11 +260,6 @@ namespace Carpet {
     } else {
       // Local operation: call once per component
       
-      assert (not attribute->loop_meta);
-      assert (not attribute->loop_global);
-      assert (not attribute->loop_level);
-      assert (not attribute->loop_singlemap);
-      
       BEGIN_MAP_LOOP(cctkGH, CCTK_GF) {
         BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) {
           CallScheduledFunction
diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc
index ec3eeb32e..f6ea07eab 100644
--- a/Carpet/Carpet/src/Evolve.cc
+++ b/Carpet/Carpet/src/Evolve.cc
@@ -247,9 +247,17 @@ namespace Carpet {
     assert (is_level_mode());
     
     bool const old_do_global_mode = do_global_mode;
+    bool const old_do_early_global_mode = do_early_global_mode;
+    bool const old_do_late_global_mode = do_late_global_mode;
     bool const old_do_meta_mode = do_meta_mode;
+    bool const old_do_early_meta_mode = do_early_meta_mode;
+    bool const old_do_late_meta_mode = do_late_meta_mode;
     do_global_mode = true;
+    do_early_global_mode = true;
+    do_late_global_mode = true;
     do_meta_mode = true;
+    do_early_meta_mode = true;
+    do_late_meta_mode = true;
     
     Waypoint ("Preregrid at iteration %d time %g%s%s",
               cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -278,8 +286,12 @@ namespace Carpet {
           if (did_recompose or rl == reflevels - 1) {
             BEGIN_MGLEVEL_LOOP (cctkGH) {
               ENTER_LEVEL_MODE (cctkGH, rl) {
-                do_global_mode = reflevel == reflevels - 1;
-                do_meta_mode = do_global_mode and mglevel==mglevels-1;
+                do_early_global_mode = reflevel==0;
+                do_late_global_mode = reflevel==reflevels-1;
+                do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+                do_late_meta_mode = do_late_global_mode and mglevel==0;
+                do_global_mode = do_late_global_mode;
+                do_meta_mode = do_late_meta_mode;
                 
                 Waypoint ("Postregrid at iteration %d time %g%s%s",
                           cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -338,7 +350,11 @@ namespace Carpet {
     } // if did_regrid
     
     do_global_mode = old_do_global_mode;
+    do_early_global_mode = old_do_early_global_mode;
+    do_late_global_mode = old_do_late_global_mode;
     do_meta_mode = old_do_meta_mode;
+    do_early_meta_mode = old_do_early_meta_mode;
+    do_late_meta_mode = old_do_late_meta_mode;
     
     timer.stop();
   }
@@ -366,11 +382,12 @@ namespace Carpet {
           ENTER_GLOBAL_MODE (cctkGH, ml) {
             ENTER_LEVEL_MODE (cctkGH, rl) {
               
-              do_global_mode
-                = (global_mode_on_finest_grid
-                   ? reflevel == reflevels - 1
-                   : not have_done_global_mode);
-              do_meta_mode = do_global_mode and mglevel==mglevels-1;
+              do_early_global_mode = not have_done_global_mode;
+              do_late_global_mode = reflevel==reflevels-1;
+              do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+              do_late_meta_mode = do_late_global_mode and mglevel==0;
+              do_global_mode = do_early_global_mode;
+              do_meta_mode = do_early_meta_mode;
               assert (not (have_done_global_mode and do_global_mode));
               have_done_global_mode |= do_global_mode;
               have_done_anything = true;
@@ -498,8 +515,12 @@ namespace Carpet {
           ENTER_GLOBAL_MODE (cctkGH, ml) {
             ENTER_LEVEL_MODE (cctkGH, rl) {
               
-              do_global_mode = reflevel == reflevels - 1;
-              do_meta_mode = do_global_mode and mglevel==mglevels-1;
+              do_early_global_mode = not have_done_global_mode;
+              do_late_global_mode = reflevel==reflevels-1;
+              do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+              do_late_meta_mode = do_late_global_mode and mglevel==0;
+              do_global_mode = do_late_global_mode;
+              do_meta_mode = do_global_mode and do_late_meta_mode;
               assert (not (have_done_global_mode and do_global_mode));
               have_done_global_mode |= do_global_mode;
               have_done_anything = true;
@@ -526,7 +547,7 @@ namespace Carpet {
               // Analysis
               ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH);
               
-              if (do_global_mode) {
+              if (do_late_global_mode) {
                 // Timing statistics
                 UpdateTimingStats (cctkGH);
               }
@@ -537,7 +558,7 @@ namespace Carpet {
               // Checking
               CheckChecksums (cctkGH, alltimes);
               
-              if (do_global_mode) {
+              if (do_late_global_mode) {
                 // Timing statistics
                 PrintTimingStats (cctkGH);
               }
diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc
index 08a00aa84..75b6c56ad 100644
--- a/Carpet/Carpet/src/Initialise.cc
+++ b/Carpet/Carpet/src/Initialise.cc
@@ -56,7 +56,11 @@ namespace Carpet {
     CCTKi_AddGH (fc, convlev, cctkGH);
     
     do_global_mode = true;
+    do_early_global_mode = true;
+    do_late_global_mode = true;
     do_meta_mode = true;
+    do_early_meta_mode = true;
+    do_late_meta_mode = true;
     global_time = cctk_initial_time;
     delta_time = 1.0;
     for (int ml = 0; ml < mglevel; ++ ml) {
@@ -140,8 +144,12 @@ namespace Carpet {
     timer.start();
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
+      do_early_global_mode = true;
+      do_late_global_mode = true;
       do_global_mode = true;
-      do_meta_mode = mglevel==mglevels-1; // on first iteration, coarsest grid
+      do_early_meta_mode = mglevel==mglevels-1;
+      do_late_meta_mode = mglevel==0;
+      do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid
       
       // Checking
       {
@@ -179,8 +187,12 @@ namespace Carpet {
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cctkGH) {
         ENTER_LEVEL_MODE (cctkGH, rl) {
-          do_global_mode = reflevel==0; // on first iteration, coarsest grid
-          do_meta_mode = do_global_mode and mglevel==mglevels-1; // on first iteration, coarsest grid
+          do_early_global_mode = reflevel==0;
+          do_late_global_mode = reflevel==reflevels-1;
+          do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+          do_late_meta_mode = do_late_global_mode and mglevel==0;
+          do_global_mode = do_early_global_mode; // on first iteration, coarsest grid
+          do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid
           
           // cctkGH->cctk_time = global_time;
           
@@ -200,7 +212,7 @@ namespace Carpet {
           
           // Timing statistics
           // (do this here, after cctk_time has been recovered)
-          if (do_global_mode) {
+          if (do_early_global_mode) {
             InitTimingStats (cctkGH);
           }
           
@@ -233,8 +245,12 @@ namespace Carpet {
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cctkGH) {
         ENTER_LEVEL_MODE (cctkGH, rl) {
-          do_global_mode = reflevel==reflevels-1; // on last iteration, finest grid
-          do_meta_mode = do_global_mode and mglevel==mglevels-1; // on first iteration, coarsest grid
+          do_early_global_mode = reflevel==0;
+          do_late_global_mode = reflevel==reflevels-1;
+          do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+          do_late_meta_mode = do_late_global_mode and mglevel==0;
+          do_global_mode = do_early_global_mode; // on first iteration, coarsest grid
+          do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid
           
           Waypoint ("Recovering II at iteration %d time %g%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -287,8 +303,12 @@ namespace Carpet {
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cctkGH) {
         ENTER_LEVEL_MODE (cctkGH, rl) {
-          do_global_mode = reflevel==0; // on first iteration, coarsest grid
-          do_meta_mode = do_global_mode and mglevel==mglevels-1; // on first iteration, coarsest grid
+          do_early_global_mode = reflevel==0;
+          do_late_global_mode = reflevel==reflevels-1;
+          do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+          do_late_meta_mode = do_late_global_mode and mglevel==0;
+          do_global_mode = do_early_global_mode; // on first iteration, coarsest grid
+          do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid
           
           // cctkGH->cctk_time = global_time;
           
@@ -298,7 +318,7 @@ namespace Carpet {
                     (do_meta_mode ? " (meta)" : ""));
           
           // Timing statistics
-          if (do_global_mode) {
+          if (do_early_global_mode) {
             InitTimingStats (cctkGH);
           }
           
@@ -394,8 +414,12 @@ namespace Carpet {
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cctkGH) {
         ENTER_LEVEL_MODE (cctkGH, rl) {
-          do_global_mode = reflevel==reflevels-1; // on last iteration, finest grid
-          do_meta_mode = do_global_mode and mglevel==mglevels-1; // on first iteration, coarsest grid
+          do_early_global_mode = reflevel==0;
+          do_late_global_mode = reflevel==reflevels-1;
+          do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+          do_late_meta_mode = do_late_global_mode and mglevel==0;
+          do_global_mode = do_late_global_mode; // on last iteration, finest grid
+          do_meta_mode = do_late_meta_mode; // on last iteration, finest grid
           
           Waypoint ("Initialisation II at iteration %d time %g%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -431,8 +455,12 @@ namespace Carpet {
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cctkGH) {
         ENTER_LEVEL_MODE (cctkGH, rl) {
-          do_global_mode = reflevel==reflevels-1; // last iteration, finest grid
-          do_meta_mode = do_global_mode and mglevel==mglevels-1; // first iteration, coarsest grid
+          do_early_global_mode = reflevel==0;
+          do_late_global_mode = reflevel==reflevels-1;
+          do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+          do_late_meta_mode = do_late_global_mode and mglevel==0;
+          do_global_mode = do_late_global_mode; // on last iteration, finest grid
+          do_meta_mode = do_late_meta_mode; // on last iteration, finest grid
           
           Waypoint ("Initialisation III at iteration %d time %g%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -449,7 +477,7 @@ namespace Carpet {
             // Analysis
             ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH);
             
-            if (do_global_mode) {
+            if (do_late_global_mode) {
               // Timing statistics
               UpdateTimingStats (cctkGH);
             }
@@ -703,9 +731,17 @@ namespace Carpet {
     assert (is_level_mode());
     
     bool const old_do_global_mode = do_global_mode;
+    bool const old_do_early_global_mode = do_early_global_mode;
+    bool const old_do_late_global_mode = do_late_global_mode;
     bool const old_do_meta_mode = do_meta_mode;
+    bool const old_do_early_meta_mode = do_early_meta_mode;
+    bool const old_do_late_meta_mode = do_late_meta_mode;
     do_global_mode = true;
+    do_early_global_mode = true;
+    do_late_global_mode = true;
     do_meta_mode = true;
+    do_early_meta_mode = true;
+    do_late_meta_mode = true;
     
     // Preregrid
     Waypoint ("Preregrid at iteration %d time %g%s%s",
@@ -730,8 +766,12 @@ namespace Carpet {
           if (did_recompose) {
             BEGIN_MGLEVEL_LOOP (cctkGH) {
               ENTER_LEVEL_MODE (cctkGH, rl) {
-                do_global_mode = reflevel == reflevels - 1;
-                do_meta_mode = do_global_mode and mglevel==mglevels-1;
+                do_early_global_mode = reflevel==0;
+                do_late_global_mode = reflevel==reflevels-1;
+                do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+                do_late_meta_mode = do_late_global_mode and mglevel==0;
+                do_global_mode = do_late_global_mode;
+                do_meta_mode = do_late_meta_mode;
                 
                 Waypoint ("Postregrid at iteration %d time %g%s%s",
                           cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -790,7 +830,11 @@ namespace Carpet {
     } // if did_regrid
     
     do_global_mode = old_do_global_mode;
+    do_early_global_mode = old_do_early_global_mode;
+    do_late_global_mode = old_do_late_global_mode;
     do_meta_mode = old_do_meta_mode;
+    do_early_meta_mode = old_do_early_meta_mode;
+    do_late_meta_mode = old_do_late_meta_mode;
     
     timer.stop();
   }
@@ -809,9 +853,17 @@ namespace Carpet {
     assert (is_meta_mode());
     
     bool const old_do_global_mode = do_global_mode;
+    bool const old_do_early_global_mode = do_early_global_mode;
+    bool const old_do_late_global_mode = do_late_global_mode;
     bool const old_do_meta_mode = do_meta_mode;
+    bool const old_do_early_meta_mode = do_early_meta_mode;
+    bool const old_do_late_meta_mode = do_late_meta_mode;
     do_global_mode = true;
+    do_early_global_mode = true;
+    do_late_global_mode = true;
     do_meta_mode = true;
+    do_early_meta_mode = true;
+    do_late_meta_mode = true;
     
     ENTER_GLOBAL_MODE (cctkGH, 0) {
       ENTER_LEVEL_MODE (cctkGH, 0) {
@@ -842,7 +894,11 @@ namespace Carpet {
     } LEAVE_GLOBAL_MODE;
     
     do_global_mode = old_do_global_mode;
+    do_early_global_mode = old_do_early_global_mode;
+    do_late_global_mode = old_do_late_global_mode;
     do_meta_mode = old_do_meta_mode;
+    do_early_meta_mode = old_do_early_meta_mode;
+    do_late_meta_mode = old_do_late_meta_mode;
     
     timer.stop();
   }
@@ -864,9 +920,17 @@ namespace Carpet {
     assert (is_level_mode());
     
     bool const old_do_global_mode = do_global_mode;
+    bool const old_do_early_global_mode = do_early_global_mode;
+    bool const old_do_late_global_mode = do_late_global_mode;
     bool const old_do_meta_mode = do_meta_mode;
+    bool const old_do_early_meta_mode = do_early_meta_mode;
+    bool const old_do_late_meta_mode = do_late_meta_mode;
     do_global_mode = true;
+    do_early_global_mode = true;
+    do_late_global_mode = true;
     do_meta_mode = true;
+    do_early_meta_mode = true;
+    do_late_meta_mode = true;
     
     // Preregrid
     Waypoint ("Preregridinitial at iteration %d time %g%s%s",
@@ -891,8 +955,12 @@ namespace Carpet {
           if (did_recompose) {
             BEGIN_MGLEVEL_LOOP (cctkGH) {
               ENTER_LEVEL_MODE (cctkGH, rl) {
-                do_global_mode = reflevel == reflevels - 1;
-                do_meta_mode = do_global_mode and mglevel==mglevels-1;
+                do_early_global_mode = reflevel==0;
+                do_late_global_mode = reflevel==reflevels-1;
+                do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+                do_late_meta_mode = do_late_global_mode and mglevel==0;
+                do_global_mode = do_late_global_mode;
+                do_meta_mode = do_late_meta_mode;
                 
                 Waypoint ("Postregridinitial at iteration %d time %g%s%s",
                           cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -952,7 +1020,11 @@ namespace Carpet {
     } // if did_regrid
     
     do_global_mode = old_do_global_mode;
+    do_early_global_mode = old_do_early_global_mode;
+    do_late_global_mode = old_do_late_global_mode;
     do_meta_mode = old_do_meta_mode;
+    do_early_meta_mode = old_do_early_meta_mode;
+    do_late_meta_mode = old_do_late_meta_mode;
     
     timer.stop();
   }
@@ -988,8 +1060,12 @@ namespace Carpet {
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
       BEGIN_REFLEVEL_LOOP(cctkGH) {
-        do_global_mode = reflevel==0;
-        do_meta_mode = do_global_mode and mglevel==mglevels-1;
+        do_early_global_mode = reflevel==0;
+        do_late_global_mode = reflevel==reflevels-1;
+        do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+        do_late_meta_mode = do_late_global_mode and mglevel==0;
+        do_global_mode = do_early_global_mode;
+        do_meta_mode = do_early_meta_mode;
         
         initialise_3tl_advance_time (cctkGH);
         initialise_3tl_evolve_Ia (cctkGH);
@@ -1006,8 +1082,12 @@ namespace Carpet {
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
       BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) {
-        do_global_mode = reflevel==0;
-        do_meta_mode = do_global_mode and mglevel==mglevels-1;
+        do_early_global_mode = reflevel==reflevels-1;
+        do_late_global_mode = reflevel==0;
+        do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+        do_late_meta_mode = do_late_global_mode and mglevel==0;
+        do_global_mode = do_early_global_mode;
+        do_meta_mode = do_early_meta_mode;
         
         initialise_3tl_evolve_IIb (cctkGH);
         initialise_3tl_advance_time_2 (cctkGH);
@@ -1020,8 +1100,12 @@ namespace Carpet {
     
     BEGIN_MGLEVEL_LOOP(cctkGH) {
       BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) {
-        do_global_mode = reflevel==0;
-        do_meta_mode = do_global_mode and mglevel==mglevels-1;
+        do_early_global_mode = reflevel==reflevels-1;
+        do_late_global_mode = reflevel==0;
+        do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1;
+        do_late_meta_mode = do_late_global_mode and mglevel==0;
+        do_global_mode = do_early_global_mode;
+        do_meta_mode = do_early_meta_mode;
         
         initialise_3tl_reset_time (cctkGH);
         
diff --git a/Carpet/Carpet/src/Shutdown.cc b/Carpet/Carpet/src/Shutdown.cc
index 374157bef..38e05bcae 100644
--- a/Carpet/Carpet/src/Shutdown.cc
+++ b/Carpet/Carpet/src/Shutdown.cc
@@ -32,8 +32,13 @@ namespace Carpet {
     for (int rl=reflevels-1; rl>=0; --rl) {
       BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) {
         enter_level_mode (cctkGH, rl);
-        do_global_mode = reflevel==0;
-        do_meta_mode = do_global_mode and mglevel==mglevels-1;
+        
+        do_early_global_mode = reflevel==reflevels-1;
+        do_late_global_mode = reflevel==0;
+        do_early_meta_mode = do_early_global_mode and mglevel==0;
+        do_late_meta_mode = do_late_global_mode and mglevel==mglevels-1;
+        do_global_mode = do_late_global_mode;
+        do_meta_mode = do_late_meta_mode;
         
         Checkpoint ("Shutdown at iteration %d time %g%s%s",
                     cctkGH->cctk_iteration, (double)cctkGH->cctk_time,
@@ -49,8 +54,12 @@ namespace Carpet {
     } // for rl
     
     BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) {
-      do_global_mode = true;
-      do_meta_mode = mglevel==mglevels-1;
+      do_early_global_mode = true;
+      do_late_global_mode = true;
+      do_early_meta_mode = do_early_global_mode and mglevel==0;
+      do_late_meta_mode = do_late_global_mode and mglevel==mglevels-1;
+      do_global_mode = do_late_global_mode;
+      do_meta_mode = do_late_meta_mode;
       
       // Shutdown
       Checkpoint ("Scheduling SHUTDOWN");
diff --git a/Carpet/Carpet/src/variables.cc b/Carpet/Carpet/src/variables.cc
index 6b98f4312..a27a5bacb 100644
--- a/Carpet/Carpet/src/variables.cc
+++ b/Carpet/Carpet/src/variables.cc
@@ -80,7 +80,11 @@ namespace Carpet {
   
   // Is this the time for a global mode call?
   bool do_meta_mode;
+  bool do_early_meta_mode;
+  bool do_late_meta_mode;
   bool do_global_mode;
+  bool do_early_global_mode;
+  bool do_late_global_mode;
   
   // Can past time levels be accessed?
   bool do_allow_past_timelevels;
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index a50d471ca..23440aea5 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -117,7 +117,11 @@ namespace Carpet {
   
   // Is this the time for a global mode call?
   extern bool do_meta_mode;
+  extern bool do_early_meta_mode;
+  extern bool do_late_meta_mode;
   extern bool do_global_mode;
+  extern bool do_early_global_mode;
+  extern bool do_late_global_mode;
   
   // Can past time levels be accessed?
   extern bool do_allow_past_timelevels;