Fix checkpointing at arbitrary iterations

Checkpointing/recovery is now possible at arbitrary iterations.

Carpet/CarpetIOHDF5/src/Recover.cc: synchronize all groups are recovery and
filereader. This fixes differences in the initial data on finer levels where no
interpolation from the coarser levels has been done yet.

Carpet/CarpetIOHDF5/src/Checkpoint.cc: removed ugly code to self-modify the
IO::checkpoint_every parameter in order to force checkpointing only on the
coarsest refinement level

Carpet/Carpet/src/Evolve.cc: check the termination conditions on each active
refinement level (not only the coarsest one)

darcs-hash:20041209135748-3fd61-92910a41aa8547473dc7e7a947fb98e8d5617517.gz

1 files changed, 66 insertions, 69 deletions

diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc
index 3a2db3a9d..b723c924b 100644
--- a/Carpet/Carpet/src/Evolve.cc
+++ b/Carpet/Carpet/src/Evolve.cc
@@ -38,33 +38,30 @@ extern "C" {
 
 
 namespace Carpet {
-  
+
   using namespace std;
-  
-  
-  
+
+
+
   static bool do_terminate (const cGH *cgh,
                             const CCTK_REAL time, const int iteration)
   {
     DECLARE_CCTK_PARAMETERS;
-    
+
     bool term;
-    
-    // Early shortcut
-    if (iteration % maxreflevelfact != 0) {
-      
-      // Terminate only after complete coarse grid steps
-      // TODO: once checkpointing works correctly, change this to
-      // "after complete time steps"
+
+    // Early return for non-active reflevels to save the MPI_Allreduce() below
+    if (iteration % (maxreflevelfact / ipow(reffact, reflevels-1)) != 0) {
+
       return false;
-      
+
     } else if (terminate_next || CCTK_TerminationReached(cgh)) {
-      
+
       // Terminate if someone or something said so
       term = true;
-      
+
     } else {
-      
+
       const bool term_iter = iteration >= cctk_itlast;
       const bool term_time
         = (delta_time > 0
@@ -75,21 +72,21 @@ namespace Carpet {
       struct timeval tv;
       gettimeofday (&tv, 0);
       const double thetime = tv.tv_sec + tv.tv_usec / 1e6;
-      
+
       static bool firsttime = true;
       static double initial_runtime;
       if (firsttime) {
         firsttime = false;
         initial_runtime = thetime;
       }
-      
+
       const double runtime = thetime - initial_runtime;
       const bool term_runtime = (max_runtime > 0
                                  && runtime >= 60.0 * max_runtime);
 #else
       const bool term_runtime = false;
 #endif
-      
+
       if (CCTK_Equals(terminate, "never")) {
         term = false;
       } else if (CCTK_Equals(terminate, "iteration")) {
@@ -111,36 +108,36 @@ namespace Carpet {
       } else {
         CCTK_WARN (0, "Unsupported termination condition");
       }
-      
+
     }
-    
+
     {
       int local, global;
       local = term;
       MPI_Allreduce (&local, &global, 1, MPI_INT, MPI_LOR, dist::comm);
       term = global;
     }
-    
+
     return term;
   }
-  
-  
+
+
   static void AdvanceTime( cGH* cgh, CCTK_REAL initial_time );
   static bool Regrid( cGH* cgh );
   static void PostRegrid( cGH* cgh );
   static void EvolutionI( cGH* cgh );
   static void Evolution_Restrict( cGH* cgh );
   static void EvolutionII( cGH* cgh );
-  
+
   int Evolve (tFleshConfig* fc)
   {
     DECLARE_CCTK_PARAMETERS;
-    
+
     Waypoint ("Starting evolution loop");
-    
+
     const int convlev = 0;
     cGH* cgh = fc->GH[convlev];
-    
+
     // Main loop
     while (! do_terminate(cgh, cgh->cctk_time, cgh->cctk_iteration)) {
 
@@ -151,19 +148,19 @@ namespace Carpet {
         Waypoint ("Evolving iteration %d at t=%g",
                   cgh->cctk_iteration, (double)cgh->cctk_time);
       }
-  
+
       if( Regrid( cgh ) )
         PostRegrid( cgh );
-      
+
       EvolutionI( cgh );
-      
+
       Evolution_Restrict( cgh );
-      
+
       EvolutionII( cgh );
     }
-    
+
     Waypoint ("Done with evolution loop");
-    
+
     return 0;
   }
 
@@ -185,10 +182,10 @@ namespace Carpet {
       if ((cgh->cctk_iteration-1) % do_every == 0) {
         enter_global_mode (cgh, ml);
         enter_level_mode (cgh, rl);
-        
+
         Checkpoint ("Regrid");
         did_regrid |= Regrid (cgh, false, true);
-         
+
         leave_level_mode (cgh);
         leave_global_mode (cgh);
       }
@@ -202,18 +199,18 @@ namespace Carpet {
       for (int ml=mglevels-1; ml>=0; --ml) {
         enter_global_mode (cgh, ml);
         enter_level_mode (cgh, rl);
-        
+
         do_global_mode = reflevel==0;
         do_meta_mode = do_global_mode && mglevel==mglevels-1;
-        
+
         Waypoint ("Postregrid at iteration %d time %g%s%s",
                   cgh->cctk_iteration, (double)cgh->cctk_time,
                   (do_global_mode ? " (global)" : ""),
                   (do_meta_mode ? " (meta)" : ""));
-        
+
         Checkpoint ("Scheduling POSTREGRID");
         CCTK_ScheduleTraverse ("CCTK_POSTREGRID", cgh, CallFunction);
-        
+
         leave_level_mode (cgh);
         leave_global_mode (cgh);
       }
@@ -223,23 +220,23 @@ namespace Carpet {
   void EvolutionI( cGH* cgh )
   {
     for (int ml=mglevels-1; ml>=0; --ml) {
-      
+
       bool have_done_global_mode = false;
       bool have_done_anything = false;
-      
+
       for (int rl=0; rl<reflevels; ++rl) {
         const int do_every
           = ipow(mgfact, ml) * (maxreflevelfact / ipow(reffact, rl));
         if ((cgh->cctk_iteration-1) % do_every == 0) {
           enter_global_mode (cgh, ml);
           enter_level_mode (cgh, rl);
-          
+
           do_global_mode = ! have_done_global_mode;
           do_meta_mode = do_global_mode && mglevel==mglevels-1;
           assert (! (have_done_global_mode && do_global_mode));
           have_done_global_mode |= do_global_mode;
           have_done_anything = true;
-          
+
           // Advance times
           for (int m=0; m<maps; ++m) {
             vtt.at(m)->advance_time (reflevel, mglevel);
@@ -248,33 +245,33 @@ namespace Carpet {
                             - delta_time / maxreflevelfact
                             + delta_time * mglevelfact / reflevelfact);
           CycleTimeLevels (cgh);
-   
+
           Waypoint ("Evolution I at iteration %d time %g%s%s",
                     cgh->cctk_iteration, (double)cgh->cctk_time,
                     (do_global_mode ? " (global)" : ""),
                     (do_meta_mode ? " (meta)" : ""));
-          
+
           // Checking
           CalculateChecksums (cgh, allbutcurrenttime);
           Poison (cgh, currenttimebutnotifonly);
-   
+
           // Evolve
           Checkpoint ("Scheduling PRESTEP");
           CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
           Checkpoint ("Scheduling EVOL");
           CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
-          
+
           // Checking
           PoisonCheck (cgh, currenttime);
-          
+
           leave_level_mode (cgh);
           leave_global_mode (cgh);
         }
       }
-      
+
       if (have_done_anything)
         assert (have_done_global_mode);
-        
+
     }
   }
 
@@ -287,12 +284,12 @@ namespace Carpet {
         if (cgh->cctk_iteration % do_every == 0) {
           enter_global_mode (cgh, ml);
           enter_level_mode (cgh, rl);
-          
+
           Waypoint ("Evolution/Restrict at iteration %d time %g",
                     cgh->cctk_iteration, (double)cgh->cctk_time);
-          
+
           Restrict (cgh);
-          
+
           leave_level_mode (cgh);
           leave_global_mode (cgh);
         }
@@ -303,17 +300,17 @@ namespace Carpet {
   void EvolutionII( cGH* cgh )
   {
     for (int ml=mglevels-1; ml>=0; --ml) {
-      
+
       bool have_done_global_mode = false;
       bool have_done_anything = false;
-      
+
       for (int rl=0; rl<reflevels; ++rl) {
         const int do_every
           = ipow(mgfact, ml) * (maxreflevelfact / ipow(reffact, rl));
         if (cgh->cctk_iteration % do_every == 0) {
           enter_global_mode (cgh, ml);
           enter_level_mode (cgh, rl);
-            
+
           int finest_active_reflevel = -1;
           {
             for (int rl_=0; rl_<reflevels; ++rl_) {
@@ -330,47 +327,47 @@ namespace Carpet {
           assert (! (have_done_global_mode && do_global_mode));
           have_done_global_mode |= do_global_mode;
           have_done_anything = true;
-          
+
           Waypoint ("Evolution II at iteration %d time %g%s%s",
                     cgh->cctk_iteration, (double)cgh->cctk_time,
                     (do_global_mode ? " (global)" : ""),
                     (do_meta_mode ? " (meta)" : ""));
-          
+
           Checkpoint ("Scheduling POSTRESTRICT");
           CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction);
-          
+
           // Poststep
           Checkpoint ("Scheduling POSTSTEP");
           CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-   
+
           // Checking
           PoisonCheck (cgh, currenttime);
           CalculateChecksums (cgh, currenttime);
-          
+
           // Checkpoint
           Checkpoint ("Scheduling CHECKPOINT");
           CCTK_ScheduleTraverse ("CCTK_CHECKPOINT", cgh, CallFunction);
-   
+
           // Analysis
           Checkpoint ("Scheduling ANALYSIS");
           CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
-          
+
           // Output
           Checkpoint ("OutputGH");
           CCTK_OutputGH (cgh);
-          
+
           // Checking
           CheckChecksums (cgh, alltimes);
-          
+
           leave_level_mode (cgh);
           leave_global_mode (cgh);
         }
       }
- 
+
       if (have_done_anything)
         assert (have_done_global_mode);
-        
+
     }
   }
-  
+
 } // namespace Carpet