Initialise_decompose

Broke this mama of a function into a couple of levels of functional
parts.

darcs-hash:20041201155737-32473-ce364db1ab14403a823d0bf0937ff4520ed04639.gz

6 files changed, 779 insertions, 455 deletions

diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc
index 421846255..c2a893416 100644
--- a/Carpet/Carpet/src/Initialise.cc
+++ b/Carpet/Carpet/src/Initialise.cc
@@ -23,51 +23,107 @@ namespace Carpet {
   using namespace std;
   
   
+  static void output_the_grid_structure( cGH *cgh );
+  static void register_coordinates_and_check_parameters( cGH *cgh );
+  static void recovery_I( cGH * cgh, int rl );
+  static void recovery_Regrid( cGH * cgh, int rl );
+  static void recovery_II( cGH * cgh );
+  static void 
+  initialisation_I( cGH * cgh, int rl, int init_each_timelevel );
+  static void initialise_rewind( cGH * cgh, int num_tl );
+  static void initialise_Schedule_INITIAL( cGH * cgh, int num_tl );
+  static void
+  initialise_Regrid( cGH * cgh, int rl, int prolongate_initial_data );
+  static void initialise_Restrict( cGH * cgh );
+  static void initialisation_II( cGH * cgh );
+  static double
+  get_two_extra_timelevels_of_data( cGH * cgh, double delta_time );
+  static void initialise_3_Timelevels( cGH * cgh );
+  static void initialise_Flip_Timelevels( cGH * cgh, double delta_time );
+  static void
+  initialise_evolve_3TL_backwards_Ib( cGH * cgh, double delta_time );
+  static void
+  initialise_evolve_3TL_backwards_IIb_Ic( cGH * cgh, double delta_time );
+  static void
+  initialise_Flip_Timelevels_back( cGH * cgh, double delta_time );
+  static void initialisation_III( cGH * cgh );
   
   int Initialise (tFleshConfig* fc)
   {
     DECLARE_CCTK_PARAMETERS;
     
-    // Initialise stuff
     const int convlev = 0;
     cGH* const cgh = CCTK_SetupGH (fc, convlev);
     CCTKi_AddGH (fc, convlev, cgh);
     
-    // Delay checkpoint until MPI has been initialised
-    Waypoint ("Starting initialisation");
-    
-    // Initialise stuff
-    cgh->cctk_iteration = 0;
+    do_global_mode = true;
+    do_meta_mode = true;
     global_time = cctk_initial_time;
     delta_time = 1.0;
+    
+    cgh->cctk_iteration = 0;
     cgh->cctk_time = global_time;
     cgh->cctk_delta_time = delta_time;
+
+    // Delay checkpoint until MPI has been initialised
+    Waypoint ("Starting initialisation");
     
-    do_global_mode = true;
-    do_meta_mode = true;
-    
-    // Enable storage and communtication
-    CCTKi_ScheduleGHInit (cgh);
+    CCTKi_ScheduleGHInit (cgh); // Enable storage and communication
     
-    // Initialise stuff
     CCTKi_InitGHExtensions (cgh);
     
+    output_the_grid_structure( cgh );
     
+    register_coordinates_and_check_parameters( cgh );
     
-    // Output the grid structure
-    {
-      // Loop over maps
-      for (int m=0; m<maps; ++m) {
-        // Write grid structure to file
-        OutputGridStructure
-          (cgh, m,
-           vhh.at(m)->extents, vhh.at(m)->outer_boundaries,
-           vhh.at(m)->processors);
-      } // loop over maps
-    }
+    if (fc->recovered) {
+      
+      for (int rl=0; rl<reflevels; ++rl) {
+        recovery_I( cgh, rl );
+        recovery_Regrid( cgh, rl );
+      } 
+
+      recovery_II( cgh );
+      
+    } else {
+
+      for (int rl=0; rl<reflevels; ++rl) {
+        initialisation_I( cgh, rl, init_each_timelevel );
+        initialise_Regrid( cgh, rl, prolongate_initial_data );
+      }
+      
+      initialise_Restrict( cgh );
+      
+      initialisation_II( cgh );
     
+      if (init_3_timelevels) {
+        delta_time = get_two_extra_timelevels_of_data( cgh, delta_time );
+      } 
+    }
     
+    initialisation_III( cgh );
+ 
+    Waypoint ("Done with initialisation");
     
+    return 0;
+  }
+
+  void
+  output_the_grid_structure( cGH *cgh )
+  {
+    // Loop over maps
+    for (int m=0; m<maps; ++m) {
+      // Write grid structure to file
+      OutputGridStructure
+        (cgh, m,
+         vhh.at(m)->extents, vhh.at(m)->outer_boundaries,
+         vhh.at(m)->processors);
+    } // loop over maps
+  }
+
+  void
+  register_coordinates_and_check_parameters( cGH *cgh )
+  {
     BEGIN_MGLEVEL_LOOP(cgh) {
       do_global_mode = true;
       do_meta_mode = mglevel==mglevels-1;
@@ -81,476 +137,494 @@ namespace Carpet {
       CCTK_ScheduleTraverse ("CCTK_PARAMCHECK", cgh, CallFunction);
       CCTKi_FinaliseParamWarn();
     } END_MGLEVEL_LOOP;
+  }
     
-    
-    
-    if (fc->recovered) {
-      // if recovering
+  void
+  recovery_I( cGH * cgh, int rl )
+  { 
+    BEGIN_MGLEVEL_LOOP(cgh) {
+      enter_level_mode (cgh, rl);
+      do_global_mode = reflevel==0;
+      do_meta_mode = do_global_mode && mglevel==mglevels-1;
       
+      cgh->cctk_time = global_time;
       
+      Waypoint ("Recovering I at iteration %d time %g%s%s",
+                cgh->cctk_iteration, (double)cgh->cctk_time,
+                (do_global_mode ? " (global)" : ""),
+                (do_meta_mode ? " (meta)" : ""));
       
-      for (int rl=0; rl<reflevels; ++rl) {
-        BEGIN_MGLEVEL_LOOP(cgh) {
-          enter_level_mode (cgh, rl);
-          do_global_mode = reflevel==0;
-          do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-          cgh->cctk_time = global_time;
+      // Set up the grids
+      Checkpoint ("Scheduling BASEGRID");
+      CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
           
-          Waypoint ("Recovering I at iteration %d time %g%s%s",
-                    cgh->cctk_iteration, (double)cgh->cctk_time,
-                    (do_global_mode ? " (global)" : ""),
-                    (do_meta_mode ? " (meta)" : ""));
-          
-          // Set up the grids
-          Checkpoint ("Scheduling BASEGRID");
-          CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
-          
-          // Recover
-          Checkpoint ("Scheduling RECOVER_VARIABLES");
-          CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction);
-          
-          leave_level_mode (cgh);
-        } END_MGLEVEL_LOOP;
+      // Recover
+      Checkpoint ("Scheduling RECOVER_VARIABLES");
+      CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction);
+      
+      leave_level_mode (cgh);
+    } END_MGLEVEL_LOOP;
+  }
+
+  void
+  recovery_Regrid( cGH * cgh, int rl )
+  { 
+    bool did_regrid = false;
+    {
+      const int ml=0;
+      enter_global_mode (cgh, ml);
+      enter_level_mode (cgh, rl);
+      
+      // Regrid
+      Checkpoint ("Regrid");
+      did_regrid |= Regrid (cgh, true, false);
+      
+      leave_level_mode (cgh);
+      leave_global_mode (cgh);
+    } // ml
+    
+    if (did_regrid) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = true;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
         
-        // Regrid
-        {
-          bool did_regrid = false;
-          {
-            const int ml=0;
-            enter_global_mode (cgh, ml);
-            enter_level_mode (cgh, rl);
-            
-            // Regrid
-            Checkpoint ("Regrid");
-            did_regrid |= Regrid (cgh, true, false);
-            
-            leave_level_mode (cgh);
-            leave_global_mode (cgh);
-          } // ml
-          
-          if (did_regrid) {
-            BEGIN_MGLEVEL_LOOP(cgh) {
-              enter_level_mode (cgh, rl);
-              do_global_mode = true;
-              do_meta_mode = do_global_mode && mglevel==mglevels-1;
-              
-              Waypoint ("Postregrid at iteration %d time %g%s%s",
-                        cgh->cctk_iteration, (double)cgh->cctk_time,
-                        (do_global_mode ? " (global)" : ""),
-                        (do_meta_mode ? " (meta)" : ""));
-              
-              // Postregrid
-              Checkpoint ("Scheduling POSTREGRID");
-              CCTK_ScheduleTraverse ("CCTK_POSTREGRID", cgh, CallFunction);
-              
-              leave_level_mode (cgh);
-            } END_MGLEVEL_LOOP;
-          } // if did_regrid
-        }
+        Waypoint ("Postregrid at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+        
+        // Postregrid
+        Checkpoint ("Scheduling POSTREGRID");
+        CCTK_ScheduleTraverse ("CCTK_POSTREGRID", cgh, CallFunction);
         
-      } // for rl
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    } // if did_regrid
+  }
+
+  void
+  recovery_II( cGH * cgh )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==reflevels-1;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
       
+        Waypoint ("Recovering II at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
       
+        // Post recover
+        Checkpoint ("Scheduling POST_RECOVER_VARIABLES");
+        CCTK_ScheduleTraverse
+          ("CCTK_POST_RECOVER_VARIABLES", cgh, CallFunction);
       
-      for (int rl=0; rl<reflevels; ++rl) {
-        BEGIN_MGLEVEL_LOOP(cgh) {
-          enter_level_mode (cgh, rl);
-          do_global_mode = reflevel==reflevels-1;
-          do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-          Waypoint ("Recovering II at iteration %d time %g%s%s",
-                    cgh->cctk_iteration, (double)cgh->cctk_time,
-                    (do_global_mode ? " (global)" : ""),
-                    (do_meta_mode ? " (meta)" : ""));
-          
-          // Post recover
-          Checkpoint ("Scheduling POST_RECOVER_VARIABLES");
-          CCTK_ScheduleTraverse
-            ("CCTK_POST_RECOVER_VARIABLES", cgh, CallFunction);
-          
-          // Checking
-          PoisonCheck (cgh, alltimes);
-          CheckChecksums (cgh, allbutcurrenttime);
-          
-          leave_level_mode (cgh);
-        } END_MGLEVEL_LOOP;
-      } // for rl
+        // Checking
+        PoisonCheck (cgh, alltimes);
+        CheckChecksums (cgh, allbutcurrenttime);
       
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+  
+  void
+  initialisation_I( cGH * cgh, int rl, int init_each_timelevel )
+  { 
+    BEGIN_MGLEVEL_LOOP(cgh) {
+      enter_level_mode (cgh, rl);
+      do_global_mode = reflevel==0;
+      do_meta_mode = do_global_mode && mglevel==mglevels-1;
       
+      cgh->cctk_time = global_time;
       
-    } else {
-      // if not recovering
+      Waypoint ("Initialisation I at iteration %d time %g%s%s",
+                cgh->cctk_iteration, (double)cgh->cctk_time,
+                (do_global_mode ? " (global)" : ""),
+                (do_meta_mode ? " (meta)" : ""));
       
+      // Checking
+      Poison (cgh, alltimes);
       
+      // Set up the grids
+      Checkpoint ("Scheduling BASEGRID");
+      CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
       
-      for (int rl=0; rl<reflevels; ++rl) {
-        BEGIN_MGLEVEL_LOOP(cgh) {
-          enter_level_mode (cgh, rl);
-          do_global_mode = reflevel==0;
-          do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-          cgh->cctk_time = global_time;
-          
-          Waypoint ("Initialisation I at iteration %d time %g%s%s",
-                    cgh->cctk_iteration, (double)cgh->cctk_time,
-                    (do_global_mode ? " (global)" : ""),
-                    (do_meta_mode ? " (meta)" : ""));
-          
-          // Checking
-          Poison (cgh, alltimes);
-          
-          // Set up the grids
-          Checkpoint ("Scheduling BASEGRID");
-          CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction);
-          
-          const int num_tl = init_each_timelevel ? 3 : 1;
-          
-          // Rewind
-          for (int m=0; m<maps; ++m) {
-            vtt.at(m)->set_delta
-              (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
-            FlipTimeLevels (cgh);
-            for (int tl=0; tl<num_tl; ++tl) {
-              vtt.at(m)->advance_time (reflevel, mglevel);
-              CycleTimeLevels (cgh);
-            }
-            vtt.at(m)->set_delta
-              (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
-            FlipTimeLevels (cgh);
-          }
-          
-          const bool outer_do_global_mode = do_global_mode;
-          for (int tl=num_tl-1; tl>=0; --tl) {
-            do_global_mode = outer_do_global_mode && tl==0;
-            
-            // Advance times
-            for (int m=0; m<maps; ++m) {
-              vtt.at(m)->advance_time (reflevel, mglevel);
-            }
-            cgh->cctk_time
-              = global_time - tl * delta_time * mglevelfact / reflevelfact;
-            CycleTimeLevels (cgh);
-            
-            // Set up the initial data
-            Checkpoint ("Scheduling INITIAL");
-            CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction);
-            
-          } // for tl
-          do_global_mode = outer_do_global_mode;
-          
-          // Checking
-          PoisonCheck (cgh, currenttime);
-          
-          leave_level_mode (cgh);
-        } END_MGLEVEL_LOOP;
-        
-        // Regrid
-        {
-          bool did_regrid = false;
-          {
-            const int ml=0;
-            enter_global_mode (cgh, ml);
-            enter_level_mode (cgh, rl);
-            
-            // Regrid
-            Checkpoint ("Regrid");
-            did_regrid |= Regrid (cgh, false, prolongate_initial_data);
-            
-            leave_level_mode (cgh);
-            leave_global_mode (cgh);
-          } // ml
-          
-          if (did_regrid) {
-            BEGIN_MGLEVEL_LOOP(cgh) {
-              enter_level_mode (cgh, rl);
-              do_global_mode = true;
-              do_meta_mode = do_global_mode && mglevel==mglevels-1;
-              
-              Waypoint ("Postregrid at iteration %d time %g%s%s",
-                        cgh->cctk_iteration, (double)cgh->cctk_time,
-                        (do_global_mode ? " (global)" : ""),
-                        (do_meta_mode ? " (meta)" : ""));
-              
-              // Postregrid
-              Checkpoint ("Scheduling POSTREGRID");
-              CCTK_ScheduleTraverse ("CCTK_POSTREGRID", cgh, CallFunction);
-              
-              leave_level_mode (cgh);
-            } END_MGLEVEL_LOOP;
-          } // if did_regrid
-        }
-        
-      } // for rl
+      const int num_tl = init_each_timelevel ? 3 : 1;
       
+      initialise_rewind( cgh, num_tl );
+
+      initialise_Schedule_INITIAL( cgh, num_tl );
       
+      // Checking
+      PoisonCheck (cgh, currenttime);
       
-      for (int rl=reflevels-1; rl>=0; --rl) {
-        BEGIN_MGLEVEL_LOOP(cgh) {
-          enter_level_mode (cgh, rl);
-          
-          Waypoint ("Initialisation/Restrict at iteration %d time %g",
-                    cgh->cctk_iteration, (double)cgh->cctk_time);
-          
-          // Restrict
-          Restrict (cgh);
-          
-          leave_level_mode (cgh);
-        } END_MGLEVEL_LOOP;
-      } // for rl
+      leave_level_mode (cgh);
+    } END_MGLEVEL_LOOP;
+  }
+  
+  void
+  initialise_rewind( cGH * cgh, int num_tl )
+  { 
+    for (int m=0; m<maps; ++m) {
+      vtt.at(m)->set_delta
+        (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
+      FlipTimeLevels (cgh);
+      for (int tl=0; tl<num_tl; ++tl) {
+        vtt.at(m)->advance_time (reflevel, mglevel);
+        CycleTimeLevels (cgh);
+      }
+      vtt.at(m)->set_delta
+        (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
+      FlipTimeLevels (cgh);
+    }
+  } 
+  
+  void
+  initialise_Schedule_INITIAL( cGH * cgh, int num_tl )
+  { 
+    const bool outer_do_global_mode = do_global_mode;
+    for (int tl=num_tl-1; tl>=0; --tl) {
+      do_global_mode = outer_do_global_mode && tl==0;
       
+      // Advance times
+      for (int m=0; m<maps; ++m) {
+        vtt.at(m)->advance_time (reflevel, mglevel);
+      }
+      cgh->cctk_time
+        = global_time - tl * delta_time * mglevelfact / reflevelfact;
+      CycleTimeLevels (cgh);
       
+      // Set up the initial data
+      Checkpoint ("Scheduling INITIAL");
+      CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction);
       
-      for (int rl=0; rl<reflevels; ++rl) {
-        BEGIN_MGLEVEL_LOOP(cgh) {
-          enter_level_mode (cgh, rl);
-          do_global_mode = reflevel==reflevels-1;
-          do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-          Waypoint ("Initialisation II at iteration %d time %g%s%s",
-                    cgh->cctk_iteration, (double)cgh->cctk_time,
-                    (do_global_mode ? " (global)" : ""),
-                    (do_meta_mode ? " (meta)" : ""));
-          
-          Checkpoint ("Scheduling POSTRESTRICTINITIAL");
-          CCTK_ScheduleTraverse
-            ("CCTK_POSTRESTRICTINITIAL", cgh, CallFunction);
-          
-          // Postinitial
-          Checkpoint ("Scheduling POSTINITIAL");
-          CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction);
-          
-          // Poststep
-          Checkpoint ("Scheduling POSTSTEP");
-          CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-          
-          // Checking
-          PoisonCheck (cgh, alltimes);
-          CheckChecksums (cgh, allbutcurrenttime);
-          
-          leave_level_mode (cgh);
-        } END_MGLEVEL_LOOP;
-      } // for rl
+    } // for tl
+    do_global_mode = outer_do_global_mode;
+  } 
+
+  void
+  initialise_Regrid( cGH * cgh, int rl, int prolongate_initial_data )
+  { 
+    bool did_regrid = false;
+    {
+      const int ml=0;
+      enter_global_mode (cgh, ml);
+      enter_level_mode (cgh, rl);
       
+      // Regrid
+      Checkpoint ("Regrid");
+      did_regrid |= Regrid (cgh, false, prolongate_initial_data);
+      
+      leave_level_mode (cgh);
+      leave_global_mode (cgh);
+    } // ml
     
-    
-      if (init_3_timelevels) {
-        // Use Scott Hawley's algorithm for getting two extra
-        // timelevels of data
+    if (did_regrid) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = true;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+        
+        Waypoint ("Postregrid at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+        
+        // Postregrid
+        Checkpoint ("Scheduling POSTREGRID");
+        CCTK_ScheduleTraverse ("CCTK_POSTREGRID", cgh, CallFunction);
+        
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    } // if did_regrid
+  }
+
+  void
+  initialise_Restrict( cGH * cgh )
+  { 
+    for (int rl=reflevels-1; rl>=0; --rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        
+        Waypoint ("Initialisation/Restrict at iteration %d time %g",
+                  cgh->cctk_iteration, (double)cgh->cctk_time);
+        
+        Restrict (cgh);
+        
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  void
+  initialisation_II( cGH * cgh )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==reflevels-1;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+      
+        Waypoint ("Initialisation II at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+      
+        Checkpoint ("Scheduling POSTRESTRICTINITIAL");
+        CCTK_ScheduleTraverse
+          ("CCTK_POSTRESTRICTINITIAL", cgh, CallFunction);
+      
+        Checkpoint ("Scheduling POSTINITIAL");
+        CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction);
+      
+        Checkpoint ("Scheduling POSTSTEP");
+        CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
+      
+        PoisonCheck (cgh, alltimes);
+        CheckChecksums (cgh, allbutcurrenttime);
+      
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  // Use Scott Hawley's algorithm for getting two extra
+  // timelevels of data
+  double
+  get_two_extra_timelevels_of_data( cGH * cgh, double delta_time )
+  { 
         Waypoint ("Initialising three timelevels");
       
-        for (int rl=0; rl<reflevels; ++rl) {
-          BEGIN_MGLEVEL_LOOP(cgh) {
-            enter_level_mode (cgh, rl);
-            do_global_mode = reflevel==0;
-            do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-            // Advance times
-            for (int m=0; m<maps; ++m) {
-              vtt.at(m)->advance_time (reflevel, mglevel);
-            }
-            cgh->cctk_time
-              = global_time + delta_time * mglevelfact / reflevelfact;
-            CycleTimeLevels (cgh);
-          
-            Waypoint ("Initialisation 3TL evolution I (a) (forwards) at iteration %d time %g%s%s",
-                      cgh->cctk_iteration, (double)cgh->cctk_time,
-                      (do_global_mode ? " (global)" : ""),
-                      (do_meta_mode ? " (meta)" : ""));
-          
-            // Checking
-            CalculateChecksums (cgh, allbutcurrenttime);
-            Poison (cgh, currenttimebutnotifonly);
-          
-            // Evolve forward
-            Checkpoint ("Scheduling PRESTEP");
-            CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
-            Checkpoint ("Scheduling EVOL");
-            CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
-          
-            // Checking
-            PoisonCheck (cgh, currenttime);
-          
-            leave_level_mode (cgh);
-          } END_MGLEVEL_LOOP;
-        } // for rl
+        initialise_3_Timelevels( cgh );
       
         delta_time *= -1;
-        for (int rl=0; rl<reflevels; ++rl) {
-          BEGIN_MGLEVEL_LOOP(cgh) {
-            enter_level_mode (cgh, rl);
-            do_global_mode = reflevel==0;
-            do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-            // Flip time levels
-            Waypoint ("Flipping timelevels");
-            FlipTimeLevels (cgh);
-          
-            cgh->cctk_time
-              = global_time + delta_time * mglevelfact / reflevelfact;
-          
-            leave_level_mode (cgh);
-          } END_MGLEVEL_LOOP;
-        } // for rl
-      
-        for (int rl=0; rl<reflevels; ++rl) {
-          BEGIN_MGLEVEL_LOOP(cgh) {
-            enter_level_mode (cgh, rl);
-            do_global_mode = reflevel==0;
-            do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-            Waypoint ("Initialisation 3TL evolution I (b) (backwards) at iteration %d time %g%s%s",
-                      cgh->cctk_iteration, (double)cgh->cctk_time,
-                      (do_global_mode ? " (global)" : ""),
-                      (do_meta_mode ? " (meta)" : ""));
-          
-            // Checking
-            CalculateChecksums (cgh, allbutcurrenttime);
-            Poison (cgh, currenttimebutnotifonly);
-          
-            // Evolve backward
-            Checkpoint ("Scheduling PRESTEP");
-            CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
-            Checkpoint ("Scheduling EVOL");
-            CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
-          
-            // Checking
-            PoisonCheck (cgh, alltimes);
-          
-            leave_level_mode (cgh);
-          } END_MGLEVEL_LOOP;
-        } // for rl
+        initialise_Flip_Timelevels( cgh, delta_time );
       
-        Waypoint ("Hourglass structure in place");
+        initialise_evolve_3TL_backwards_Ib( cgh, delta_time );
       
-        // Evolve each level "backwards" one more timestep
-        // Starting with the finest level and proceeding to the coarsest
-        for (int rl=reflevels-1; rl>=0; --rl) {
-          BEGIN_MGLEVEL_LOOP(cgh) {
-            enter_level_mode (cgh, rl);
-            do_global_mode = reflevel==0;
-            do_meta_mode = do_global_mode && mglevel==mglevels-1;
-            
-            Waypoint ("Initialisation 3TL evolution II (b) (backwards) at iteration %d time %g%s%s",
-                      cgh->cctk_iteration, (double)cgh->cctk_time,
-                      (do_global_mode ? " (global)" : ""),
-                      (do_meta_mode ? " (meta)" : ""));
-            
-            // Restrict
-            Restrict (cgh);
-            
-            Checkpoint ("Scheduling POSTRESTRICT");
-            CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction);
-            
-            // Poststep
-            Checkpoint ("Scheduling POSTSTEP");
-            CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-          
-            // Checking
-            PoisonCheck (cgh, alltimes);
-          
-            // Advance times
-            for (int m=0; m<maps; ++m) {
-              vtt.at(m)->advance_time (reflevel, mglevel);
-            }
-            cgh->cctk_time
-              = global_time + 2 * delta_time * mglevelfact / reflevelfact;
-            CycleTimeLevels (cgh);
-          
-            Waypoint ("Initialisation 3TL evolution I (c) (backwards) at iteration %d time %g%s%s",
-                      cgh->cctk_iteration, (double)cgh->cctk_time,
-                      (do_global_mode ? " (global)" : ""),
-                      (do_meta_mode ? " (meta)" : ""));
-          
-            // Checking
-            CalculateChecksums (cgh, allbutcurrenttime);
-            Poison (cgh, currenttimebutnotifonly);
-          
-            // Evolve backward
-            Checkpoint ("Scheduling PRESTEP");
-            CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
-            Checkpoint ("Scheduling EVOL");
-            CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
-            Checkpoint ("Scheduling POSTSTEP");
-            CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
-          
-            // Checking
-            PoisonCheck (cgh, alltimes);
-          
-            leave_level_mode (cgh);
-          } END_MGLEVEL_LOOP;
-        } // for rl
+        Waypoint ("Hourglass structure in place");
       
+        initialise_evolve_3TL_backwards_IIb_Ic( cgh, delta_time );
+
         delta_time *= -1;
-        for (int rl=0; rl<reflevels; ++rl) {
-          BEGIN_MGLEVEL_LOOP(cgh) {
-            enter_level_mode (cgh, rl);
-            do_global_mode = reflevel==0;
-            do_meta_mode = do_global_mode && mglevel==mglevels-1;
-          
-            // Flip time levels back
-            Waypoint ("Flipping timelevels back");
-            FlipTimeLevels (cgh);
-          
-            // Invert level times back
-            for (int m=0; m<maps; ++m) {
-              vtt.at(m)->set_delta
-                (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
-              vtt.at(m)->advance_time (reflevel, mglevel);
-              vtt.at(m)->advance_time (reflevel, mglevel);
-              vtt.at(m)->set_delta
-                (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
-            }
-            cgh->cctk_time = global_time;
-          
-            leave_level_mode (cgh);
-          } END_MGLEVEL_LOOP;
-        } // for rl
+        initialise_Flip_Timelevels_back( cgh, delta_time );
       
         Waypoint ("Finished initialising three timelevels");
+
+        return delta_time;
+  }
+
+  void
+  initialise_3_Timelevels( cGH * cgh )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==0;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+    
+        // Advance times
+        for (int m=0; m<maps; ++m) {
+          vtt.at(m)->advance_time (reflevel, mglevel);
+        }
+        cgh->cctk_time
+          = global_time + delta_time * mglevelfact / reflevelfact;
+        CycleTimeLevels (cgh);
+    
+        Waypoint ("Initialisation 3TL evolution I (a) (forwards) at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+    
+        CalculateChecksums (cgh, allbutcurrenttime);
+        Poison (cgh, currenttimebutnotifonly);
+    
+        // Evolve forward
+        Checkpoint ("Scheduling PRESTEP");
+        CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
+        Checkpoint ("Scheduling EVOL");
+        CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
+    
+        PoisonCheck (cgh, currenttime);
+    
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+// SW the delta_time should probably be some CCTK type
+  void
+  initialise_Flip_Timelevels( cGH * cgh, double delta_time )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==0;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+    
+        // Flip time levels
+        Waypoint ("Flipping timelevels");
+        FlipTimeLevels (cgh);
+    
+        cgh->cctk_time
+          = global_time + delta_time * mglevelfact / reflevelfact;
+    
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  void
+  initialise_evolve_3TL_backwards_Ib( cGH * cgh, double delta_time )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==0;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+    
+        Waypoint ("Initialisation 3TL evolution I (b) (backwards) at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+    
+        // Checking
+        CalculateChecksums (cgh, allbutcurrenttime);
+        Poison (cgh, currenttimebutnotifonly);
+    
+        // Evolve backward
+        Checkpoint ("Scheduling PRESTEP");
+        CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
+        Checkpoint ("Scheduling EVOL");
+        CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
+    
+        // Checking
+        PoisonCheck (cgh, alltimes);
+    
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  // Evolve each level "backwards" one more timestep
+  // Starting with the finest level and proceeding to the coarsest
+  void
+  initialise_evolve_3TL_backwards_IIb_Ic( cGH * cgh, double delta_time )
+  { 
+    for (int rl=reflevels-1; rl>=0; --rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==0;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+      
+        Waypoint ("Initialisation 3TL evolution II (b) (backwards) at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+      
+        Restrict (cgh);
+      
+        Checkpoint ("Scheduling POSTRESTRICT");
+        CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction);
       
-      } // if init_3_timelevels
+        Checkpoint ("Scheduling POSTSTEP");
+        CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
     
-    } // if not recovering
+        PoisonCheck (cgh, alltimes);
     
+        // Advance times
+        for (int m=0; m<maps; ++m) {
+          vtt.at(m)->advance_time (reflevel, mglevel);
+        }
+        cgh->cctk_time
+          = global_time + 2 * delta_time * mglevelfact / reflevelfact;
+        CycleTimeLevels (cgh);
     
+        Waypoint ("Initialisation 3TL evolution I (c) (backwards) at iteration %d time %g%s%s",
+                  cgh->cctk_iteration, (double)cgh->cctk_time,
+                  (do_global_mode ? " (global)" : ""),
+                  (do_meta_mode ? " (meta)" : ""));
+    
+        CalculateChecksums (cgh, allbutcurrenttime);
+        Poison (cgh, currenttimebutnotifonly);
+    
+        // Evolve backward
+        Checkpoint ("Scheduling PRESTEP");
+        CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction);
+        Checkpoint ("Scheduling EVOL");
+        CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction);
+        Checkpoint ("Scheduling POSTSTEP");
+        CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction);
     
+        PoisonCheck (cgh, alltimes);
+    
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  void
+  initialise_Flip_Timelevels_back( cGH * cgh, double delta_time )
+  { 
+    for (int rl=0; rl<reflevels; ++rl) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+        enter_level_mode (cgh, rl);
+        do_global_mode = reflevel==0;
+        do_meta_mode = do_global_mode && mglevel==mglevels-1;
+    
+        // Flip time levels back
+        Waypoint ("Flipping timelevels back");
+        FlipTimeLevels (cgh);
+    
+        // Invert level times back
+        for (int m=0; m<maps; ++m) {
+          vtt.at(m)->set_delta
+            (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
+          vtt.at(m)->advance_time (reflevel, mglevel);
+          vtt.at(m)->advance_time (reflevel, mglevel);
+          vtt.at(m)->set_delta
+            (reflevel, mglevel, - vtt.at(m)->get_delta (reflevel, mglevel));
+        }
+        cgh->cctk_time = global_time;
+    
+        leave_level_mode (cgh);
+      } END_MGLEVEL_LOOP;
+    }
+  }
+
+  void
+  initialisation_III( cGH * cgh )
+  { 
     for (int rl=0; rl<reflevels; ++rl) {
       BEGIN_MGLEVEL_LOOP(cgh) {
         enter_level_mode (cgh, rl);
         do_global_mode = reflevel==reflevels-1;
         do_meta_mode = do_global_mode && mglevel==mglevels-1;
-        
+      
         Waypoint ("Initialisation III at iteration %d time %g%s%s",
                   cgh->cctk_iteration, (double)cgh->cctk_time,
                   (do_global_mode ? " (global)" : ""),
                   (do_meta_mode ? " (meta)" : ""));
-        
-        // Checkpoint
+      
         Checkpoint ("Scheduling CPINITIAL");
         CCTK_ScheduleTraverse ("CCTK_CPINITIAL", cgh, CallFunction);
-        
-        // Analysis
+      
         Checkpoint ("Scheduling ANALYSIS");
         CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction);
-        
-        // Output
+      
         Checkpoint ("OutputGH");
         CCTK_OutputGH (cgh);
-        
-        // Checking
+      
         PoisonCheck (cgh, alltimes);
         CheckChecksums (cgh, allbutcurrenttime);
-        
+      
         leave_level_mode (cgh);
       } END_MGLEVEL_LOOP;
-    } // for rl
-    
-    
-      
-    Waypoint ("Done with initialisation");
-    
-    return 0;
+    }
   }
-  
+
 } // namespace Carpet
diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc
index 2991e8200..ba810c2da 100644
--- a/Carpet/Carpet/src/Recompose.cc
+++ b/Carpet/Carpet/src/Recompose.cc
@@ -366,15 +366,19 @@ namespace Carpet {
       const int zstep = locnpz * cstr[dir];
       clb[dir] = rlb[dir] + zstep *  c;
       cub[dir] = rlb[dir] + zstep * (c+1);
-      if (clb[dir] > rub[dir]) clb[dir] = rub[dir];
-      if (cub[dir] > rub[dir]) cub[dir] = rub[dir];
+      if (clb[dir] > rub[dir])
+        clb[dir] = rub[dir];
+      if (cub[dir] > rub[dir])
+        cub[dir] = rub[dir];
       assert (clb[dir] <= cub[dir]);
       assert (cub[dir] <= rub[dir]);
       bbs.at(c) = ibbox(clb, cub-cstr, cstr);
       obs.at(c) = obnd;
       ps.at(c) = c;
-      if (c>0)        obs.at(c)[dir][0] = false;
-      if (c<nprocs-1) obs.at(c)[dir][1] = false;
+      if (c>0)
+        obs.at(c)[dir][0] = false;
+      if (c<nprocs-1)
+        obs.at(c)[dir][1] = false;
     }
     
     for (int n=0; n<(int)ps.size(); ++n) {
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index b2793943c..2684dd1ce 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -136,8 +136,7 @@ namespace Carpet {
     // not reached
     return false;
   }
-  
-  
+
   
   static operator_type GetTransportOperator (const cGH * const cgh,
                                              const int group)
diff --git a/Carpet/CarpetLib/src/FCALL_proto.h b/Carpet/CarpetLib/src/FCALL_proto.h
new file mode 100644
index 000000000..defdabc6f
--- /dev/null
+++ b/Carpet/CarpetLib/src/FCALL_proto.h
@@ -0,0 +1,248 @@
+#ifndef FCALL_PROTO_H
+#define FCALL_PROTO_H
+
+#include "cctk.h"
+
+extern "C" {
+  void CCTK_FCALL CCTK_FNAME(copy_3d_int4)
+    (const CCTK_INT4* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_INT4* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(copy_3d_real8)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(copy_3d_complex16)
+    (const CCTK_COMPLEX16* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_COMPLEX16* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+}
+
+extern "C" {
+  
+  void CCTK_FCALL CCTK_FNAME(restrict_3d_real8)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(restrict_3d_real8_rf2)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  
+  
+  
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_rf2)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_o3)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_o3_rf2)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_minmod)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_eno)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_o5)
+    (const CCTK_REAL8* src,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_rf2)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_o3)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_o3_rf2)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_minmod)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_eno)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_o5)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_rf2)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_o3)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_o3_rf2)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_minmod)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_eno)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+  void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_3tl_o5)
+    (const CCTK_REAL8* src1, const CCTK_REAL8& t1,
+     const CCTK_REAL8* src2, const CCTK_REAL8& t2,
+     const CCTK_REAL8* src3, const CCTK_REAL8& t3,
+     const int& srciext, const int& srcjext, const int& srckext,
+     CCTK_REAL8* dst, const CCTK_REAL8& t,
+     const int& dstiext, const int& dstjext, const int& dstkext,
+     const int srcbbox[3][3],
+     const int dstbbox[3][3],
+     const int regbbox[3][3]);
+}
+
+#endif
+
diff --git a/Carpet/CarpetLib/src/ggf.cc b/Carpet/CarpetLib/src/ggf.cc
index 6f8a282fa..9461ba21e 100644
--- a/Carpet/CarpetLib/src/ggf.cc
+++ b/Carpet/CarpetLib/src/ggf.cc
@@ -94,13 +94,16 @@ void ggf<D>::recompose_allocate (const int rl)
   // Resize structure and allocate storage
   storage.resize(tmax-tmin+1);
   for (int tl=tmin; tl<=tmax; ++tl) {
-    storage.at(tl-tmin).resize(h.reflevels());
-    storage.at(tl-tmin).at(rl).resize(h.components(rl));
+    rdata &rd = storage.at(tl-tmin);
+    rd.resize(h.reflevels());
+    cdata &cd = rd.at(rl);
+    cd.resize(h.components(rl));
     for (int c=0; c<h.components(rl); ++c) {
-      storage.at(tl-tmin).at(rl).at(c).resize(h.mglevels(rl,c));
+      mdata &md = cd.at(c);
+      md.resize(h.mglevels(rl,c));
       for (int ml=0; ml<h.mglevels(rl,c); ++ml) {
-        storage.at(tl-tmin).at(rl).at(c).at(ml) = typed_data(tl,rl,c,ml);
-        storage.at(tl-tmin).at(rl).at(c).at(ml)->allocate
+        md.at(ml) = typed_data(tl,rl,c,ml);
+        md.at(ml)->allocate
           (d.boxes.at(rl).at(c).at(ml).exterior, h.proc(rl,c));
       } // for ml
     } // for c
diff --git a/Carpet/CarpetRegrid/src/regrid.cc b/Carpet/CarpetRegrid/src/regrid.cc
index 1dd1e2a1d..a3de7c44d 100644
--- a/Carpet/CarpetRegrid/src/regrid.cc
+++ b/Carpet/CarpetRegrid/src/regrid.cc
@@ -163,23 +163,19 @@ namespace CarpetRegrid {
                  
     } else if (CCTK_EQUALS(refined_regions, "manual-gridpoints")) {
       
-      do_recompose
-        = ManualGridpoints (cctkGH, hh, bbsss, obss, pss);
+      do_recompose = ManualGridpoints (cctkGH, hh, bbsss, obss, pss);
                  
     } else if (CCTK_EQUALS(refined_regions, "manual-coordinates")) {
       
-      do_recompose
-        = ManualCoordinates (cctkGH, hh, bbsss, obss, pss);
+      do_recompose = ManualCoordinates (cctkGH, hh, bbsss, obss, pss);
                  
     } else if (CCTK_EQUALS(refined_regions, "manual-gridpoint-list")) {
       
-      do_recompose
-        = ManualGridpointList (cctkGH, hh, bbsss, obss, pss);
+      do_recompose = ManualGridpointList (cctkGH, hh, bbsss, obss, pss);
                  
     } else if (CCTK_EQUALS(refined_regions, "manual-coordinate-list")) {
       
-      do_recompose
-        = ManualCoordinateList (cctkGH, hh, bbsss, obss, pss);
+      do_recompose = ManualCoordinateList (cctkGH, hh, bbsss, obss, pss);
                  
     } else if (CCTK_EQUALS(refined_regions, "moving")) {