CarpetLib: Various changes to class gf

Update to use new communication mechanism.

darcs-hash:20070419014851-dae7b-313adb8952e62048a096bba95023c99be48e9658.gz

2 files changed, 252 insertions, 465 deletions

diff --git a/Carpet/CarpetLib/src/ggf.cc b/Carpet/CarpetLib/src/ggf.cc
index 979e05b54..34cd80374 100644
--- a/Carpet/CarpetLib/src/ggf.cc
+++ b/Carpet/CarpetLib/src/ggf.cc
@@ -96,6 +96,9 @@ void ggf::set_timelevels (const int ml, const int rl, const int new_timelevels)
 void ggf::recompose_crop ()
 {
   // Free storage that will not be needed
+  static Timer timer ("ggf::recompose_crop");
+  timer.start ();
+  
   for (int ml=0; ml<h.mglevels(); ++ml) {
     for (int rl=h.reflevels(); rl<(int)storage.AT(ml).size(); ++rl) {
       for (int c=0; c<(int)storage.AT(ml).AT(rl).size(); ++c) {
@@ -106,16 +109,21 @@ void ggf::recompose_crop ()
     } // for rl
     storage.AT(ml).resize(h.reflevels());
   } // for ml
+  
+  timer.stop (0);
 }
 
 void ggf::recompose_allocate (const int rl)
 {
   // Retain storage that might be needed
+  static Timer timer ("ggf::recompose_allocate");
+  timer.start ();
+  
   oldstorage.resize(storage.size());
   for (int ml=0; ml<(int)storage.size(); ++ml) {
     oldstorage.AT(ml).resize(storage.AT(ml).size());
     oldstorage.AT(ml).AT(rl) = storage.AT(ml).AT(rl);
-    storage.AT(ml).AT(rl).resize(0);
+    storage.AT(ml).AT(rl).clear();
   }
   
   for (int ml=0; ml<d.h.mglevels(); ++ml) {
@@ -136,20 +144,29 @@ void ggf::recompose_allocate (const int rl)
       } // for tl
     } // for c
   } // for ml
+  
+  timer.stop (0);
 }
 
 void ggf::recompose_fill (comm_state & state, int const rl,
                           bool const do_prolongate)
 {
   // Initialise the new storage
+  static Timer timer ("ggf::recompose_fill");
+  timer.start ();
+
   for (int ml = 0; ml < h.mglevels(); ++ ml) {
     
-    int const numtl = timelevels (ml, rl);
-    vector <int> tls (numtl);
-    vector <CCTK_REAL> times (numtl);
-    for (int tl = 0; tl < numtl; ++ tl) {
-      tls.AT(tl) = tl;
-      times.AT(tl) = t.time (tls.AT(tl), rl - 1, ml);
+    vector <int> tls;
+    vector <CCTK_REAL> times;
+    if (do_prolongate and rl > 0 and transport_operator != op_none) {
+      int const numtl = timelevels (ml, rl);
+      tls.resize (numtl);
+      times.resize (numtl);
+      for (int tl = 0; tl < numtl; ++ tl) {
+        tls.AT(tl) = tl;
+        times.AT(tl) = t.time (tls.AT(tl), rl - 1, ml);
+      }
     }
     
     for (int c = 0; c < h.components (rl); ++c) {
@@ -158,11 +175,12 @@ void ggf::recompose_fill (comm_state & state, int const rl,
       // possible
       if (rl < (int)oldstorage.AT(ml).size()) {
         for (int tl = 0; tl < timelevels (ml, rl); ++tl) {
-          copycat (state,
-                   tl, rl, c, ml,
-                   & dh::dboxes::old2new_recv_sync_fast,
-                   tl, rl, ml,
-                   & oldstorage);
+          transfer_from (state,
+                         tl, rl, c, ml,
+                         & dh::dboxes::fast_old2new_sync_recv,
+                         & dh::dboxes::fast_old2new_sync_send,
+                         tl, rl, ml,
+                         & oldstorage);
         } // for tl
       } // if rl
       
@@ -172,11 +190,12 @@ void ggf::recompose_fill (comm_state & state, int const rl,
         if (rl > 0) {
           if (transport_operator != op_none) {
             for (int tl = 0; tl < timelevels (ml, rl); ++tl) {
-              intercat (state,
-                        tl, rl, c, ml,
-                        & dh::dboxes::old2new_recv_ref_coarse_fast,
-                        tls, rl - 1, ml,
-                        times.AT(tl));
+              transfer_from (state,
+                             tl, rl, c, ml,
+                             & dh::dboxes::fast_old2new_ref_prol_recv,
+                             & dh::dboxes::fast_old2new_ref_prol_send,
+                             tls, rl - 1, ml,
+                             times.AT(tl));
             } // for tl
           } // if transport_operator
         } // if rl
@@ -189,61 +208,28 @@ void ggf::recompose_fill (comm_state & state, int const rl,
       
     } // for c
   } // for ml
+  
+  timer.stop (0);
 }
 
 void ggf::recompose_free_old (const int rl)
 {
   // Delete old storage
+  static Timer timer ("dh::recompose");
+  timer.start ();
+
   for (int ml=0; ml<(int)oldstorage.size(); ++ml) {
     for (int c=0; c<(int)oldstorage.AT(ml).AT(rl).size(); ++c) {
       for (int tl=0; tl<timelevels(ml,rl); ++tl) {
         delete oldstorage.AT(ml).AT(rl).AT(c).AT(tl);
       } // for tl
     } // for c
-    oldstorage.AT(ml).AT(rl).resize(0);
+    oldstorage.AT(ml).AT(rl).clear();
   } // for ml
+  
+  timer.stop (0);
 }
 
-#if 0
-void ggf::recompose_bnd_prolongate (comm_state& state, const int rl,
-                                    const bool do_prolongate)
-{
-  if (do_prolongate) {
-    // Set boundaries
-    if (rl>0) {
-      for (int ml=0; ml<h.mglevels(); ++ml) {
-        for (int c=0; c<h.components(rl); ++c) {
-          for (int tl=0; tl<timelevels(ml,rl); ++tl) {
-            
-            // TODO: assert that reflevel 0 boundaries are copied
-            const CCTK_REAL time = t.time(tl,rl,ml);
-            ref_bnd_prolongate (state,tl,rl,c,ml,time);
-            
-          } // for tl
-        } // for c
-      } // for ml
-    } // if rl
-  } // if do_prolongate
-}
-
-void ggf::recompose_sync (comm_state& state, const int rl,
-                          const bool do_prolongate)
-{
-  if (do_prolongate) {
-    // Set boundaries
-    for (int ml=0; ml<h.mglevels(); ++ml) {
-      for (int c=0; c<h.components(rl); ++c) {
-        for (int tl=0; tl<timelevels(ml,rl); ++tl) {
-          
-          sync (state,tl,rl,c,ml);
-          
-        } // for tl
-      } // for c
-    } // for ml
-  } // if do_prolongate
-}
-#endif
-
 
 
 // Cycle the time levels by rotating the data sets
@@ -278,322 +264,47 @@ void ggf::flip (int rl, int c, int ml) {
 
 
 
-// Operations
-
-// Copy a region
-void ggf::copycat (comm_state& state,
-                   int tl1, int rl1, int c1, int ml1,
-                   const ibbox dh::dboxes::* recv_box,
-                   int tl2, int rl2, int ml2)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (rl2>=0 and rl2<h.reflevels());
-  int const c2= c1;
-  assert (ml2<h.mglevels());
-  assert (tl2>=0 and tl2<timelevels(ml2,rl2));
-  static Timer copycat1 ("copycat_1");
-  copycat1.start ();
-  ibbox const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_box;
-  // copy the content
-  gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
-  gdata const * const src = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2);
-  dst->copy_from(state, src, recv);
-  copycat1.stop (0);
-}
-
-// Copy regions
-void ggf::copycat (comm_state& state,
-                   int tl1, int rl1, int c1, int ml1,
-                   const iblist dh::dboxes::* recv_list,
-                   int tl2, int rl2, int ml2)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (           ml2<h.mglevels());
-  assert (rl2>=0 and rl2<h.reflevels());
-  int const c2 = c1;
-  assert (tl2>=0 and tl2<timelevels(ml2,rl2));
-  static Timer copycat1 ("copycat_list_1");
-  copycat1.start ();
-  iblist const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
-  // walk all boxes
-  for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r) {
-    // (use the send boxes for communication)
-    // copy the content
-    gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
-    gdata const * const src = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2);
-    dst->copy_from(state, src, *r);
-  }
-  copycat1.stop (0);
-}
-
-// Copy regions
-void ggf::copycat (comm_state& state,
-                   int tl1, int rl1, int c1, int ml1,
-                   const iblistvect dh::dboxes::* recv_listvect,
-                   int tl2, int rl2, int ml2)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (           ml2<h.mglevels());
-  assert (rl2>=0 and rl2<h.reflevels());
-  assert (tl2>=0 and tl2<timelevels(ml2,rl2));
-  // walk all components
-  static Timer copycat1 ("copycat_listvect_1");
-  copycat1.start ();
-  iblistvect::const_iterator recvi =
-    (d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_listvect).begin();
-  gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
-  cdata::const_iterator srci = storage.AT(ml2).AT(rl2).begin();
-  int const nc = h.components(rl2);
-  for (int c2=0; c2<nc; ++c2) {
-    iblist const & recv = (*recvi);
-    gdata const * const src = (*srci).AT(tl2);
-    // walk all boxes
-    for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r) {
-      // (use the send boxes for communication)
-      // copy the content
-      dst->copy_from(state, src, *r);
-    }
-    ++ recvi;
-    ++ srci;
-  }
-  copycat1.stop (0);
-}
-
-// Copy regions
-void ggf::copycat (comm_state& state,
-                   int tl1, int rl1, int c1, int ml1,
-                   const pvect dh::dboxes::* recv_pvect,
-                   int tl2, int rl2, int ml2,
-                   mdata * const srcstorage_)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  mdata & srcstorage = srcstorage_ ? * srcstorage_ : storage;
-  // Uses wrong grid hierarchy; should use "old h" instead
-  // TODO: keep old grid hierarchy around
-  //assert (           ml2<h.mglevels());
-  //assert (rl2>=0 and rl2<h.reflevels());
-  //assert (tl2>=0 and tl2<timelevels(ml2,rl2));
-  // Cannot use srcstorage, since it may be empty when *recv_pvect is empty
-  //assert (           ml2<(int)srcstorage.size());
-  //assert (rl2>=0 and rl2<(int)srcstorage.AT(ml2).size());
-  //assert (tl2>=0 and tl2<(int)srcstorage.AT(ml2).AT(rl2).AT(0).size());
-  // walk all components
-  static Timer copycat1 ("copycat_pvect_1");
-  copycat1.start ();
-  gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
-  cdata const & srcs = srcstorage.AT(ml2).AT(rl2);
-  pvect const & prs = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_pvect;
-  for (pvect::const_iterator ipr=prs.begin(); ipr!=prs.end(); ++ipr) {
-    pseudoregion const & pr = * ipr;
-    ibbox const & r = pr.extent;
-    int const c2 = pr.processor;
-    gdata const * const src = srcs.AT(c2).AT(tl2);
-    dst->copy_from(state, src, r);
-  }
-  copycat1.stop (0);
-}
-
-// Interpolate a region
-void ggf::intercat (comm_state& state,
-                    int tl1, int rl1, int c1, int ml1,
-                    const ibbox dh::dboxes::* recv_list,
-                    const vector<int> tl2s, int rl2, int ml2,
-                    CCTK_REAL time)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (rl2>=0 and rl2<h.reflevels());
-  int const c2 = c1;
-  assert (ml2>=0 and ml2<h.mglevels());
-  for (int i=0; i<(int)tl2s.size(); ++i) {
-    assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
-  }
-  
-  vector<const gdata*> gsrcs(tl2s.size());
-  vector<CCTK_REAL> times(tl2s.size());
-  for (int i=0; i<(int)gsrcs.size(); ++i) {
-    gsrcs.AT(i) = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2s.AT(i));
-    times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
-  }
-  
-  static Timer intercat1 ("intercat_1");
-  intercat1.start ();
-  ibbox const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
-  // interpolate the content
-  storage.AT(ml1).AT(rl1).AT(c1).AT(tl1)->interpolate_from
-    (state, gsrcs, times, recv, time,
-     d.prolongation_order_space, prolongation_order_time);
-  intercat1.stop (0);
-}
-
-// Interpolate regions
-void ggf::intercat (comm_state& state,
-                    int tl1, int rl1, int c1, int ml1,
-                    const iblist dh::dboxes::* recv_list,
-                    const vector<int> tl2s, int rl2, int ml2,
-                    const CCTK_REAL time)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (rl2>=0 and rl2<h.reflevels());
-  int const c2 = c1;
-  assert (ml2>=0 and ml2<h.mglevels());
-  for (int i=0; i<(int)tl2s.size(); ++i) {
-    assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
-  }
-  
-  vector<const gdata*> gsrcs(tl2s.size());
-  vector<CCTK_REAL> times(tl2s.size());
-  for (int i=0; i<(int)gsrcs.size(); ++i) {
-    gsrcs.AT(i) = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2s.AT(i));
-    times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
-  }
-  
-  static Timer intercat1 ("intercat_list_1");
-  intercat1.start ();
-  iblist const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
-  // walk all boxes
-  for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r)
-  {
-    // (use the send boxes for communication)
-    // interpolate the content
-    storage.AT(ml1).AT(rl1).AT(c1).AT(tl1)->interpolate_from
-      (state, gsrcs, times, *r, time,
-       d.prolongation_order_space, prolongation_order_time);
-  }
-  intercat1.stop (0);
-}
-
-// Interpolate regions
-void ggf::intercat (comm_state& state,
-                    int tl1, int rl1, int c1, int ml1,
-                    const iblistvect dh::dboxes::* recv_listvect,
-                    const vector<int> tl2s, int rl2, int ml2,
-                    const CCTK_REAL time)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (rl2>=0 and rl2<h.reflevels());
-  for (int i=0; i<(int)tl2s.size(); ++i) {
-    assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
-  }
-  // walk all components
-  static Timer intercat1 ("intercat_listvect_1");
-  intercat1.start ();
-  for (int c2=0; c2<h.components(rl2); ++c2) {
-    assert (ml2>=0 and ml2<h.mglevels());
-    
-    vector<const gdata*> gsrcs(tl2s.size());
-    vector<CCTK_REAL> times(tl2s.size());
-    for (int i=0; i<(int)gsrcs.size(); ++i) {
-      gsrcs.AT(i) = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2s.AT(i));
-      times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
-    }
-    
-    iblist const &recv =
-      (d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_listvect).AT(c2);
-    // walk all boxes
-    for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r)
-    {
-      // (use the send boxes for communication)
-      // interpolate the content
-      int const pos = d.prolongation_order_space;
-      int const pot =
-        transport_operator == op_copy ? 0 : prolongation_order_time;
-      storage.AT(ml1).AT(rl1).AT(c1).AT(tl1)->interpolate_from
-      	(state, gsrcs, times, *r, time, pos, pot);
-    }
-  }
-  intercat1.stop (0);
-}
-
-// Interpolate regions
-void ggf::intercat (comm_state& state,
-                    int tl1, int rl1, int c1, int ml1,
-                    const pvect dh::dboxes::* recv_pvect,
-                    const vector<int> tl2s, int rl2, int ml2,
-                    const CCTK_REAL time)
-{
-  assert (rl1>=0 and rl1<h.reflevels());
-  assert (c1>=0 and c1<h.components(rl1));
-  assert (ml1>=0 and ml1<h.mglevels());
-  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
-  assert (rl2>=0 and rl2<h.reflevels());
-  assert (ml2>=0 and ml2<h.mglevels());
-  vector<CCTK_REAL> times(tl2s.size());
-  for (int i=0; i<(int)tl2s.size(); ++i) {
-    assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
-    times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
-  }
-  // walk all components
-  static Timer intercat1 ("intercat_pvect_1");
-  intercat1.start ();
-  int const pos = d.prolongation_order_space;
-  int const pot = transport_operator == op_copy ? 0 : prolongation_order_time;
-  gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
-  cdata const & srcs = storage.AT(ml2).AT(rl2);
-  vector<const gdata*> gsrcs(tl2s.size());
-  pvect const & prs = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_pvect;
-  for (pvect::const_iterator ipr=prs.begin(); ipr!=prs.end(); ++ipr) {
-    pseudoregion const & pr = * ipr;
-    ibbox const & r = pr.extent;
-    int const c2 = pr.processor;
-    for (int i=0; i<(int)gsrcs.size(); ++i) {
-      gsrcs.AT(i) = srcs.AT(c2).AT(tl2s.AT(i));
-    }
-    dst->interpolate_from (state, gsrcs, times, r, time, pos, pot);
-  }
-  intercat1.stop (0);
-}
-
-
-
+#if 0
 // Copy a component from the next time level
 void ggf::copy (comm_state& state, int tl, int rl, int c, int ml)
 {
   // Copy
   static Timer timer ("copy");
   timer.start ();
-  copycat (state,
-           tl  ,rl,c,ml, &dh::dboxes::exterior,
-      	   tl+1,rl,  ml);
+  transfer_from (state,
+                 tl  ,rl,c,ml, &dh::dboxes::exterior,
+                 tl+1,rl,  ml);
   timer.stop (0);
 }
+#endif
+
+
 
 // Synchronise the boundaries a component
-void ggf::sync (comm_state& state, int tl, int rl, int c, int ml)
+void
+ggf::
+sync (comm_state & state,
+      int const tl, int const rl, int const c, int const ml)
 {
   // Copy
   static Timer timer ("sync");
   timer.start ();
-  copycat (state,
-           tl,rl,c,ml, &dh::dboxes::recv_sync_fast,
-      	   tl,rl,  ml);
+  transfer_from (state,
+                 tl,rl,c,ml,
+                 & dh::dboxes::fast_sync_recv,
+                 & dh::dboxes::fast_sync_send,
+                 tl,rl,  ml);
   timer.stop (0);
 }
 
+
+
 // Prolongate the boundaries of a component
-void ggf::ref_bnd_prolongate (comm_state& state, 
-                              int tl, int rl, int c, int ml,
-                              CCTK_REAL time)
+void
+ggf::
+ref_bnd_prolongate (comm_state & state, 
+                    int const tl, int const rl, int const c, int const ml,
+                    CCTK_REAL const time)
 {
   // Interpolate
   assert (rl>=1);
@@ -619,52 +330,73 @@ void ggf::ref_bnd_prolongate (comm_state& state,
     tl2s.resize(1);
     tl2s.AT(0) = 0;
   }
-  intercat (state,
-            tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_bnd_coarse_fast,
-            tl2s,rl-1,  ml, time);
+  transfer_from (state,
+                 tl  ,rl  ,c,ml,
+                 & dh::dboxes::fast_ref_bnd_prol_recv,
+                 & dh::dboxes::fast_ref_bnd_prol_send,
+                 tl2s,rl-1,  ml,
+                 time);
   timer.stop (0);
 }
 
+
+
 // Restrict a multigrid level
-void ggf::mg_restrict (comm_state& state,
-                       int tl, int rl, int c, int ml,
-                       CCTK_REAL time)
+void
+ggf::
+mg_restrict (comm_state & state,
+             int const tl, int const rl, int const c, int const ml,
+             CCTK_REAL const time)
 {
+  static Timer timer ("mg_restrict");
+  timer.start ();
   // Require same times
   static_assert (abs(0.1) > 0, "Function CarpetLib::abs has wrong signature");
   assert (abs(t.get_time(rl,ml) - t.get_time(rl,ml-1))
 	  <= 1.0e-8 * abs(t.get_time(rl,ml)));
-  const vector<int> tl2s(1,tl);
-  static Timer timer ("mg_restrict");
-  timer.start ();
-  intercat (state,
-            tl  ,rl,c,ml,   &dh::dboxes::recv_mg_coarse,
-	    tl2s,rl,  ml-1, time);
+  vector<int> const tl2s(1,tl);
+  transfer_from (state,
+                 tl  ,rl,c,ml,
+                 & dh::dboxes::fast_mg_rest_recv,
+                 & dh::dboxes::fast_mg_rest_send,
+                 tl2s,rl,  ml-1,
+                 time);
   timer.stop (0);
 }
 
+
+
 // Prolongate a multigrid level
-void ggf::mg_prolongate (comm_state& state,
-                         int tl, int rl, int c, int ml,
-                         CCTK_REAL time)
+void
+ggf::
+mg_prolongate (comm_state & state,
+               int const tl, int const rl, int const c, int const ml,
+               CCTK_REAL const time)
 {
+  static Timer timer ("mg_prolongate");
+  timer.start ();
   // Require same times
   static_assert (abs(0.1) > 0, "Function CarpetLib::abs has wrong signature");
   assert (abs(t.get_time(rl,ml) - t.get_time(rl,ml+1))
 	  <= 1.0e-8 * abs(t.get_time(rl,ml)));
-  static Timer timer ("mg_prolongate");
-  timer.start ();
-  const vector<int> tl2s(1,tl);
-  intercat (state,
-            tl  ,rl,c,ml,   &dh::dboxes::recv_mg_coarse,
-	    tl2s,rl,  ml+1, time);
+  vector<int> const tl2s(1,tl);
+  transfer_from (state,
+                 tl  ,rl,c,ml,
+                 & dh::dboxes::fast_mg_prol_recv,
+                 & dh::dboxes::fast_mg_prol_send,
+                 tl2s,rl,  ml+1,
+                 time);
   timer.stop (0);
 }
 
+
+
 // Restrict a refinement level
-void ggf::ref_restrict (comm_state& state,
-                        int tl, int rl, int c, int ml,
-                        CCTK_REAL time)
+void
+ggf::
+ref_restrict (comm_state & state,
+              int const tl, int const rl, int const c, int const ml,
+              CCTK_REAL const time)
 {
   // Require same times
   static_assert (abs(0.1) > 0, "Function CarpetLib::abs has wrong signature");
@@ -674,16 +406,23 @@ void ggf::ref_restrict (comm_state& state,
   static Timer timer ("ref_restrict");
   timer.start ();
   vector<int> const tl2s(1,tl);
-  intercat (state,
-            tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_fine_fast,
-	    tl2s,rl+1,  ml, time);
+  transfer_from (state,
+                 tl  ,rl  ,c,ml,
+                 & dh::dboxes::fast_ref_rest_recv,
+                 & dh::dboxes::fast_ref_rest_send,
+                 tl2s,rl+1,  ml,
+                 time);
   timer.stop (0);
 }
 
+
+
 // Prolongate a refinement level
-void ggf::ref_prolongate (comm_state& state,
-                          int tl, int rl, int c, int ml,
-                          CCTK_REAL time)
+void
+ggf::
+ref_prolongate (comm_state & state,
+                int const tl, int const rl, int const c, int const ml,
+                CCTK_REAL const time)
 {
   assert (rl>=1);
   if (transport_operator == op_none) return;
@@ -694,8 +433,88 @@ void ggf::ref_prolongate (comm_state& state,
   assert (timelevels(ml,rl) >= prolongation_order_time+1);
   tl2s.resize(prolongation_order_time+1);
   for (int i=0; i<=prolongation_order_time; ++i) tl2s.AT(i) = i;
-  intercat (state,
-            tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_coarse_fast,
-	    tl2s,rl-1,  ml, time);
+  transfer_from (state,
+                 tl  ,rl  ,c,ml,
+                 & dh::dboxes::fast_ref_prol_recv,
+                 & dh::dboxes::fast_ref_prol_send,
+                 tl2s,rl-1,  ml,
+                 time);
   timer.stop (0);
 }
+
+
+
+// Transfer regions
+void
+ggf::
+transfer_from (comm_state & state,
+               int const tl1, int const rl1, int const c1, int const ml1,
+               pvect const dh::dboxes::* recvs,
+               pvect const dh::dboxes::* sends,
+               vector<int> const & tl2s, int const rl2, int const ml2,
+               CCTK_REAL const & time,
+               mdata * const srcstorage_)
+{
+  assert (rl1>=0 and rl1<h.reflevels());
+  assert (c1>=0 and c1<h.components(rl1));
+  assert (ml1>=0 and ml1<h.mglevels());
+  assert (tl1>=0 and tl1<timelevels(ml1,rl1));
+  
+  pvect const & precvs = d.boxes.AT(ml1).AT(rl1).AT(c1).*recvs;
+  pvect const & psends = d.boxes.AT(ml1).AT(rl1).AT(c1).*sends;
+  assert (precvs.size() == psends.size());
+  
+  mdata & srcstorage = srcstorage_ ? * srcstorage_ : storage;
+  
+  if (not precvs.empty()) {
+    // Check only when *recvs is not empty
+    assert (           ml2<(int)srcstorage.size());
+    assert (rl2>=0 and rl2<(int)srcstorage.AT(ml2).size());
+    for (size_t i = 0; i < tl2s.size(); ++ i) {
+      int const tl2 = tl2s.AT(i);
+      assert (tl2>=0 and tl2<(int)srcstorage.AT(ml2).AT(rl2).AT(0).size());
+    }
+  }
+  
+  static Timer total ("transfer_from");
+  total.start ();
+  
+  // Interpolation orders
+  int const pos =
+    transport_operator == op_copy ? 0 : d.prolongation_order_space;
+  int const pot =
+    transport_operator == op_copy ? 0 : prolongation_order_time;
+  
+  // Source and destination data
+  gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
+  cdata const & srcs = srcstorage.AT(ml2).AT(rl2);
+  vector<const gdata*> gsrcs(tl2s.size());
+  
+  // Set up source times
+  vector<CCTK_REAL> times(tl2s.size());
+  for (size_t i=0; i<tl2s.size(); ++i) {
+    assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
+    for (size_t j=0; j<i; ++j) {
+      assert (tl2s.AT(i) != tl2s.AT(j));
+    }
+    times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
+  }
+  
+  // Walk all regions
+  for (pvect::const_iterator iprecv=precvs.begin(), ipsend=psends.begin();
+       iprecv!=precvs.end(); ++ iprecv, ++ ipsend)
+  {
+    pseudoregion const & precv = * iprecv;
+    pseudoregion const & psend = * ipsend;
+    ibbox const & recv = precv.extent;
+    ibbox const & send = psend.extent;
+    int const c2 = precv.processor;
+    assert (psend.processor == c2);
+    for (int i=0; i<(int)gsrcs.size(); ++i) {
+      gsrcs.AT(i) = srcs.AT(c2).AT(tl2s.AT(i));
+    }
+    dst->transfer_from (state, gsrcs, times, recv, send, time, pos, pot);
+  }
+  
+  total.stop (0);
+}
diff --git a/Carpet/CarpetLib/src/ggf.hh b/Carpet/CarpetLib/src/ggf.hh
index 8908c30f7..624b51ef5 100644
--- a/Carpet/CarpetLib/src/ggf.hh
+++ b/Carpet/CarpetLib/src/ggf.hh
@@ -64,7 +64,7 @@ public:
   const ggf* vectorleader;      // first vector element
   
 private:
-  mdata oldstorage;            // temporary storage
+  mdata oldstorage;             // temporary storage
   
 public:
 
@@ -96,87 +96,12 @@ public:
   void recompose_allocate (int rl);
   void recompose_fill (comm_state& state, int rl, bool do_prolongate);
   void recompose_free_old (int rl);
-#if 0
-  void recompose_bnd_prolongate (comm_state& state, int rl, bool do_prolongate);
-  void recompose_sync (comm_state& state, int rl, bool do_prolongate);
-#endif
 
   // Cycle the time levels by rotating the data sets
   void cycle (int rl, int c, int ml);
   
   // Flip the time levels by exchanging the data sets
   void flip (int rl, int c, int ml);
-  
-  
-  
-  // Helpers
-  
-protected:
-  
-  virtual gdata* typed_data (int tl, int rl, int c, int ml) = 0;
-  
-  
-  
-  // Operations
-  
-protected:
-  
-  // Copy a region
-  void copycat (comm_state& state,
-                int tl1, int rl1, int c1, int ml1,
-		const ibbox dh::dboxes::* recv_list,
-		int tl2, int rl2, int ml2);
-
-  // Copy regions
-  void copycat (comm_state& state,
-                int tl1, int rl1, int c1, int ml1,
-		const iblist dh::dboxes::* recv_list,
-		int tl2, int rl2, int ml2);
-
-  // Copy regions
-  void copycat (comm_state& state,
-                int tl1, int rl1, int c1, int ml1,
-		const iblistvect dh::dboxes::* recv_listvect,
-		int tl2, int rl2, int ml2);
-
-  // Copy regions
-  void copycat (comm_state& state,
-                int tl1, int rl1, int c1, int ml1,
-		const pvect dh::dboxes::* recv_pvect,
-                int tl2, int rl2, int ml2,
-                mdata * srcstorage = 0);
-  
-  // Interpolate a region
-  void intercat (comm_state& state,
-                 int tl1, int rl1, int c1, int ml1,
-		 const ibbox dh::dboxes::* recv_list,
-		 const vector<int> tl2s, int rl2, int ml2,
-		 CCTK_REAL time);
-
-  // Interpolate regions
-  void intercat (comm_state& state,
-                 int tl1, int rl1, int c1, int ml1,
-		 const iblist dh::dboxes::* recv_list,
-		 const vector<int> tl2s, int rl2, int ml2,
-		 CCTK_REAL time);
-
-  // Interpolate regions
-  void intercat (comm_state& state,
-                 int tl1, int rl1, int c1, int ml1,
-		 const iblistvect dh::dboxes::* recv_listvect,
-		 const vector<int> tl2s, int rl2, int ml2,
-		 CCTK_REAL time);
-
-  // Interpolate regions
-  void intercat (comm_state& state,
-                 int tl1, int rl1, int c1, int ml1,
-		 const pvect dh::dboxes::* recv_pvect,
-		 const vector<int> tl2s, int rl2, int ml2,
-		 CCTK_REAL time);
-
-
-
-public:
 
   // The grid boundaries have to be updated after calling mg_restrict,
   // mg_prolongate, ref_restrict, or ref_prolongate.
@@ -212,6 +137,49 @@ public:
   
   
   
+  // Helpers
+  
+protected:
+  
+  virtual gdata* typed_data (int tl, int rl, int c, int ml) = 0;
+  
+  
+  
+protected:
+  
+  // Transfer regions
+  void
+  transfer_from (comm_state & state,
+                 int tl1, int rl1, int c1, int ml1,
+                 pvect const dh::dboxes::* recvs,
+                 pvect const dh::dboxes::* sends,
+                 vector<int> const & tl2s, int rl2, int ml2,
+                 CCTK_REAL const & time,
+                 mdata * srcstorage = 0);
+  
+  void
+  transfer_from (comm_state & state,
+                 int tl1, int rl1, int c1, int ml1,
+                 pvect const dh::dboxes::* recvs,
+                 pvect const dh::dboxes::* sends,
+                 int tl2, int rl2, int ml2,
+                 mdata * srcstorage = 0)
+  {
+    vector <int> tl2s(1);
+    tl2s.AT(0) = tl2;
+    CCTK_REAL const time = t.time (tl2,rl2,ml2);
+    transfer_from (state,
+                   tl1, rl1, c1, ml1,
+                   recvs, sends,
+                   tl2s, rl2, ml2,
+                   time,
+                   srcstorage);
+  }
+
+
+
+public:
+  
   // Access to the data
   virtual const gdata* operator() (int tl, int rl, int c, int ml) const = 0;