CarpetLib: Add new parameter combine_recompose

combine_recompose recomposes all grid functions at once.  This increases
memory usage, but combines the communications and may thus also increase
the speed.  The default behaviour is unchanged, recomposing all grid
functions sequentially.

2 files changed, 33 insertions, 6 deletions

diff --git a/Carpet/CarpetLib/param.ccl b/Carpet/CarpetLib/param.ccl
index c1d3754ec..45724f25d 100644
--- a/Carpet/CarpetLib/param.ccl
+++ b/Carpet/CarpetLib/param.ccl
@@ -85,6 +85,14 @@ STRING memstat_file "File name in which memstat output is collected (because std
 
 
 
+# Experimental recomposing parameters
+
+BOOLEAN combine_recompose "Recompose all grid functions of one refinement levels at once" STEERABLE=always
+{
+} "no"
+
+
+
 # Experimental communication parameters
 
 BOOLEAN interleave_communications "Try to interleave communications with each other; each processor begins to communicate with its 'right neighbour' in rank, instead of with the root processor" STEERABLE=always
diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index fa990a2b0..a3d50d08a 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -71,7 +71,6 @@ inline
 int
 dh::this_proc (int const rl, int const c) const
 {
-  // return c % dist::size();
   return h.processor (rl, c);
 }
 
@@ -1006,6 +1005,8 @@ void
 dh::
 recompose (int const rl, bool const do_prolongate)
 {
+  DECLARE_CCTK_PARAMETERS;
+  
   assert (rl>=0 and rl<h.reflevels());
   
   static Timer timer ("dh::recompose");
@@ -1015,13 +1016,31 @@ recompose (int const rl, bool const do_prolongate)
     (*f)->recompose_crop ();
   }
   
-  for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
-    (*f)->recompose_allocate (rl);
+  if (combine_recompose) {
+    // Recompose all grid functions of this refinement levels at once.
+    // This may be faster, but requires more memory.
+    for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
+      (*f)->recompose_allocate (rl);
+    }
     for (comm_state state; not state.done(); state.step()) {
-      (*f)->recompose_fill (state, rl, do_prolongate);
+      for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
+        (*f)->recompose_fill (state, rl, do_prolongate);
+      }
     }
-    (*f)->recompose_free_old (rl);
-  } // for all grid functions of same vartype
+    for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
+      (*f)->recompose_free_old (rl);
+    }
+  } else {
+    // Recompose the grid functions sequentially.  This may be slower,
+    // but requires less memory.  This is the default.
+    for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
+      (*f)->recompose_allocate (rl);
+      for (comm_state state; not state.done(); state.step()) {
+        (*f)->recompose_fill (state, rl, do_prolongate);
+      }
+      (*f)->recompose_free_old (rl);
+    }
+  }
   
   timer.stop (0);
 }