CarpetLib: resolve conflict between collective communication and mem<T>

darcs-hash:20050411203309-891bb-5b74d6135f6cd6995f1eed6cc74dd2f29c42f8a8.gz

1 files changed, 28 insertions, 46 deletions

diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index e1feacb65..526df3bd5 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -547,7 +547,27 @@ void dh::calculate_bases ()
 
 void dh::save_time (bool do_prolongate)
 {
-  for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) {
+  DECLARE_CCTK_PARAMETERS;
+
+  // sort all grid functions into sets of the same vartype
+  vector<gf_set> ggfs;
+  for (list<ggf*>::iterator f = gfs.begin(); f != gfs.end(); ++f) {
+    gf_set newset;
+    newset.vartype = CCTK_VarTypeI ((*f)->varindex);
+    assert (newset.vartype >= 0);
+    int c;
+    for (c = 0; c < ggfs.size(); c++) {
+      if (newset.vartype == ggfs[c].vartype) {
+        break;
+      }
+    }
+    if (c == ggfs.size()) {
+      ggfs.push_back (newset);
+    }
+    ggfs[c].members.push_back (*f);
+  }
+
+  for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
     (*f)->recompose_crop ();
   }
   for (int rl=0; rl<h.reflevels(); ++rl) {
@@ -559,35 +579,11 @@ void dh::save_time (bool do_prolongate)
         for (int g = 0; g < ggfs[c].members.size(); g++) {
           ggfs[c].members[g]->recompose_fill (state, rl, do_prolongate);
         }
-        for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
-          for (int g = 0; g < ggfs[c].members.size(); g++) {
-            ggfs[c].members[g]->recompose_fill (state, rl, do_prolongate);
-          }
-        }
-        // free in the opposite order
-        for (int g = ggfs[c].members.size() - 1; g >= 0; g--) {
-          ggfs[c].members[g]->recompose_free (rl);
-        }
       }
-    }
-    for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) {
-      (*f)->recompose_free (rl);
-    }
-    for (comm_state state; !state.done(); state.step()) {
-      for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
-        for (comm_state state; !state.done(); state.step()) {
-          (*f)->recompose_bnd_prolongate (state, rl, do_prolongate);
-        }
-        for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
-          for (int g = 0; g < ggfs[c].members.size(); g++) {
-            ggfs[c].members[g]->recompose_sync (state, rl, do_prolongate);
-          }
-        }
-      } // for vartype
-    } else {
-      // make the comm_state loop the innermost
-      // in order to minimise the number of outstanding communications
-      for (int c = 0; c < ggfs.size(); c++) {
+      for (int g = 0; g < ggfs[c].members.size(); g++) {
+        ggfs[c].members[g]->recompose_free (rl);
+      }
+      for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
         for (int g = 0; g < ggfs[c].members.size(); g++) {
           ggfs[c].members[g]->recompose_bnd_prolongate (state, rl, do_prolongate);
         }
@@ -596,23 +592,9 @@ void dh::save_time (bool do_prolongate)
         for (int g = 0; g < ggfs[c].members.size(); g++) {
           ggfs[c].members[g]->recompose_sync (state, rl, do_prolongate);
         }
-        // free in the opposite order
-        for (int g = ggfs[c].members.size() - 1; g >= 0; g--) {
-          ggfs[c].members[g]->recompose_free (rl);
-        }
-        for (int g = 0; g < ggfs[c].members.size(); g++) {
-          for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
-            ggfs[c].members[g]->recompose_bnd_prolongate (state, rl, do_prolongate);
-          }
-        }
-        for (int g = 0; g < ggfs[c].members.size(); g++) {
-          for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
-            ggfs[c].members[g]->recompose_sync (state, rl, do_prolongate);
-          }
-        }
-      } // for vartype
-    } // if use_collective_communication_buffers
-  } // for rl
+      }
+    } // for all grid functions of same vartype
+  } // for all refinement levels
 }
 
 void dh::save_memory (bool do_prolongate) {