diff options
Diffstat (limited to 'Carpet/CarpetLib')
| -rw-r--r-- | Carpet/CarpetLib/param.ccl | 7 | ||||
| -rw-r--r-- | Carpet/CarpetLib/src/dh.cc | 47 |
2 files changed, 43 insertions, 11 deletions
diff --git a/Carpet/CarpetLib/param.ccl b/Carpet/CarpetLib/param.ccl index 55754ef0f..77b8acbec 100644 --- a/Carpet/CarpetLib/param.ccl +++ b/Carpet/CarpetLib/param.ccl @@ -1,5 +1,12 @@ # Parameter definitions for thorn CarpetLib +restricted: + +BOOLEAN minimise_outstanding_communications "Minimise the number of Isend/Irecv operations that are submitted concurrently" +{ +} "no" + + private: BOOLEAN verbose "Print info to the screen" STEERABLE=always diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc index 2078b88a3..b28f0ac7d 100644 --- a/Carpet/CarpetLib/src/dh.cc +++ b/Carpet/CarpetLib/src/dh.cc @@ -540,6 +540,8 @@ void dh::calculate_bases () void dh::save_time (bool do_prolongate) { + DECLARE_CCTK_PARAMETERS; + for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) { (*f)->recompose_crop (); } @@ -547,22 +549,45 @@ void dh::save_time (bool do_prolongate) for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { (*f)->recompose_allocate (rl); } - for (comm_state state; !state.done(); state.step()) { + // make the comm_state loop the innermost + // in order to minimise the number of outstanding communications + if (minimise_outstanding_communications) { for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { - (*f)->recompose_fill (state, rl, do_prolongate); + for (comm_state state; !state.done(); state.step()) { + (*f)->recompose_fill (state, rl, do_prolongate); + } + } + for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) { + (*f)->recompose_free (rl); } - } - for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) { - (*f)->recompose_free (rl); - } - for (comm_state state; !state.done(); state.step()) { for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { - (*f)->recompose_bnd_prolongate (state, rl, do_prolongate); + for (comm_state state; !state.done(); state.step()) { + (*f)->recompose_bnd_prolongate (state, rl, do_prolongate); + } } - } - for (comm_state state; !state.done(); state.step()) { for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { - (*f)->recompose_sync (state, rl, do_prolongate); + for (comm_state state; !state.done(); state.step()) { + (*f)->recompose_sync (state, rl, do_prolongate); + } + } + } else { + for (comm_state state; !state.done(); state.step()) { + for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { + (*f)->recompose_fill (state, rl, do_prolongate); + } + } + for (list<ggf*>::reverse_iterator f=gfs.rbegin(); f!=gfs.rend(); ++f) { + (*f)->recompose_free (rl); + } + for (comm_state state; !state.done(); state.step()) { + for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { + (*f)->recompose_bnd_prolongate (state, rl, do_prolongate); + } + } + for (comm_state state; !state.done(); state.step()) { + for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) { + (*f)->recompose_sync (state, rl, do_prolongate); + } } } } // for rl |