From e91c85225521026738519d1d81455b22bbdcb2ee Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 11 Jul 2008 15:57:07 -0500 Subject: Carpet: Add new modes "global_early" and "global_late" Add new modes meta_early, meta_late, global_early, and global_late, which execute like meta mode or global mode, but execut either as early or as late as possible in the schedule. --- Carpet/Carpet/param.ccl | 6 -- Carpet/Carpet/src/CallFunction.cc | 38 +++++------ Carpet/Carpet/src/Evolve.cc | 43 +++++++++---- Carpet/Carpet/src/Initialise.cc | 132 +++++++++++++++++++++++++++++++------- Carpet/Carpet/src/Shutdown.cc | 17 +++-- Carpet/Carpet/src/variables.cc | 4 ++ Carpet/Carpet/src/variables.hh | 4 ++ 7 files changed, 181 insertions(+), 63 deletions(-) diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl index 94b4c7e4e..f311c6d96 100644 --- a/Carpet/Carpet/param.ccl +++ b/Carpet/Carpet/param.ccl @@ -499,9 +499,3 @@ BOOLEAN init_3_timelevels "Set up 3 timelevels of initial data" STEERABLE=always BOOLEAN adaptive_stepsize "Allow adaptive timestep sizes" { } "no" - - - -BOOLEAN global_mode_on_finest_grid "Run global mode routines on the finest grin" STEERABLE=always -{ -} "no" diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc index ea64fa115..f5c79dbf1 100644 --- a/Carpet/Carpet/src/CallFunction.cc +++ b/Carpet/Carpet/src/CallFunction.cc @@ -46,7 +46,11 @@ namespace Carpet { cGH * cctkGH = static_cast (data); assert (int (not not attribute->meta) + + int (not not attribute->meta_early) + + int (not not attribute->meta_late) + int (not not attribute->global) + + int (not not attribute->global_early) + + int (not not attribute->global_late) + int (not not attribute->level) + int (not not attribute->singlemap) + int (not not attribute->local) @@ -58,10 +62,16 @@ namespace Carpet { not not attribute->loop_local <= 1); - if (attribute->meta or is_meta_mode()) { + if (attribute->meta or attribute->meta_early or attribute->meta_late or + is_meta_mode()) + { // Convtest operation - if (do_meta_mode) { + if ((attribute->meta and do_meta_mode) or + (attribute->meta_early and do_early_meta_mode) or + (attribute->meta_late and do_late_meta_mode) or + is_meta_mode()) + { if (attribute->loop_local) { BEGIN_META_MODE(cctkGH) { BEGIN_MGLEVEL_LOOP(cctkGH) { @@ -136,12 +146,16 @@ namespace Carpet { } } - } else if (attribute->global or is_global_mode()) { + } else if (attribute->global or attribute->global_early or + attribute->global_late or is_global_mode()) + { // Global operation: call once - assert (not attribute->loop_meta); - - if (do_global_mode) { + if ((attribute->global and do_global_mode) or + (attribute->global_early and do_early_global_mode) or + (attribute->global_late and do_late_global_mode) or + is_global_mode()) + { if (attribute->loop_local) { BEGIN_GLOBAL_MODE(cctkGH) { BEGIN_REFLEVEL_LOOP(cctkGH) { @@ -198,9 +212,6 @@ namespace Carpet { } else if (attribute->level) { // Level operation: call once per refinement level - assert (not attribute->loop_meta); - assert (not attribute->loop_global); - if (attribute->loop_local) { BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { @@ -227,10 +238,6 @@ namespace Carpet { } else if (attribute->singlemap) { // Single map operation: call once per refinement level and map - assert (not attribute->loop_meta); - assert (not attribute->loop_global); - assert (not attribute->loop_level); - if (attribute->loop_local) { BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { @@ -253,11 +260,6 @@ namespace Carpet { } else { // Local operation: call once per component - assert (not attribute->loop_meta); - assert (not attribute->loop_global); - assert (not attribute->loop_level); - assert (not attribute->loop_singlemap); - BEGIN_MAP_LOOP(cctkGH, CCTK_GF) { BEGIN_LOCAL_COMPONENT_LOOP(cctkGH, CCTK_GF) { CallScheduledFunction diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc index ec3eeb32e..f6ea07eab 100644 --- a/Carpet/Carpet/src/Evolve.cc +++ b/Carpet/Carpet/src/Evolve.cc @@ -247,9 +247,17 @@ namespace Carpet { assert (is_level_mode()); bool const old_do_global_mode = do_global_mode; + bool const old_do_early_global_mode = do_early_global_mode; + bool const old_do_late_global_mode = do_late_global_mode; bool const old_do_meta_mode = do_meta_mode; + bool const old_do_early_meta_mode = do_early_meta_mode; + bool const old_do_late_meta_mode = do_late_meta_mode; do_global_mode = true; + do_early_global_mode = true; + do_late_global_mode = true; do_meta_mode = true; + do_early_meta_mode = true; + do_late_meta_mode = true; Waypoint ("Preregrid at iteration %d time %g%s%s", cctkGH->cctk_iteration, (double)cctkGH->cctk_time, @@ -278,8 +286,12 @@ namespace Carpet { if (did_recompose or rl == reflevels - 1) { BEGIN_MGLEVEL_LOOP (cctkGH) { ENTER_LEVEL_MODE (cctkGH, rl) { - do_global_mode = reflevel == reflevels - 1; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==0; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_late_global_mode; + do_meta_mode = do_late_meta_mode; Waypoint ("Postregrid at iteration %d time %g%s%s", cctkGH->cctk_iteration, (double)cctkGH->cctk_time, @@ -338,7 +350,11 @@ namespace Carpet { } // if did_regrid do_global_mode = old_do_global_mode; + do_early_global_mode = old_do_early_global_mode; + do_late_global_mode = old_do_late_global_mode; do_meta_mode = old_do_meta_mode; + do_early_meta_mode = old_do_early_meta_mode; + do_late_meta_mode = old_do_late_meta_mode; timer.stop(); } @@ -366,11 +382,12 @@ namespace Carpet { ENTER_GLOBAL_MODE (cctkGH, ml) { ENTER_LEVEL_MODE (cctkGH, rl) { - do_global_mode - = (global_mode_on_finest_grid - ? reflevel == reflevels - 1 - : not have_done_global_mode); - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = not have_done_global_mode; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_early_global_mode; + do_meta_mode = do_early_meta_mode; assert (not (have_done_global_mode and do_global_mode)); have_done_global_mode |= do_global_mode; have_done_anything = true; @@ -498,8 +515,12 @@ namespace Carpet { ENTER_GLOBAL_MODE (cctkGH, ml) { ENTER_LEVEL_MODE (cctkGH, rl) { - do_global_mode = reflevel == reflevels - 1; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = not have_done_global_mode; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_late_global_mode; + do_meta_mode = do_global_mode and do_late_meta_mode; assert (not (have_done_global_mode and do_global_mode)); have_done_global_mode |= do_global_mode; have_done_anything = true; @@ -526,7 +547,7 @@ namespace Carpet { // Analysis ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH); - if (do_global_mode) { + if (do_late_global_mode) { // Timing statistics UpdateTimingStats (cctkGH); } @@ -537,7 +558,7 @@ namespace Carpet { // Checking CheckChecksums (cctkGH, alltimes); - if (do_global_mode) { + if (do_late_global_mode) { // Timing statistics PrintTimingStats (cctkGH); } diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc index 08a00aa84..75b6c56ad 100644 --- a/Carpet/Carpet/src/Initialise.cc +++ b/Carpet/Carpet/src/Initialise.cc @@ -56,7 +56,11 @@ namespace Carpet { CCTKi_AddGH (fc, convlev, cctkGH); do_global_mode = true; + do_early_global_mode = true; + do_late_global_mode = true; do_meta_mode = true; + do_early_meta_mode = true; + do_late_meta_mode = true; global_time = cctk_initial_time; delta_time = 1.0; for (int ml = 0; ml < mglevel; ++ ml) { @@ -140,8 +144,12 @@ namespace Carpet { timer.start(); BEGIN_MGLEVEL_LOOP(cctkGH) { + do_early_global_mode = true; + do_late_global_mode = true; do_global_mode = true; - do_meta_mode = mglevel==mglevels-1; // on first iteration, coarsest grid + do_early_meta_mode = mglevel==mglevels-1; + do_late_meta_mode = mglevel==0; + do_meta_mode = do_early_meta_mode; // on first iteration, coarsest grid // Checking { @@ -179,8 +187,12 @@ namespace Carpet { for (int rl=0; rlcctk_time = global_time; @@ -200,7 +212,7 @@ namespace Carpet { // Timing statistics // (do this here, after cctk_time has been recovered) - if (do_global_mode) { + if (do_early_global_mode) { InitTimingStats (cctkGH); } @@ -233,8 +245,12 @@ namespace Carpet { for (int rl=0; rlcctk_iteration, (double)cctkGH->cctk_time, @@ -287,8 +303,12 @@ namespace Carpet { for (int rl=0; rlcctk_time = global_time; @@ -298,7 +318,7 @@ namespace Carpet { (do_meta_mode ? " (meta)" : "")); // Timing statistics - if (do_global_mode) { + if (do_early_global_mode) { InitTimingStats (cctkGH); } @@ -394,8 +414,12 @@ namespace Carpet { for (int rl=0; rlcctk_iteration, (double)cctkGH->cctk_time, @@ -431,8 +455,12 @@ namespace Carpet { for (int rl=0; rlcctk_iteration, (double)cctkGH->cctk_time, @@ -449,7 +477,7 @@ namespace Carpet { // Analysis ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH); - if (do_global_mode) { + if (do_late_global_mode) { // Timing statistics UpdateTimingStats (cctkGH); } @@ -703,9 +731,17 @@ namespace Carpet { assert (is_level_mode()); bool const old_do_global_mode = do_global_mode; + bool const old_do_early_global_mode = do_early_global_mode; + bool const old_do_late_global_mode = do_late_global_mode; bool const old_do_meta_mode = do_meta_mode; + bool const old_do_early_meta_mode = do_early_meta_mode; + bool const old_do_late_meta_mode = do_late_meta_mode; do_global_mode = true; + do_early_global_mode = true; + do_late_global_mode = true; do_meta_mode = true; + do_early_meta_mode = true; + do_late_meta_mode = true; // Preregrid Waypoint ("Preregrid at iteration %d time %g%s%s", @@ -730,8 +766,12 @@ namespace Carpet { if (did_recompose) { BEGIN_MGLEVEL_LOOP (cctkGH) { ENTER_LEVEL_MODE (cctkGH, rl) { - do_global_mode = reflevel == reflevels - 1; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==0; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_late_global_mode; + do_meta_mode = do_late_meta_mode; Waypoint ("Postregrid at iteration %d time %g%s%s", cctkGH->cctk_iteration, (double)cctkGH->cctk_time, @@ -790,7 +830,11 @@ namespace Carpet { } // if did_regrid do_global_mode = old_do_global_mode; + do_early_global_mode = old_do_early_global_mode; + do_late_global_mode = old_do_late_global_mode; do_meta_mode = old_do_meta_mode; + do_early_meta_mode = old_do_early_meta_mode; + do_late_meta_mode = old_do_late_meta_mode; timer.stop(); } @@ -809,9 +853,17 @@ namespace Carpet { assert (is_meta_mode()); bool const old_do_global_mode = do_global_mode; + bool const old_do_early_global_mode = do_early_global_mode; + bool const old_do_late_global_mode = do_late_global_mode; bool const old_do_meta_mode = do_meta_mode; + bool const old_do_early_meta_mode = do_early_meta_mode; + bool const old_do_late_meta_mode = do_late_meta_mode; do_global_mode = true; + do_early_global_mode = true; + do_late_global_mode = true; do_meta_mode = true; + do_early_meta_mode = true; + do_late_meta_mode = true; ENTER_GLOBAL_MODE (cctkGH, 0) { ENTER_LEVEL_MODE (cctkGH, 0) { @@ -842,7 +894,11 @@ namespace Carpet { } LEAVE_GLOBAL_MODE; do_global_mode = old_do_global_mode; + do_early_global_mode = old_do_early_global_mode; + do_late_global_mode = old_do_late_global_mode; do_meta_mode = old_do_meta_mode; + do_early_meta_mode = old_do_early_meta_mode; + do_late_meta_mode = old_do_late_meta_mode; timer.stop(); } @@ -864,9 +920,17 @@ namespace Carpet { assert (is_level_mode()); bool const old_do_global_mode = do_global_mode; + bool const old_do_early_global_mode = do_early_global_mode; + bool const old_do_late_global_mode = do_late_global_mode; bool const old_do_meta_mode = do_meta_mode; + bool const old_do_early_meta_mode = do_early_meta_mode; + bool const old_do_late_meta_mode = do_late_meta_mode; do_global_mode = true; + do_early_global_mode = true; + do_late_global_mode = true; do_meta_mode = true; + do_early_meta_mode = true; + do_late_meta_mode = true; // Preregrid Waypoint ("Preregridinitial at iteration %d time %g%s%s", @@ -891,8 +955,12 @@ namespace Carpet { if (did_recompose) { BEGIN_MGLEVEL_LOOP (cctkGH) { ENTER_LEVEL_MODE (cctkGH, rl) { - do_global_mode = reflevel == reflevels - 1; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==0; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_late_global_mode; + do_meta_mode = do_late_meta_mode; Waypoint ("Postregridinitial at iteration %d time %g%s%s", cctkGH->cctk_iteration, (double)cctkGH->cctk_time, @@ -952,7 +1020,11 @@ namespace Carpet { } // if did_regrid do_global_mode = old_do_global_mode; + do_early_global_mode = old_do_early_global_mode; + do_late_global_mode = old_do_late_global_mode; do_meta_mode = old_do_meta_mode; + do_early_meta_mode = old_do_early_meta_mode; + do_late_meta_mode = old_do_late_meta_mode; timer.stop(); } @@ -988,8 +1060,12 @@ namespace Carpet { BEGIN_MGLEVEL_LOOP(cctkGH) { BEGIN_REFLEVEL_LOOP(cctkGH) { - do_global_mode = reflevel==0; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==0; + do_late_global_mode = reflevel==reflevels-1; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_early_global_mode; + do_meta_mode = do_early_meta_mode; initialise_3tl_advance_time (cctkGH); initialise_3tl_evolve_Ia (cctkGH); @@ -1006,8 +1082,12 @@ namespace Carpet { BEGIN_MGLEVEL_LOOP(cctkGH) { BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) { - do_global_mode = reflevel==0; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==reflevels-1; + do_late_global_mode = reflevel==0; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_early_global_mode; + do_meta_mode = do_early_meta_mode; initialise_3tl_evolve_IIb (cctkGH); initialise_3tl_advance_time_2 (cctkGH); @@ -1020,8 +1100,12 @@ namespace Carpet { BEGIN_MGLEVEL_LOOP(cctkGH) { BEGIN_REVERSE_REFLEVEL_LOOP(cctkGH) { - do_global_mode = reflevel==0; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + do_early_global_mode = reflevel==reflevels-1; + do_late_global_mode = reflevel==0; + do_early_meta_mode = do_early_global_mode and mglevel==mglevels-1; + do_late_meta_mode = do_late_global_mode and mglevel==0; + do_global_mode = do_early_global_mode; + do_meta_mode = do_early_meta_mode; initialise_3tl_reset_time (cctkGH); diff --git a/Carpet/Carpet/src/Shutdown.cc b/Carpet/Carpet/src/Shutdown.cc index 374157bef..38e05bcae 100644 --- a/Carpet/Carpet/src/Shutdown.cc +++ b/Carpet/Carpet/src/Shutdown.cc @@ -32,8 +32,13 @@ namespace Carpet { for (int rl=reflevels-1; rl>=0; --rl) { BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) { enter_level_mode (cctkGH, rl); - do_global_mode = reflevel==0; - do_meta_mode = do_global_mode and mglevel==mglevels-1; + + do_early_global_mode = reflevel==reflevels-1; + do_late_global_mode = reflevel==0; + do_early_meta_mode = do_early_global_mode and mglevel==0; + do_late_meta_mode = do_late_global_mode and mglevel==mglevels-1; + do_global_mode = do_late_global_mode; + do_meta_mode = do_late_meta_mode; Checkpoint ("Shutdown at iteration %d time %g%s%s", cctkGH->cctk_iteration, (double)cctkGH->cctk_time, @@ -49,8 +54,12 @@ namespace Carpet { } // for rl BEGIN_REVERSE_MGLEVEL_LOOP(cctkGH) { - do_global_mode = true; - do_meta_mode = mglevel==mglevels-1; + do_early_global_mode = true; + do_late_global_mode = true; + do_early_meta_mode = do_early_global_mode and mglevel==0; + do_late_meta_mode = do_late_global_mode and mglevel==mglevels-1; + do_global_mode = do_late_global_mode; + do_meta_mode = do_late_meta_mode; // Shutdown Checkpoint ("Scheduling SHUTDOWN"); diff --git a/Carpet/Carpet/src/variables.cc b/Carpet/Carpet/src/variables.cc index 6b98f4312..a27a5bacb 100644 --- a/Carpet/Carpet/src/variables.cc +++ b/Carpet/Carpet/src/variables.cc @@ -80,7 +80,11 @@ namespace Carpet { // Is this the time for a global mode call? bool do_meta_mode; + bool do_early_meta_mode; + bool do_late_meta_mode; bool do_global_mode; + bool do_early_global_mode; + bool do_late_global_mode; // Can past time levels be accessed? bool do_allow_past_timelevels; diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh index a50d471ca..23440aea5 100644 --- a/Carpet/Carpet/src/variables.hh +++ b/Carpet/Carpet/src/variables.hh @@ -117,7 +117,11 @@ namespace Carpet { // Is this the time for a global mode call? extern bool do_meta_mode; + extern bool do_early_meta_mode; + extern bool do_late_meta_mode; extern bool do_global_mode; + extern bool do_early_global_mode; + extern bool do_late_global_mode; // Can past time levels be accessed? extern bool do_allow_past_timelevels; -- cgit v1.2.3