From 555a1a6fcc6b067affbdd7d21a2335aa2bfa0436 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 30 Aug 2012 10:27:41 -0400 Subject: Carpet: Check schedule reads/writes statements at run time --- Carpet/Carpet/param.ccl | 10 +- Carpet/Carpet/src/CallFunction.cc | 5 +- Carpet/Carpet/src/Comm.cc | 11 +- Carpet/Carpet/src/Cycle.cc | 7 +- Carpet/Carpet/src/Evolve.cc | 10 +- Carpet/Carpet/src/Initialise.cc | 26 +- Carpet/Carpet/src/Requirements.cc | 1270 +++++++++++++++++++++++++++++-------- Carpet/Carpet/src/Requirements.hh | 53 ++ Carpet/Carpet/src/Restrict.cc | 4 + Carpet/Carpet/src/SetupGH.cc | 2 + Carpet/Carpet/src/Storage.cc | 22 + Carpet/Carpet/src/carpet.hh | 9 +- 12 files changed, 1147 insertions(+), 282 deletions(-) create mode 100644 Carpet/Carpet/src/Requirements.hh diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl index 586af8eb8..259012844 100644 --- a/Carpet/Carpet/param.ccl +++ b/Carpet/Carpet/param.ccl @@ -596,6 +596,14 @@ BOOLEAN use_unusedpoints_mask "Turn on storage and usage of 'unusedpoints_mask'" -BOOLEAN requirement_inconsistencies_are_fatal "Abort when encountering inconsistencies in requirements" STEERABLE=recover +BOOLEAN check_requirements "Check schedule requirements" STEERABLE=always +{ +} "no" + +BOOLEAN requirements_verbose "Output details when checking requirements" STEERABLE=always +{ +} "no" + +BOOLEAN requirement_inconsistencies_are_fatal "Abort when encountering inconsistencies in requirements" STEERABLE=always { } "no" diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc index 65be1f244..2cf661040 100644 --- a/Carpet/Carpet/src/CallFunction.cc +++ b/Carpet/Carpet/src/CallFunction.cc @@ -16,6 +16,7 @@ #include #include "adler32.hh" +#include "Requirements.hh" @@ -357,6 +358,7 @@ namespace Carpet { CCTK_REAL const saved_cctk_delta_time = cctkGH->cctk_delta_time; user_timer.start(); + Requirements::BeforeRoutine(attribute, reflevel, map, timelevel); timer.start(); if (CCTK_IsFunctionAliased("Accelerator_PreCallFunction")) { Timer pre_timer("PreCall"); @@ -373,6 +375,7 @@ namespace Carpet { post_timer.stop(); } timer.stop(); + Requirements::AfterRoutine(attribute, reflevel, map, timelevel); user_timer.stop(); // Manage the time step size. If the time step size changes @@ -448,7 +451,7 @@ namespace Carpet { } sync_timer.start(); - SyncProlongateGroups (cctkGH, sync_groups); + SyncProlongateGroups (cctkGH, sync_groups, attribute); sync_timer.stop(); } diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc index 55be38251..df529aeaa 100644 --- a/Carpet/Carpet/src/Comm.cc +++ b/Carpet/Carpet/src/Comm.cc @@ -14,6 +14,8 @@ #include #include +#include "Requirements.hh" + namespace Carpet { @@ -66,7 +68,8 @@ namespace Carpet { // synchronises ghostzones and prolongates boundaries of a set of groups // // returns 0 for success and -1 if the set contains a group with no storage - int SyncProlongateGroups (const cGH* cctkGH, const vector& groups) + int SyncProlongateGroups (const cGH* cctkGH, const vector& groups, + cFunctionData const* function_data) { int retval = 0; DECLARE_CCTK_PARAMETERS; @@ -161,9 +164,11 @@ namespace Carpet { SyncGroups (cctkGH, goodgroups); timer.stop(); } - + + Requirements::Sync(function_data, goodgroups, reflevel, timelevel); + } - + return retval; } diff --git a/Carpet/Carpet/src/Cycle.cc b/Carpet/Carpet/src/Cycle.cc index 449a5a7b3..79137ad4e 100644 --- a/Carpet/Carpet/src/Cycle.cc +++ b/Carpet/Carpet/src/Cycle.cc @@ -8,7 +8,10 @@ #include #include -#include "Timers.hh" +#include + +#include "Requirements.hh" + namespace Carpet { @@ -105,6 +108,8 @@ namespace Carpet { Accelerator_Cycle(cctkGH); } + Requirements::Cycle(reflevel); + if (errors > 0) { CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING, "Errors in %d groups detected; aborting", errors); diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc index c1d4ee5b6..114fb25f9 100644 --- a/Carpet/Carpet/src/Evolve.cc +++ b/Carpet/Carpet/src/Evolve.cc @@ -21,7 +21,9 @@ #include #include #include -#include "TimerNode.hh" +#include + +#include "Requirements.hh" @@ -292,6 +294,7 @@ namespace Carpet { assert (not did_remove_level or did_regrid); if (did_regrid) { + Requirements::Regrid(reflevels); bool did_any_recompose = false; BEGIN_META_MODE (cctkGH) { @@ -304,6 +307,10 @@ namespace Carpet { bool const did_recompose = Recompose (cctkGH, rl, true); did_any_recompose = did_any_recompose or did_recompose; + Requirements::Recompose(reflevel, + not did_recompose ? + Requirements::valid::everywhere : + Requirements::valid::interior); // Carpet assumes that a regridding operation always changes // "level N and all finer levels" so we should call @@ -364,6 +371,7 @@ namespace Carpet { if (have_done_anything) assert (have_done_late_global_mode); } END_META_MODE; + Requirements::RegridFree(); } // if did_regrid RegridFree (cctkGH, true); diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc index 9a6c2150b..ec007a882 100644 --- a/Carpet/Carpet/src/Initialise.cc +++ b/Carpet/Carpet/src/Initialise.cc @@ -20,6 +20,8 @@ #include #include +#include "Requirements.hh" + namespace Carpet { @@ -192,7 +194,7 @@ namespace Carpet { ScheduleTraverse (where, "CCTK_PARAMCHECK", cctkGH); } END_MGLEVEL_LOOP; - Requirements::CheckRequirements (cctkGH); + // Requirements::CheckRequirements (cctkGH); CCTKi_FinaliseParamWarn(); @@ -919,6 +921,7 @@ namespace Carpet { assert (not did_remove_level or did_regrid); if (did_regrid) { + Requirements::Regrid(reflevels); bool did_any_recompose = false; BEGIN_META_MODE (cctkGH) { @@ -932,6 +935,12 @@ namespace Carpet { bool did_recompose = false; did_recompose = Recompose (cctkGH, rl, prolongate_initial_data); did_any_recompose = did_any_recompose or did_recompose; + Requirements::Recompose(rl, + not did_recompose ? + Requirements::valid::everywhere : + prolongate_initial_data ? + Requirements::valid::interior : + Requirements::valid::nowhere); // Carpet assumes that a regridding operation always changes "level N // and all finer levels" so we should call POSTREGRID on all finer levels @@ -984,6 +993,7 @@ namespace Carpet { if (have_done_anything) assert (have_done_late_global_mode); } END_META_MODE; + Requirements::RegridFree(); } // if did_regrid RegridFree (cctkGH, prolongate_initial_data); @@ -1040,11 +1050,17 @@ namespace Carpet { bool const did_regrid = Regrid (cctkGH, true, prolongate_initial_data); if (did_regrid) { + Requirements::Regrid(reflevels); for (int rl=0; rl -#include -#include +#include + +#include #include +#include #include - #include #include -#include +#include +#include +#include +#include +#include +#include +#include using namespace std; -#ifdef CACTUS_HAS_REQUIRES_CLAUSES - -// Illegally copied from ScheduleInterface.c: - -typedef enum {sched_none, sched_group, sched_function} iSchedType; -typedef enum {schedpoint_misc, schedpoint_analysis} iSchedPoint; - -typedef struct t_timer -{ - struct t_timer *next; - int timer_handle; - char *schedule_bin; - int has_been_output; -} t_timer; - -typedef struct -{ - /* Static data */ - char *description; - - /*char *thorn; MOVED TO FunctionData */ - char *implementation; - - iSchedType type; - - cFunctionData FunctionData; - - int n_mem_groups; - int *mem_groups; - int *timelevels; - - int n_comm_groups; - int *comm_groups; - - /* Timer data */ - t_timer *timers; - - /* Dynamic data */ - int *CommOnEntry; - int *StorageOnEntry; - - int done_entry; - int synchronised; - -} t_attribute; - - - namespace Carpet { namespace Requirements { @@ -73,281 +31,1057 @@ namespace Carpet { // another routine which is scheduled earlier. // // 2. Things can be provided only once, not multiple times. - // Except when they are also provided. + // Except when they are also required. + + + + // Represent scheduled functions and their dependencies + + struct clause_t { + bool everywhere; // all grid points (everywhere) + bool interior; // all interior points + bool boundary; // all boundary points, excluding + // ghostzones + bool boundary_ghostzones; // all boundary ghost points + bool all_timelevels; // all time levels + bool all_maps; // all maps (i.e. level mode) + bool all_reflevels; // all refinement levels (i.e. global mode) + vector vars; + clause_t(): + everywhere(false), + interior(false), boundary(false), boundary_ghostzones(false), + all_timelevels(false), all_maps(false), all_reflevels(false) + {} + void parse(char const* clause); + }; + + void clause_t::parse(char const* const clause1) + { + char* const clause = strdup(clause1); + char* p = clause; + + // Remove trailing "(...)" modifier, if any + p = strchr(p, '('); + if (p) *p = '\0'; + int const gi = CCTK_GroupIndex(clause); + if (gi >= 0) { + // A group + int const v0 = CCTK_FirstVarIndexI(gi); assert(v0 >= 0); + int const nv = CCTK_NumVarsInGroupI(gi); assert(nv >= 0); + for (int vi=v0; vi= 0); + vars.push_back(vi); + } + + // Parse modifiers + if (p) { + ++p; + for (;;) { + size_t const len = strcspn(p, ",)"); + char const c = p[len]; + assert(c); + p[len] = '\0'; + if (CCTK_EQUALS(p, "everywhere")) { + assert(not everywhere and + not interior and not boundary and not boundary_ghostzones); + everywhere = true; + } else if (CCTK_EQUALS(p, "interior")) { + assert(not everywhere and not interior); + interior = true; + } else if (CCTK_EQUALS(p, "boundary")) { + assert(not everywhere and not boundary); + boundary = true; + } else if (CCTK_EQUALS(p, "boundary_ghostzones")) { + assert(not everywhere and not boundary_ghostzones); + boundary_ghostzones = true; + } else if (CCTK_EQUALS(p, "all_timelevels")) { + // TODO: look at OPTIONS instead + assert(not all_timelevels); + all_timelevels = true; + } else if (CCTK_EQUALS(p, "all_maps")) { + // TODO: look at OPTIONS instead + assert(not all_maps); + all_maps = true; + } else if (CCTK_EQUALS(p, "all_reflevels")) { + // TODO: look at OPTIONS instead + assert(not all_reflevels); + all_reflevels = true; + } else { + assert(0); + } + if (c == ')') break; + p += len+1; + } + } + + free(clause); + } + + + + struct clauses_t { + vector reads, writes; + clauses_t() {} + void parse(cFunctionData const* function_data); + private: + void parse1(cFunctionData const* function_data, + vector clauses_t::* clauses, + int cFunctionData::* n_Clauses, + const char** cFunctionData::* Clauses); + }; + + void clauses_t::parse(cFunctionData const* function_data) + { + parse1(function_data, &clauses_t::reads, + &cFunctionData::n_ReadsClauses, &cFunctionData::ReadsClauses); + parse1(function_data, &clauses_t::writes, + &cFunctionData::n_WritesClauses, &cFunctionData::WritesClauses); + } + + void clauses_t::parse1(cFunctionData const* const function_data, + vector clauses_t::* const clauses, + int cFunctionData::* const n_Clauses, + const char** cFunctionData::* const Clauses) + { + assert((this->*clauses).empty()); + (this->*clauses).reserve(function_data->*n_Clauses); + for (int n=0; n*n_Clauses; ++n) { + clause_t clause; + clause.parse((function_data->*Clauses)[n]); + (this->*clauses).push_back(clause); + } + } + + + + class all_clauses_t { + // TODO: Represent I/O as well? + typedef std::map clauses_map_t; + clauses_map_t clauses_map; + // Singleton + all_clauses_t(all_clauses_t const&); + all_clauses_t& operator=(all_clauses_t const&); + public: + all_clauses_t() {} + clauses_t const& get_clauses(cFunctionData const* function_data); + }; + + clauses_t const& all_clauses_t:: + get_clauses(cFunctionData const* const function_data) + { + clauses_map_t::const_iterator const iclauses = + clauses_map.find(function_data); + if (iclauses != clauses_map.end()) return *iclauses->second; + clauses_t* const clauses = new clauses_t; + clauses->parse(function_data); + pair const ret = + clauses_map.insert(clauses_map_t::value_type(function_data, clauses)); + assert(ret.second); + return *ret.first->second; + } + + all_clauses_t all_clauses; + + // Keep track of which time levels contain good data; modify this - // while time level cycling; routine specify how many time levels - // they require/provide + // while time level cycling; routines should specify how many time + // levels they require/provide + + bool there_was_an_error = false; + struct gridpoint_t { + bool interior, boundary, ghostzones, boundary_ghostzones; + gridpoint_t(): + interior(false), boundary(false), ghostzones(false), + boundary_ghostzones(false) + {} + gridpoint_t(clause_t const& clause): + interior(clause.everywhere or clause.interior), + boundary(clause.everywhere or clause.boundary), + ghostzones(clause.everywhere), + boundary_ghostzones(clause.everywhere or clause.boundary_ghostzones) + {} + void check_state(clause_t const& clause, + cFunctionData const* function_data, + int vi, int rl, int m, int tl) const; + static void report_error(cFunctionData const* function_data, + int vi, int rl, int m, int tl, + char const* what, char const* where); + void update_state(clause_t const& clause); + }; + void gridpoint_t::check_state(clause_t const& clause, + cFunctionData const* const function_data, + int const vi, + int const rl, int const m, int const tl) + const + { + if (not interior) { + if (clause.everywhere or clause.interior) { + report_error(function_data, vi, rl, m, rl, + "calling function", "interior"); + } + } + if (not boundary) { + if (clause.everywhere or clause.boundary) { + report_error(function_data, vi, rl, m, rl, + "calling function", "boundary"); + } + } + if (not ghostzones) { + if (clause.everywhere) { + report_error(function_data, vi, rl, m, rl, + "calling function", "ghostzones"); + } + } + if (not boundary_ghostzones) { + if (clause.everywhere or clause.boundary_ghostzones) { + report_error(function_data, vi, rl, m, rl, + "calling", "boundary-ghostzones"); + } + } + } + + void gridpoint_t::report_error(cFunctionData const* const function_data, + int const vi, + int const rl, int const m, int const tl, + char const* const what, + char const* const where) + { + char* const fullname = CCTK_FullName(vi); + if (function_data) { + // The error is related to a scheduled function + CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, + "Schedule READS clause not satisfied: " + "Function %s::%s in %s: " + "Variable %s reflevel=%d map=%d timelevel=%d: " + "%s not valid for %s", + function_data->thorn, function_data->routine, + function_data->where, + fullname, rl, m, tl, + where, what); + } else { + // The error is not related to a scheduled function + CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, + "Schedule READS clause not satisfied: " + "Variable %s reflevel=%d map=%d timelevel=%d: " + "%s not valid for %s", + fullname, rl, m, tl, + where, what); + } + free(fullname); + there_was_an_error = true; + } + + void gridpoint_t::update_state(clause_t const& clause) + { + if (clause.everywhere or clause.interior) { + interior = true; + } + if (clause.everywhere or clause.boundary) { + boundary = true; + } + if (clause.everywhere) { + ghostzones = true; + } + if (clause.everywhere or clause.boundary_ghostzones) { + boundary_ghostzones = true; + } + } - int CheckEntry (void * attribute, void * data); - int CheckExit (void * attribute, void * data); - int CheckWhile (int n_whiles, char ** whiles, void * attribute, void * data, int first); - int CheckIf (int n_ifs, char ** ifs, void * attribute, void * data); - int CheckCall (void * function, void * attribute, void * data); - void CheckOneGroup (cGH const * cctkGH, char const * where); + class all_state_t { + typedef vector timelevels_t; + typedef vector maps_t; + typedef vector reflevels_t; + typedef vector variables_t; + variables_t vars; + variables_t old_vars; // for regridding + public: + void setup(int maps); + void change_storage(vector const& groups, + vector const& timelevels, + int reflevel); + void regrid(int reflevels); + void recompose(int reflevel, valid::valid_t where); + void regrid_free(); + void cycle(int reflevel); + void before_routine(cFunctionData const* function_data, + int reflevel, int map, int timelevel) const; + void after_routine(cFunctionData const* function_data, + int reflevel, int map, int timelevel); + void sync(cFunctionData const* function_data, + vector const& groups, int reflevel, int timelevel); + void restrict1(vector const& groups, int reflevel); + }; + all_state_t all_state; - int - CheckEntry (void * const attribute, - void * const data) + + + void Setup(int const maps) { DECLARE_CCTK_PARAMETERS; - - if (not attribute) { - // Nothing to check - return 1; + if (check_requirements) { + if (requirements_verbose) { + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Setup maps=%d", maps); + } + all_state.setup(maps); } - - int (*const warn) (char const *thorn, char const *format, ...) = - requirement_inconsistencies_are_fatal ? CCTK_VParamWarn : CCTK_VInfo; - - // Convert argument types - cFunctionData & function_data = - (static_cast (attribute))->FunctionData; - set & active_provisions = * static_cast *> (data); - - // Gather all required items - set requires; - for (int n = 0; n < function_data.n_RequiresClauses; ++ n) { - requires.insert (string (function_data.RequiresClauses[n])); + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); } - - // Check whether all required items have already been provided - set required_but_not_provided; - set_difference - (requires.begin(), requires.end(), - active_provisions.begin(), active_provisions.end(), - insert_iterator > - (required_but_not_provided, required_but_not_provided.begin())); - - // Are there unmet requirements? - if (not required_but_not_provided.empty()) { - for (set::const_iterator ri = required_but_not_provided.begin(); - ri != required_but_not_provided.end(); ++ ri) - { - string const req = * ri; - warn (CCTK_THORNSTRING, - "Requirement inconsistency:\n" - " Group %s, function %s::%s requires \"%s\" which has not been provided", - function_data.where, - function_data.thorn, function_data.routine, - req.c_str()); + } + + void all_state_t::setup(int const maps) + { + DECLARE_CCTK_PARAMETERS; + assert(vars.empty()); + vars.resize(CCTK_NumVars()); + for (variables_t::iterator + ivar = vars.begin(); ivar != vars.end(); ++ivar) + { + reflevels_t& rls = *ivar; + int const vi = &*ivar - &*vars.begin(); + assert(rls.empty()); + // Allocate one refinement level initially + int const nrls = 1; + rls.resize(nrls); + for (reflevels_t::iterator irl = rls.begin(); irl != rls.end(); ++irl) { + maps_t& ms = *irl; + assert(ms.empty()); + int const group_type = CCTK_GroupTypeFromVarI(vi); + int const nms = group_type==CCTK_GF ? maps : 1; + if (requirements_verbose) { + char* const fullname = CCTK_FullName(vi); + int const rl = &*irl - &*rls.begin(); + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Setting up %d maps for variable %s(rl=%d)", + nms, fullname, rl); + free(fullname); + } + ms.resize(nms); + for (maps_t::iterator im = ms.begin(); im != ms.end(); ++im) { + timelevels_t& tls = *im; + assert(tls.empty()); + // Not allocating any time levels here + } } } - - // Do traverse this schedule item - return 1; } - int - CheckExit (void * const attribute, - void * const data) + void ChangeStorage(vector const& groups, + vector const& timelevels, + int const reflevel) { DECLARE_CCTK_PARAMETERS; - - if (not attribute) { - // Nothing to check - return 1; + if (check_requirements) { + if (requirements_verbose) { + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: ChangeStorage reflevel=%d", reflevel); + } + all_state.change_storage(groups, timelevels, reflevel); } - - int (*const warn) (char const *thorn, char const *format, ...) = - requirement_inconsistencies_are_fatal ? CCTK_VParamWarn : CCTK_VInfo; - - // Convert argument types - cFunctionData & function_data = - (static_cast (attribute))->FunctionData; - set & active_provisions = * static_cast *> (data); - - // Gather all required and provided items - set requires; - for (int n = 0; n < function_data.n_RequiresClauses; ++ n) { - requires.insert (string (function_data.RequiresClauses[n])); + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); } - set provides; - for (int n = 0; n < function_data.n_ProvidesClauses; ++ n) { - provides.insert (string (function_data.ProvidesClauses[n])); + } + + void all_state_t::change_storage(vector const& groups, + vector const& timelevels, + int const reflevel) + { + DECLARE_CCTK_PARAMETERS; + assert(groups.size() == timelevels.size()); + for (vector::const_iterator + igi = groups.begin(), itl = timelevels.begin(); + igi != groups.end(); ++igi, ++itl) + { + int const gi = *igi; + int const tl = *itl; + bool const is_array = CCTK_GroupTypeI(gi) != CCTK_GF; + int const v0 = CCTK_FirstVarIndexI(gi); + int const nv = CCTK_NumVarsInGroupI(gi); + for (int vi=v0; vi=0 and max_rl<=reflevels); + for (int rl=min_rl; rl ntls) { + // Allocate new storage + if (requirements_verbose) { + char* const fullname = CCTK_FullName(vi); + int const m = &*im - &*ms.begin(); + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Increasing storage to %d time levels for variable %s(rl=%d,m=%d)", + tl, fullname, rl, m); + free(fullname); + } + // The default constructor for gridpoint_t sets all + // data to "invalid" + tls.resize(tl); + } + } + } + } } - - // Check whether any of the providions have already been - // provided. (We disallow this as well, so that a routine - // cannot overwrite what another routine has already set up.) - set provided_twice; - set_intersection - (provides.begin(), provides.end(), - active_provisions.begin(), active_provisions.end(), - insert_iterator > - (provided_twice, provided_twice.begin())); - // But we do allow to provide things which are also required - set provided_too_often; - set_difference - (provided_twice.begin(), provided_twice.end(), - requires.begin(), requires.end(), - insert_iterator > - (provided_too_often, provided_too_often.begin())); - - // Are there things provided twice? - if (not provided_too_often.empty()) { - for (set::const_iterator pi = provided_too_often.begin(); - pi != provided_too_often.end(); ++ pi) - { - string const prov = * pi; - warn (CCTK_THORNSTRING, - "Requirement inconsistency:\n" - " Group %s, function %s::%s provides (and does not require) \"%s\" which has already been provided", - function_data.where, - function_data.thorn, function_data.routine, - prov.c_str()); + } + + + + void Regrid(int const reflevels) + { + DECLARE_CCTK_PARAMETERS; + if (check_requirements) { + if (requirements_verbose) { + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Regrid reflevels=%d", reflevels); } + all_state.regrid(reflevels); } - - // Add the new provisions - for (set::const_iterator pi = - provides.begin(); pi != provides.end(); ++ pi) - { - string const prov = * pi; - active_provisions.insert (prov); + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); } + } + + void all_state_t::regrid(int const reflevels) + { + DECLARE_CCTK_PARAMETERS; + assert(old_vars.empty()); + old_vars.resize(vars.size()); - // ??? - return 1; + int const ng = CCTK_NumGroups(); + for (int gi=0; gi= 0; + break; + default: + assert(0); + } + if (do_cycle) { + // Translate global mode to refinement level 0 + int const rl = reflevel >= 0 ? reflevel : 0; + int const v0 = CCTK_FirstVarIndexI(gi); + int const nv = CCTK_NumVarsInGroupI(gi); + for (int vi=v0; vi= 1) { + // Only cycle variables with sufficient storage + for (int tl=ntl-1; tl>0; --tl) { + tls.AT(tl) = tls.AT(tl-1); + } + // The new time level is uninitialised + // TODO: keep it valid to save time, since MoL will + // copy it anyway? + tls.AT(0) = gridpoint_t(); + } + } + } + } + } + } - // Check one schedule bin - void - CheckOneGroup (cGH const * const cctkGH, - char const * const where) + + void BeforeRoutine(cFunctionData const* const function_data, + int const reflevel, int const map, int const timelevel) { - CCTK_VInfo (CCTK_THORNSTRING, - "Checking requirements of schedule bin %s", where); - - // Set up initial provision (none at the moment) - set active_provisions; - - // Output initial provisions - CCTK_VInfo (CCTK_THORNSTRING, - " Initial provisions:"); - for (set::const_iterator pi = - active_provisions.begin(); pi != active_provisions.end(); ++ pi) + DECLARE_CCTK_PARAMETERS; + if (check_requirements) { + all_state.before_routine(function_data, reflevel, map, timelevel); + } + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); + } + } + + void all_state_t::before_routine(cFunctionData const* const function_data, + int const reflevel, int const map, + int const timelevel) + const + { + // Loop over all clauses + clauses_t const& clauses = all_clauses.get_clauses(function_data); + for (vector::const_iterator iclause = clauses.reads.begin(); + iclause != clauses.reads.end(); + ++iclause) { - string const prov = * pi; - CCTK_VInfo (CCTK_THORNSTRING, - " %s", prov.c_str()); + clause_t const& clause = *iclause; + for (vector::const_iterator ivar = clause.vars.begin(); + ivar != clause.vars.end(); + ++ivar) + { + int const vi = *ivar; + + // Loop over all (refinement levels, maps, time levels) + reflevels_t const& rls = vars.AT(vi); + int const reflevels = int(rls.size()); + int min_rl, max_rl; + if (clause.all_reflevels or reflevel==-1) { + min_rl = 0; max_rl = reflevels; + } else { + min_rl = reflevel; max_rl = min_rl+1; + } + for (int rl=min_rl; rl::const_iterator pi = - active_provisions.begin(); pi != active_provisions.end(); ++ pi) + } + + + + void AfterRoutine(cFunctionData const* const function_data, + int const reflevel, int const map, int const timelevel) + { + DECLARE_CCTK_PARAMETERS; + if (check_requirements) { + all_state.after_routine(function_data, reflevel, map, timelevel); + } + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); + } + } + + void all_state_t::after_routine(cFunctionData const* const function_data, + int const reflevel, int const map, + int const timelevel) + { + // Loop over all clauses + clauses_t const& clauses = all_clauses.get_clauses(function_data); + for (vector::const_iterator iclause = clauses.reads.begin(); + iclause != clauses.reads.end(); + ++iclause) { - string const prov = * pi; - CCTK_VInfo (CCTK_THORNSTRING, - " %s", prov.c_str()); + clause_t const& clause = *iclause; + for (vector::const_iterator ivar = clause.vars.begin(); + ivar != clause.vars.end(); + ++ivar) + { + int const vi = *ivar; + + // Loop over all (refinement levels, maps, time levels) + reflevels_t& rls = vars.AT(vi); + int const reflevels = int(rls.size()); + int min_rl, max_rl; + if (clause.all_reflevels or reflevel==-1) { + min_rl = 0; max_rl = reflevels; + } else { + min_rl = reflevel; max_rl = min_rl+1; + } + for (int rl=min_rl; rl const& groups, + int const reflevel, int const timelevel) { - Checkpoint ("Checking schedule requirements"); - - // Check some bins - CheckOneGroup (cctkGH, "CCTK_WRAGH"); - CheckOneGroup (cctkGH, "CCTK_BASEGRID"); - - CheckOneGroup (cctkGH, "CCTK_RECOVER_VARIABLES"); - CheckOneGroup (cctkGH, "CCTK_POST_RECOVER_VARIABLES"); - - CheckOneGroup (cctkGH, "CCTK_PREREGRIDINITIAL"); - CheckOneGroup (cctkGH, "CCTK_POSTREGRIDINITIAL"); - CheckOneGroup (cctkGH, "CCTK_INITIAL"); - CheckOneGroup (cctkGH, "CCTK_POSTRESTRICTINITIAL"); - CheckOneGroup (cctkGH, "CCTK_POSTINITIAL"); - CheckOneGroup (cctkGH, "CCTK_CPINITIAL"); - - CheckOneGroup (cctkGH, "CCTK_PREREGRID"); - CheckOneGroup (cctkGH, "CCTK_POSTREGRID"); - CheckOneGroup (cctkGH, "CCTK_PRESTEP"); - CheckOneGroup (cctkGH, "CCTK_EVOL"); - CheckOneGroup (cctkGH, "CCTK_POSTSTEP"); - CheckOneGroup (cctkGH, "CCTK_CHECKPOINT"); - CheckOneGroup (cctkGH, "CCTK_ANALYSIS"); - - CheckOneGroup (cctkGH, "CCTK_TERMINATE"); + DECLARE_CCTK_PARAMETERS; + if (check_requirements) { + if (requirements_verbose) { + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Sync reflevel=%d timelevel=%d", + reflevel, timelevel); + } + all_state.sync(function_data, groups, reflevel, timelevel); + } + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); + } } - } // namespace Carpet -} // namespace Requirements - - - -#else // #ifndef CACTUS_HAS_REQUIRES_CLAUSES - - - -namespace Carpet { - namespace Requirements { - // Check one schedule bin - void - CheckRequirements (cGH const * const cctkGH) + void all_state_t::sync(cFunctionData const* const function_data, + vector const& groups, + int const reflevel, int const timelevel) + { + // Loop over all variables + for (vector::const_iterator + igi = groups.begin(); igi != groups.end(); ++igi) + { + int const gi = *igi; + bool do_sync; + int const group_type = CCTK_GroupTypeI(gi); + switch (group_type) { + case CCTK_SCALAR: + case CCTK_ARRAY: + // Grid arrays are synced in global mode + do_sync = reflevel == -1; + break; + case CCTK_GF: + // Grid functions are synced in level mode + do_sync = reflevel >= 0; + break; + default: + assert(0); + } + if (do_sync) { + // Translate global mode to refinement level 0 + int const rl = reflevel >= 0 ? reflevel : 0; + int const v0 = CCTK_FirstVarIndexI(gi); + int const nv = CCTK_NumVarsInGroupI(gi); + for (int vi=v0; vi 0) { + int const crl = rl-1; + maps_t const& cms = rls.AT(crl); + timelevels_t const& ctls = cms.AT(m); + // TODO: use prolongation_order_time instead? + int const ctimelevels = int(ctls.size()); + for (int ctl=0; ctl const& groups, int const reflevel) { - Checkpoint ("Skipping check of schedule requirements (no flesh support)"); - // do nothing + DECLARE_CCTK_PARAMETERS; + if (check_requirements) { + if (requirements_verbose) { + CCTK_VInfo(CCTK_THORNSTRING, + "Requirements: Restrict reflevel=%d", + reflevel); + } + all_state.restrict1(groups, reflevel); + } + if (requirement_inconsistencies_are_fatal and there_was_an_error) { + CCTK_WARN(CCTK_WARN_ABORT, + "Aborting because schedule clauses were not satisfied"); + } } + + void all_state_t::restrict1(vector const& groups, int const reflevel) + { + // Loop over all variables + for (vector::const_iterator + igi = groups.begin(); igi != groups.end(); ++igi) + { + int const gi = *igi; + bool do_restrict; + int const group_type = CCTK_GroupTypeI(gi); + switch (group_type) { + case CCTK_SCALAR: + case CCTK_ARRAY: + // Grid arrays are synced in global mode + do_restrict = reflevel == -1; + break; + case CCTK_GF: + // Grid functions are synced in level mode + do_restrict = reflevel >= 0; + break; + default: + assert(0); + } + if (do_restrict) { + // Translate global mode to refinement level 0 + int const rl = reflevel >= 0 ? reflevel : 0; + int const v0 = CCTK_FirstVarIndexI(gi); + int const nv = CCTK_NumVarsInGroupI(gi); + for (int vi=v0; vi +#include + +#include + +namespace Carpet { + namespace Requirements { + + using namespace std; + + namespace valid { + enum valid_t { nowhere, interior, everywhere }; + } + + // Set up basic grid structure + void Setup(int maps); + // Change number of allocated time levels + void ChangeStorage(vector const& groups, vector const& timelevels, + int reflevel); + // Regrid, set new number of refinement levels, mark all levels as + // invalid + void Regrid(int reflevels); + // Recompose, ensures valid data on one level, indicating whether + // boundaries are valid (e.g. if recomposing was a no-op) + void Recompose(int reflevel, valid::valid_t where); + // Free data structures after regridding + void RegridFree(); + // Cycle time levels + void Cycle(int reflevel); + // Before calling a routine: ensure all reads clauses are + // satisfied + // TODO: Either combine these "before" and "after" routines, or + // split the other routines as well + void BeforeRoutine(cFunctionData const* function_data, + int reflevel, int map, int timelevel); + // After calling a routine: update according to writes clauses + void AfterRoutine(cFunctionData const* function_data, + int reflevel, int map, int timelevel); + // Synchronise and prolongate + // TODO: This does not handle variables that are not prolongated + // TODO: This does not handle buffer zones + void Sync(cFunctionData const* function_data, + vector const& groups, int reflevel, int timelevel); + // Restrict + void Restrict(vector const& groups, int reflevel); + + } // namespace Requirements +} // namespace Carpet + +#endif // #ifndef REQUIREMENTS_HH diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc index e24198dcd..4334b2a4e 100644 --- a/Carpet/Carpet/src/Restrict.cc +++ b/Carpet/Carpet/src/Restrict.cc @@ -11,6 +11,8 @@ #include #include +#include "Requirements.hh" + namespace Carpet { @@ -66,6 +68,8 @@ namespace Carpet { SyncGroups (cctkGH, groups); timer.stop(); } + + Requirements::Restrict(groups, reflevel); } diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc index 95f30ec5d..caa784077 100644 --- a/Carpet/Carpet/src/SetupGH.cc +++ b/Carpet/Carpet/src/SetupGH.cc @@ -35,6 +35,7 @@ #include #include +#include "Requirements.hh" #include "Timers.hh" @@ -717,6 +718,7 @@ namespace Carpet { maps = num_maps; } carpetGH.maps = maps; + Requirements::Setup (maps); } diff --git a/Carpet/Carpet/src/Storage.cc b/Carpet/Carpet/src/Storage.cc index 01104a401..24ae3394b 100644 --- a/Carpet/Carpet/src/Storage.cc +++ b/Carpet/Carpet/src/Storage.cc @@ -14,6 +14,8 @@ #include +#include "Requirements.hh" + namespace Carpet { @@ -217,6 +219,26 @@ namespace Carpet { min_num_timelevels = 0; } + vector vgroups(n_groups), vtimelevels(n_groups); + for (int i=0; i #include #include +#include #include @@ -44,7 +45,8 @@ namespace Carpet { void FlipTimeLevels (cGH* cgh); void FillTimeLevels (const cGH* cgh); void SyncGroups (const cGH* cgh, const vector& groups); - int SyncProlongateGroups (const cGH* cgh, const vector& groups); + int SyncProlongateGroups (const cGH* cgh, const vector& groups, + cFunctionData const* function_data = NULL); // Sanity checks enum checktimes { currenttime, @@ -74,11 +76,6 @@ namespace Carpet { cFunctionData * const attribute, void * const data); - // Requirements - namespace Requirements { - void CheckRequirements (cGH const * cctkGH); - } - // Debugging output void Output (const char* fmt, ...); void Waypoint (const char* fmt, ...); -- cgit v1.2.3