From c0c03fece68597fddfef57948334c5c99154f244 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 2 Apr 2013 17:40:41 -0400 Subject: Requirements: New thorn Move requirement handling code out of Carpet and into a new thorn. --- Carpet/Carpet/interface.ccl | 42 +- Carpet/Carpet/param.ccl | 19 - Carpet/Carpet/schedule.ccl | 17 - Carpet/Carpet/src/CallFunction.cc | 7 +- Carpet/Carpet/src/Comm.cc | 6 +- Carpet/Carpet/src/Cycle.cc | 6 +- Carpet/Carpet/src/Evolve.cc | 10 +- Carpet/Carpet/src/Initialise.cc | 25 +- Carpet/Carpet/src/Requirements.cc | 1560 ------------------------------------- Carpet/Carpet/src/Requirements.hh | 53 -- Carpet/Carpet/src/Restrict.cc | 6 +- Carpet/Carpet/src/SetupGH.cc | 5 +- Carpet/Carpet/src/Storage.cc | 6 +- Carpet/Carpet/src/make.code.defn | 1 - 14 files changed, 62 insertions(+), 1701 deletions(-) delete mode 100644 Carpet/Carpet/src/Requirements.cc delete mode 100644 Carpet/Carpet/src/Requirements.hh (limited to 'Carpet/Carpet') diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl index e36e64d20..6b5143e91 100644 --- a/Carpet/Carpet/interface.ccl +++ b/Carpet/Carpet/interface.ccl @@ -7,6 +7,8 @@ include header: carpet_public.h in carpet.h include header: Timers.hh in CarpetTimers.hh +uses include header: Requirements.hh + uses include header: nompi.h uses include header: loopcontrol.h @@ -294,21 +296,21 @@ PROVIDES FUNCTION GetLocalComponents \ LANGUAGE C CCTK_INT FUNCTION \ - GetMap \ + GetMap \ (CCTK_POINTER_TO_CONST IN cctkGH) PROVIDES FUNCTION GetMap \ WITH Carpet_GetMap \ LANGUAGE C CCTK_INT FUNCTION \ - GetMaps \ + GetMaps \ (CCTK_POINTER_TO_CONST IN cctkGH) PROVIDES FUNCTION GetMaps \ WITH Carpet_GetMaps \ LANGUAGE C CCTK_INT FUNCTION \ - GetTimeLevel \ + GetTimeLevel \ (CCTK_POINTER_TO_CONST IN cctkGH) PROVIDES FUNCTION GetTimeLevel \ WITH Carpet_GetTimeLevel \ @@ -331,40 +333,6 @@ PROVIDES FUNCTION VarDataPtrI \ -# Programmatically check a reads clause -void FUNCTION \ - Requirements_CheckReads \ - (CCTK_POINTER_TO_CONST IN cctkGH, \ - CCTK_INT IN numvars, \ - CCTK_INT ARRAY IN varinds, \ - CCTK_STRING IN reads_clause) -PROVIDES FUNCTION Requirements_CheckReads \ - WITH Carpet_Requirements_CheckReads \ - LANGUAGE C - -# Programmatically record a writes clause -void FUNCTION \ - Requirements_NotifyWrites \ - (CCTK_POINTER_TO_CONST IN cctkGH, \ - CCTK_INT IN numvars, \ - CCTK_INT ARRAY IN varinds, \ - CCTK_STRING IN write_clause) -PROVIDES FUNCTION Requirements_NotifyWrites \ - WITH Carpet_Requirements_NotifyWrites \ - LANGUAGE C - -# Programmatically invalidate a variable -void FUNCTION \ - Requirements_Invalidate \ - (CCTK_POINTER_TO_CONST IN cctkGH, \ - CCTK_INT IN numvars, \ - CCTK_INT ARRAY IN varinds) -PROVIDES FUNCTION Requirements_Invalidate \ - WITH Carpet_Requirements_Invalidate \ - LANGUAGE C - - - # The true prototype of the routine below: # int Carpet_Regrid (const cGH * cctkGH, # gh::rregs * superregss, diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl index 41061c90c..8db365016 100644 --- a/Carpet/Carpet/param.ccl +++ b/Carpet/Carpet/param.ccl @@ -618,22 +618,3 @@ BOOLEAN adaptive_stepsize "Allow adaptive timestep sizes" BOOLEAN use_unusedpoints_mask "Turn on storage and usage of 'unusedpoints_mask'" STEERABLE=recover { } "no" - - - -BOOLEAN check_requirements "Check schedule requirements" STEERABLE=always -{ -} "no" - -BOOLEAN requirements_verbose "Output details when checking requirements" STEERABLE=always -{ -} "no" - -BOOLEAN requirement_inconsistencies_are_fatal "Abort when encountering inconsistencies in requirements" STEERABLE=always -{ -} "no" - -STRING ignore_these_variables "Assume that these variables are always valid everywhere" STEERABLE=recover -{ - "([A-Za-z0-9_]+[:][:][A-Za-z0-9_]+([[:space:]]+|$))*" :: "space separated list of variables of groups" -} "" diff --git a/Carpet/Carpet/schedule.ccl b/Carpet/Carpet/schedule.ccl index 410a6bbc9..d726b58aa 100644 --- a/Carpet/Carpet/schedule.ccl +++ b/Carpet/Carpet/schedule.ccl @@ -47,20 +47,3 @@ if (use_unusedpoints_mask) LANG: C } "Set mask of unused points" } - -# Handle requirements of boundary and symmtery condition. This uses knowledge -# of the internal workings of thorn boundary. -if (check_requirements) -{ - schedule CarpetCheckReadsBeforeBoundary IN ApplyBCs BEFORE BoundaryConditions - { - LANG: C - OPTIONS: singlemap # local would also work but we don't really need it - } "Check that interior of grid function is valid before boundary conditions are applied" - - schedule CarpetNotifyWritesAfterBoundary IN ApplyBCs AFTER Boundary_ApplyPhysicalBCs BEFORE Boundary_ClearSelection - { - LANG: C - OPTIONS: singlemap # local would also work but we don't really need it - } "Notify that grid functions with boundary condtions are valid in boundary" -} diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc index 2cf661040..deefa523c 100644 --- a/Carpet/Carpet/src/CallFunction.cc +++ b/Carpet/Carpet/src/CallFunction.cc @@ -10,13 +10,14 @@ #include #include +#include + #include #include #include #include "adler32.hh" -#include "Requirements.hh" @@ -358,7 +359,9 @@ namespace Carpet { CCTK_REAL const saved_cctk_delta_time = cctkGH->cctk_delta_time; user_timer.start(); +#ifdef REQUIREMENTS_HH Requirements::BeforeRoutine(attribute, reflevel, map, timelevel); +#endif timer.start(); if (CCTK_IsFunctionAliased("Accelerator_PreCallFunction")) { Timer pre_timer("PreCall"); @@ -375,7 +378,9 @@ namespace Carpet { post_timer.stop(); } timer.stop(); +#ifdef REQUIREMENTS_HH Requirements::AfterRoutine(attribute, reflevel, map, timelevel); +#endif user_timer.stop(); // Manage the time step size. If the time step size changes diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc index df529aeaa..c38a3b4f2 100644 --- a/Carpet/Carpet/src/Comm.cc +++ b/Carpet/Carpet/src/Comm.cc @@ -8,14 +8,14 @@ #include #include +#include + #include #include #include #include -#include "Requirements.hh" - namespace Carpet { @@ -165,7 +165,9 @@ namespace Carpet { timer.stop(); } +#ifdef REQUIREMENTS_HH Requirements::Sync(function_data, goodgroups, reflevel, timelevel); +#endif } diff --git a/Carpet/Carpet/src/Cycle.cc b/Carpet/Carpet/src/Cycle.cc index a3c254b55..523bc694e 100644 --- a/Carpet/Carpet/src/Cycle.cc +++ b/Carpet/Carpet/src/Cycle.cc @@ -4,14 +4,14 @@ #include #include +#include + #include #include #include #include -#include "Requirements.hh" - namespace Carpet { @@ -110,7 +110,9 @@ namespace Carpet { Accelerator_Cycle(cctkGH); } +#ifdef REQUIREMENTS_HH Requirements::Cycle(reflevel); +#endif if (errors > 0) { CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING, diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc index 2d1541f72..8864ee9ee 100644 --- a/Carpet/Carpet/src/Evolve.cc +++ b/Carpet/Carpet/src/Evolve.cc @@ -13,6 +13,8 @@ #include #include +#include + #include #include @@ -23,8 +25,6 @@ #include #include -#include "Requirements.hh" - namespace Carpet { @@ -295,7 +295,9 @@ namespace Carpet { assert (not did_remove_level or did_regrid); if (did_regrid) { +#ifdef REQUIREMENTS_HH Requirements::Regrid(reflevels); +#endif bool did_any_recompose = false; BEGIN_META_MODE (cctkGH) { @@ -308,10 +310,12 @@ namespace Carpet { bool const did_recompose = Recompose (cctkGH, rl, true); did_any_recompose = did_any_recompose or did_recompose; +#ifdef REQUIREMENTS_HH Requirements::Recompose(rl, not did_recompose ? Requirements::valid::everywhere : Requirements::valid::interior); +#endif // Carpet assumes that a regridding operation always changes // "level N and all finer levels" so we should call @@ -372,7 +376,9 @@ namespace Carpet { if (have_done_anything) assert (have_done_late_global_mode); } END_META_MODE; +#ifdef REQUIREMENTS_HH Requirements::RegridFree(); +#endif } // if did_regrid RegridFree (cctkGH, true); diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc index 3599c2d98..0668e28c0 100644 --- a/Carpet/Carpet/src/Initialise.cc +++ b/Carpet/Carpet/src/Initialise.cc @@ -13,13 +13,13 @@ #include #include +#include + #include #include #include #include -#include "Requirements.hh" - namespace Carpet { @@ -188,7 +188,10 @@ namespace Carpet { ScheduleTraverse (where, "CCTK_PARAMCHECK", cctkGH); } END_MGLEVEL_LOOP; +#ifdef REQUIREMENTS_HH + // TODO: Why is this disabled? // Requirements::CheckRequirements (cctkGH); +#endif CCTKi_FinaliseParamWarn(); @@ -923,7 +926,9 @@ namespace Carpet { assert (not did_remove_level or did_regrid); if (did_regrid) { +#ifdef REQUIREMENTS_HH Requirements::Regrid(reflevels); +#endif bool did_any_recompose = false; BEGIN_META_MODE (cctkGH) { @@ -937,12 +942,14 @@ namespace Carpet { bool did_recompose = false; did_recompose = Recompose (cctkGH, rl, prolongate_initial_data); did_any_recompose = did_any_recompose or did_recompose; +#ifdef REQUIREMENTS_HH Requirements::Recompose(rl, not did_recompose ? Requirements::valid::everywhere : prolongate_initial_data ? Requirements::valid::interior : Requirements::valid::nowhere); +#endif // Carpet assumes that a regridding operation always changes "level N // and all finer levels" so we should call POSTREGRID on all finer levels @@ -995,7 +1002,9 @@ namespace Carpet { if (have_done_anything) assert (have_done_late_global_mode); } END_META_MODE; +#ifdef REQUIREMENTS_HH Requirements::RegridFree(); +#endif } // if did_regrid RegridFree (cctkGH, prolongate_initial_data); @@ -1052,17 +1061,23 @@ namespace Carpet { bool const did_regrid = Regrid (cctkGH, true, prolongate_initial_data); if (did_regrid) { +#ifdef REQUIREMENTS_HH Requirements::Regrid(reflevels); +#endif for (int rl=0; rl - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - - - -namespace Carpet { - namespace Requirements { - - - - // Rules: - // - // 1. Everything that is required by a routine must be provided by - // another routine which is scheduled earlier. - // - // 2. Things can be provided only once, not multiple times. - // Except when they are also required. - - - // taken from defs.cc and defs.hh - // Vector output - template - ostream& output (ostream& os, const vector& v) { - os << "["; - // Do not number the elements, as this would lead to a format that - // cannot be read back in. - // int cnt=0; - for (typename vector::const_iterator ti=v.begin(); ti!=v.end(); ++ti) { - if (ti!=v.begin()) os << ","; - // os << cnt++ << ":"; - os << *ti; - } - os << "]"; - return os; - } - - template - inline ostream& operator<< (ostream& os, const vector& v) { - return Carpet::Requirements::output(os,v); - } - - - // Represent scheduled functions and their dependencies - - struct clause_t { - bool everywhere; // all grid points (everywhere) - bool interior; // all interior points - bool boundary; // all boundary points, excluding - // ghostzones - bool boundary_ghostzones; // all boundary ghost points - bool timelevel0, timelevel1, timelevel2; - bool all_timelevels; // all time levels - bool all_maps; // all maps (i.e. level mode) - bool all_reflevels; // all refinement levels (i.e. global mode) - vector vars; - clause_t(): - everywhere(false), - interior(false), boundary(false), boundary_ghostzones(false), - timelevel0(false), timelevel1(false), timelevel2(false), - all_timelevels(false), all_maps(false), all_reflevels(false) - {} - void interpret_options(cFunctionData const* function_data); - void parse_clause(char const* clause); - int min_num_timelevels() const; - bool active_on_timelevel(int tl) const; - - // Input/Output helpers - void input (istream& is); - void output (ostream& os) const; - }; - - void clause_t::interpret_options(cFunctionData const* const function_data) - { - if (function_data->meta or - function_data->meta_early or - function_data->meta_late or - function_data->global or - function_data->global_early or - function_data->global_late) - { - assert(not all_reflevels); - all_reflevels = true; - } - if (function_data->level) { - assert(not all_maps); - all_maps = true; - } - // Ignore singlemap and local options - // Ignore loop_* options - } - - void clause_t::parse_clause(char const* const clause1) - { - char* const clause = strdup(clause1); - char* p = clause; - - // Remove trailing "(...)" modifier, if any - p = strchr(p, '('); - if (p) *p = '\0'; - int const gi = CCTK_GroupIndex(clause); - if (gi >= 0) { - // A group - int const v0 = CCTK_FirstVarIndexI(gi); assert(v0 >= 0); - int const nv = CCTK_NumVarsInGroupI(gi); assert(nv >= 0); - for (int vi=v0; vi= 0); - vars.push_back(vi); - } - - // Parse modifiers - // TODO: Use CarpetLib parser for this - // TODO: add user friendly error messages - // TODO: teach the flesh about commas within the READS/WRITES block - if (p) { - ++p; - for (;;) { - size_t const len = strcspn(p, ";)"); - char const c = p[len]; - assert(c); - p[len] = '\0'; - if (CCTK_EQUALS(p, "everywhere")) { - assert(not everywhere and - not interior and not boundary and not boundary_ghostzones); - everywhere = true; - } else if (CCTK_EQUALS(p, "interior")) { - assert(not everywhere and not interior); - interior = true; - } else if (CCTK_EQUALS(p, "boundary")) { - assert(not everywhere and not boundary); - boundary = true; - } else if (CCTK_EQUALS(p, "boundary_ghostzones")) { - assert(not everywhere and not boundary_ghostzones); - boundary_ghostzones = true; - } else if (CCTK_EQUALS(p, "timelevel0")) { - assert(not timelevel0 and not all_timelevels); - timelevel0 = true; - } else if (CCTK_EQUALS(p, "timelevel1")) { - assert(not timelevel1 and not all_timelevels); - timelevel1 = true; - } else if (CCTK_EQUALS(p, "timelevel2")) { - assert(not timelevel2 and not all_timelevels); - timelevel2 = true; - } else if (CCTK_EQUALS(p, "all_timelevels")) { - // TODO: look at schedule group instead - assert(not timelevel0 and not timelevel1 and not timelevel2 and - not all_timelevels); - all_timelevels = true; - } else { - CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING, - "Unknown modifier '%s' in clause '%s'", p, clause1); - } - if (c == ')') break; - assert(c==';'); - p += len+1; - } - } - - free(clause); - } - - int clause_t::min_num_timelevels() const - { - if (timelevel2) return 3; - if (timelevel1) return 2; - return 1; - } - - bool clause_t::active_on_timelevel(int const tl) const - { - assert(tl>=0); - if (all_timelevels) return true; - if (timelevel0 and tl==0) return true; - if (timelevel1 and tl==1) return true; - if (timelevel2 and tl==2) return true; - bool const no_timelevel_clause = - not timelevel0 and not timelevel1 and not timelevel2; - if (tl==0 and no_timelevel_clause) return true; - return false; - } - - inline ostream& operator<< (ostream& os, const clause_t& a) { - a.output(os); - return os; - } - - void clause_t::output(ostream& os) const - { - char* const groupname = CCTK_GroupNameFromVarI(vars.at(0)); - os << groupname; - free(groupname); - os << "{"; - for (vector::const_iterator ivi = vars.begin(); - ivi != vars.end(); - ++ivi) - { - if (ivi != vars.begin()) - os << ","; - char* const fullname = CCTK_FullName(*ivi); - os << fullname; - free(fullname); - } - os << "}("; - if(everywhere) os << "everywhere;"; - if(interior) os << "interior;"; - if(boundary) os << "boundary;"; - if(boundary_ghostzones) os << "boundary_ghostzones;"; - if(timelevel0) os << "timelevel0;"; - if(timelevel1) os << "timelevel1;"; - if(timelevel2) os << "timelevel2;"; - if(all_timelevels) os << "all_timelevels;"; - if(all_maps) os << "all_maps;"; - if(all_reflevels) os << "all_reflevels;"; - os << ")"; - } - - - - struct clauses_t { - vector reads, writes; - clauses_t() {} - void setup(cFunctionData const* function_data); - - // Input/Output helpers - void input (istream& is); - void output (ostream& os) const; - }; - - void clauses_t::setup(cFunctionData const* const function_data) - { - clause_t prototype; - prototype.interpret_options(function_data); - reads.reserve(function_data->n_ReadsClauses); - for (int n=0; nn_ReadsClauses; ++n) { - clause_t clause(prototype); - clause.parse_clause(function_data->ReadsClauses[n]); - reads.push_back(clause); - } - writes.reserve(function_data->n_WritesClauses); - for (int n=0; nn_WritesClauses; ++n) { - clause_t clause(prototype); - clause.parse_clause(function_data->WritesClauses[n]); - writes.push_back(clause); - } - } - - inline ostream& operator<< (ostream& os, const clauses_t& a) { - a.output(os); - return os; - } - - void clauses_t::output(ostream& os) const - { - os << "reads = " << reads << ", writes = " << writes; - } - - - - class all_clauses_t { - // TODO: Represent I/O as well? - typedef std::map clauses_map_t; - clauses_map_t clauses_map; - // Singleton - all_clauses_t(all_clauses_t const&); - all_clauses_t& operator=(all_clauses_t const&); - public: - all_clauses_t() {} - clauses_t const& get_clauses(cFunctionData const* function_data); - void remove_clauses(cFunctionData const* function_data); - - // Input/Output helpers - void input (istream& is); - void output (ostream& os) const; - }; - - clauses_t const& all_clauses_t:: - get_clauses(cFunctionData const* const function_data) - { - clauses_map_t::const_iterator const iclauses = - clauses_map.find(function_data); - if (iclauses != clauses_map.end()) return *iclauses->second; - clauses_t* const clauses = new clauses_t; - clauses->setup(function_data); - pair const ret = - clauses_map.insert(clauses_map_t::value_type(function_data, clauses)); - assert(ret.second); - return *ret.first->second; - } - - void all_clauses_t:: - remove_clauses(cFunctionData const* const function_data) - { - clauses_map_t::iterator const iclauses = - clauses_map.find(function_data); - if (iclauses != clauses_map.end()) { - clauses_map.erase(iclauses); - } - return; - } - - inline ostream& operator<< (ostream& os, const all_clauses_t& a) { - a.output(os); - return os; - } - - void all_clauses_t::output(ostream& os) const - { - os << "all_clauses: {" << std::endl; - for (std::map::const_iterator ti=clauses_map.begin(); - ti!=clauses_map.end(); - ++ti) - { - if (ti!=clauses_map.begin()) os << ","; - os << ti->first->thorn << "::" - << ti->first->routine << " in " - << ti->first->where << ": " << *ti->second << std::endl; - } - os << "}"; - } - - all_clauses_t all_clauses; - - // ignore requirements in these variables. Used for internally updated - // variables. Putting a variable in this set asserts that it is always - // valid. - std::set ignore_these_varindices; - - - // Keep track of which time levels contain good data; modify this - // while time level cycling; routines should specify how many time - // levels they require/provide - - bool there_was_an_error = false; - bool there_was_a_warning = false; - - struct gridpoint_t { - bool interior, boundary, ghostzones, boundary_ghostzones; - gridpoint_t(): - interior(false), boundary(false), ghostzones(false), - boundary_ghostzones(false) - {} - gridpoint_t(clause_t const& clause): - interior(clause.everywhere or clause.interior), - boundary(clause.everywhere or clause.boundary), - ghostzones(clause.everywhere), - boundary_ghostzones(clause.everywhere or clause.boundary_ghostzones) - {} - void check_state(clause_t const& clause, - cFunctionData const* function_data, - int vi, int rl, int m, int tl) const; - void report_error(cFunctionData const* function_data, - int vi, int rl, int m, int tl, - char const* what, char const* where) const; - void report_warning(cFunctionData const* function_data, - int vi, int rl, int m, int tl, - char const* what, char const* where) const; - void update_state(clause_t const& clause); - - // Input/Output helpers - void input (istream& is); - void output (ostream& os) const; - }; - - inline ostream& operator<< (ostream& os, const gridpoint_t& a) { - a.output(os); - return os; - } - - void gridpoint_t::check_state(clause_t const& clause, - cFunctionData const* const function_data, - int const vi, - int const rl, int const m, int const tl) - const - { - if (not interior) { - if (clause.everywhere or clause.interior) { - report_error(function_data, vi, rl, m, tl, - "calling function", "interior"); - } - } - if (not boundary) { - if (clause.everywhere or clause.boundary) { - report_error(function_data, vi, rl, m, tl, - "calling function", "boundary"); - } - } - if (not ghostzones) { - if (clause.everywhere) { - report_error(function_data, vi, rl, m, tl, - "calling function", "ghostzones"); - } - } - if (not boundary_ghostzones) { - if (clause.everywhere or clause.boundary_ghostzones) { - report_error(function_data, vi, rl, m, tl, - "calling", "boundary-ghostzones"); - } - } - } - - void gridpoint_t::report_error(cFunctionData const* const function_data, - int const vi, - int const rl, int const m, int const tl, - char const* const what, - char const* const where) const - { - char* const fullname = CCTK_FullName(vi); - ostringstream state; - state << "current state: " << *this << std::endl; - if (function_data) { - // The error is related to a scheduled function - CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, - "Schedule READS clause not satisfied: " - "Function %s::%s in %s: " - "Variable %s reflevel=%d map=%d timelevel=%d: " - "%s not valid for %s. %s", - function_data->thorn, function_data->routine, - function_data->where, - fullname, rl, m, tl, - where, what, state.str().c_str()); - } else { - // The error is not related to a scheduled function - CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, - "Schedule READS clause not satisfied: " - "Variable %s reflevel=%d map=%d timelevel=%d: " - "%s not valid for %s. %s", - fullname, rl, m, tl, - where, what, state.str().c_str()); - } - free(fullname); - there_was_an_error = true; - } - - void gridpoint_t::report_warning(cFunctionData const* const function_data, - int const vi, - int const rl, int const m, int const tl, - char const* const what, - char const* const where) const - { - char* const fullname = CCTK_FullName(vi); - ostringstream state; - state << "current state: " << *this << std::endl; - if (function_data) { - // The error is related to a scheduled function - CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, - "Schedule WRITES clause is superfluous: " - "Function %s::%s in %s: " - "Variable %s reflevel=%d map=%d timelevel=%d: " - "%s already valid for %s. %s", - function_data->thorn, function_data->routine, - function_data->where, - fullname, rl, m, tl, - where, what, state.str().c_str()); - } else { - // The error is not related to a scheduled function - CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING, - "Schedule WRITES clause already satisfied: " - "Variable %s reflevel=%d map=%d timelevel=%d: " - "%s already valid for %s. %s", - fullname, rl, m, tl, - where, what, state.str().c_str()); - } - free(fullname); - there_was_a_warning = true; - } - - void gridpoint_t::update_state(clause_t const& clause) - { - if (clause.everywhere or clause.interior) { - interior = true; - } - if (clause.everywhere or clause.boundary) { - boundary = true; - } - if (clause.everywhere) { - ghostzones = true; - } - if (clause.everywhere or clause.boundary_ghostzones) { - boundary_ghostzones = true; - } - } - - void gridpoint_t::output(ostream& os) const - { - os << "("; - if(interior) os << "interior;"; - if(boundary) os << "boundary;"; - if(ghostzones) os << "ghostzones;"; - if(boundary_ghostzones) os << "boundary_ghostzones;"; - os << ")"; - } - - - - class all_state_t { - typedef vector timelevels_t; - typedef vector maps_t; - typedef vector reflevels_t; - typedef vector variables_t; - variables_t vars; - variables_t old_vars; // for regridding - public: - void setup(int maps); - void change_storage(vector const& groups, - vector const& timelevels, - int reflevel); - void regrid(int reflevels); - void recompose(int reflevel, valid::valid_t where); - void regrid_free(); - void cycle(int reflevel); - void before_routine(cFunctionData const* function_data, - int reflevel, int map, int timelevel) const; - void after_routine(cFunctionData const* function_data, - int reflevel, int map, int timelevel); - void sync(cFunctionData const* function_data, - vector const& groups, int reflevel, int timelevel); - void restrict1(vector const& groups, int reflevel); - void invalidate(vector const& vars, - int reflevel, int map, int timelevel); - - // Input/Output helpers - void input (istream& is); - void output (ostream& os) const; - }; - - all_state_t all_state; - - - static void add_ignored_variable(int id, const char * opstring, void * callback_arg) - { - std::set& ignore_these_variables = - *static_cast*>(callback_arg); - - ignore_these_variables.insert(id); - } - - void Setup(int const maps) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - if (requirements_verbose) { - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Setup maps=%d", maps); - } - all_state.setup(maps); - CCTK_TraverseString(ignore_these_variables, add_ignored_variable, - (void*)&ignore_these_varindices, - CCTK_GROUP_OR_VAR); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::setup(int const maps) - { - DECLARE_CCTK_PARAMETERS; - assert(vars.empty()); - vars.resize(CCTK_NumVars()); - for (variables_t::iterator - ivar = vars.begin(); ivar != vars.end(); ++ivar) - { - reflevels_t& rls = *ivar; - int const vi = &*ivar - &*vars.begin(); - assert(rls.empty()); - // Allocate one refinement level initially - int const nrls = 1; - rls.resize(nrls); - for (reflevels_t::iterator irl = rls.begin(); irl != rls.end(); ++irl) { - maps_t& ms = *irl; - assert(ms.empty()); - int const group_type = CCTK_GroupTypeFromVarI(vi); - int const nms = group_type==CCTK_GF ? maps : 1; - if (requirements_verbose) { - char* const fullname = CCTK_FullName(vi); - int const rl = &*irl - &*rls.begin(); - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Setting up %d maps for variable %s(rl=%d)", - nms, fullname, rl); - free(fullname); - } - ms.resize(nms); - for (maps_t::iterator im = ms.begin(); im != ms.end(); ++im) { - timelevels_t& tls = *im; - assert(tls.empty()); - // Not allocating any time levels here - } - } - } - } - - - - void ChangeStorage(vector const& groups, - vector const& timelevels, - int const reflevel) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - if (requirements_verbose) { - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: ChangeStorage reflevel=%d", reflevel); - } - all_state.change_storage(groups, timelevels, reflevel); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::change_storage(vector const& groups, - vector const& timelevels, - int const reflevel) - { - DECLARE_CCTK_PARAMETERS; - assert(groups.size() == timelevels.size()); - for (vector::const_iterator - igi = groups.begin(), itl = timelevels.begin(); - igi != groups.end(); ++igi, ++itl) - { - int const gi = *igi; - int const tl = *itl; - bool const is_array = CCTK_GroupTypeI(gi) != CCTK_GF; - int const v0 = CCTK_FirstVarIndexI(gi); - int const nv = CCTK_NumVarsInGroupI(gi); - for (int vi=v0; vi=0 and max_rl<=reflevels); - for (int rl=min_rl; rl ntls) { - // Allocate new storage - if (requirements_verbose) { - char* const fullname = CCTK_FullName(vi); - int const m = &*im - &*ms.begin(); - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Increasing storage to %d time levels for variable %s(rl=%d,m=%d)", - tl, fullname, rl, m); - free(fullname); - } - // The default constructor for gridpoint_t sets all - // data to "invalid" - tls.resize(tl); - } - } - } - } - } - } - - - - void Regrid(int const reflevels) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - if (requirements_verbose) { - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Regrid reflevels=%d", reflevels); - } - all_state.regrid(reflevels); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::regrid(int const reflevels) - { - DECLARE_CCTK_PARAMETERS; - assert(old_vars.empty()); - old_vars.resize(vars.size()); - - int const ng = CCTK_NumGroups(); - for (int gi=0; gi= 0; - break; - default: - assert(0); - } - if (do_cycle) { - // Translate global mode to refinement level 0 - int const rl = reflevel >= 0 ? reflevel : 0; - int const v0 = CCTK_FirstVarIndexI(gi); - int const nv = CCTK_NumVarsInGroupI(gi); - for (int vi=v0; vi= 1) { - // Only cycle variables with sufficient storage - for (int tl=ntl-1; tl>0; --tl) { - tls.AT(tl) = tls.AT(tl-1); - } - // The new time level is uninitialised - // TODO: keep it valid to save time, since MoL will - // copy it anyway? - tls.AT(0) = gridpoint_t(); - } - } - } - } - } - } - - - - void BeforeRoutine(cFunctionData const* const function_data, - int const reflevel, int const map, int const timelevel) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - all_state.before_routine(function_data, reflevel, map, timelevel); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::before_routine(cFunctionData const* const function_data, - int const reflevel, int const map, - int const timelevel) - const - { - // Loop over all clauses - clauses_t const& clauses = all_clauses.get_clauses(function_data); - for (vector::const_iterator iclause = clauses.reads.begin(); - iclause != clauses.reads.end(); - ++iclause) - { - clause_t const& clause = *iclause; - for (vector::const_iterator ivar = clause.vars.begin(); - ivar != clause.vars.end(); - ++ivar) - { - int const vi = *ivar; - - if (ignore_these_varindices.count(vi)) - continue; - - // Loop over all (refinement levels, maps, time levels) - reflevels_t const& rls = vars.AT(vi); - int const reflevels = int(rls.size()); - int min_rl, max_rl; - if (clause.all_reflevels or reflevel==-1) { - min_rl = 0; max_rl = reflevels; - } else { - min_rl = reflevel; max_rl = min_rl+1; - } - for (int rl=min_rl; rl= clause.min_num_timelevels()); - // TODO: properly handle timelevels the way enter_local_mode() does - const int mintl = timelevel == 0 || timelevels == 1 ? 0 : timelevel; - const int maxtl = timelevel == 0 || timelevels == 1 ? timelevels-1 : timelevel; - const int tl_of = timelevels > 1 ? timelevel : 0; - for (int tl=mintl; tl<=maxtl; ++tl) { - if (timelevel==-1 or clause.active_on_timelevel(tl-tl_of)) { - gridpoint_t const& gp = tls.AT(tl); - gp.check_state(clause, function_data, vi, rl, m, tl); - } - } - - } - } - - } - } - } - - - - void AfterRoutine(cFunctionData const* const function_data, - int const reflevel, int const map, int const timelevel) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - all_state.after_routine(function_data, reflevel, map, timelevel); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::after_routine(cFunctionData const* const function_data, - int const reflevel, int const map, - int const timelevel) - { - // Loop over all clauses - clauses_t const& clauses = all_clauses.get_clauses(function_data); - for (vector::const_iterator iclause = clauses.writes.begin(); - iclause != clauses.writes.end(); - ++iclause) - { - clause_t const& clause = *iclause; - for (vector::const_iterator ivar = clause.vars.begin(); - ivar != clause.vars.end(); - ++ivar) - { - int const vi = *ivar; - - // Loop over all (refinement levels, maps, time levels) - reflevels_t& rls = vars.AT(vi); - int const reflevels = int(rls.size()); - int min_rl, max_rl; - if (clause.all_reflevels or reflevel==-1) { - min_rl = 0; max_rl = reflevels; - } else { - min_rl = reflevel; max_rl = min_rl+1; - } - for (int rl=min_rl; rl= clause.min_num_timelevels()); - // TODO: properly handle timelevels the way enter_local_mode() does - const int mintl = timelevel == 0 || timelevels == 1 ? 0 : timelevel; - const int maxtl = timelevel == 0 || timelevels == 1 ? timelevels-1 : timelevel; - const int tl_of = timelevels > 1 ? timelevel : 0; - for (int tl=mintl; tl<=maxtl; ++tl) { - if (timelevel==-1 or clause.active_on_timelevel(tl-tl_of)) { - gridpoint_t& gp = tls.AT(tl); - gp.update_state(clause); - } - } - - } - } - - } - } - } - - - - void Sync(cFunctionData const* const function_data, - vector const& groups, - int const reflevel, int const timelevel) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - if (requirements_verbose) { - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Sync reflevel=%d timelevel=%d", - reflevel, timelevel); - } - all_state.sync(function_data, groups, reflevel, timelevel); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::sync(cFunctionData const* const function_data, - vector const& groups, - int const reflevel, int const timelevel) - { - // Loop over all variables - for (vector::const_iterator - igi = groups.begin(); igi != groups.end(); ++igi) - { - int const gi = *igi; - bool do_sync; - int const group_type = CCTK_GroupTypeI(gi); - switch (group_type) { - case CCTK_SCALAR: - case CCTK_ARRAY: - // Grid arrays are synced in global mode - do_sync = reflevel == -1; - break; - case CCTK_GF: - // Grid functions are synced in level mode - do_sync = reflevel >= 0; - break; - default: - assert(0); - } - if (do_sync) { - // Translate global mode to refinement level 0 - int const rl = reflevel >= 0 ? reflevel : 0; - int const v0 = CCTK_FirstVarIndexI(gi); - int const nv = CCTK_NumVarsInGroupI(gi); - for (int vi=v0; vi 0) { - int const crl = rl-1; - maps_t const& cms = rls.AT(crl); - timelevels_t const& ctls = cms.AT(m); - // TODO: use prolongation_order_time instead? - int const ctimelevels = int(ctls.size()); - for (int ctl=0; ctl const& groups, int const reflevel) - { - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - if (requirements_verbose) { - CCTK_VInfo(CCTK_THORNSTRING, - "Requirements: Restrict reflevel=%d", - reflevel); - } - all_state.restrict1(groups, reflevel); - } - if (requirement_inconsistencies_are_fatal and there_was_an_error) { - CCTK_WARN(CCTK_WARN_ABORT, - "Aborting because schedule clauses were not satisfied"); - } - } - - void all_state_t::restrict1(vector const& groups, int const reflevel) - { - // Loop over all variables - for (vector::const_iterator - igi = groups.begin(); igi != groups.end(); ++igi) - { - int const gi = *igi; - bool do_restrict; - int const group_type = CCTK_GroupTypeI(gi); - switch (group_type) { - case CCTK_SCALAR: - case CCTK_ARRAY: - // Grid arrays are synced in global mode - do_restrict = reflevel == -1; - break; - case CCTK_GF: - // Grid functions are synced in level mode - do_restrict = reflevel >= 0; - break; - default: - assert(0); - } - if (do_restrict) { - // Translate global mode to refinement level 0 - int const rl = reflevel >= 0 ? reflevel : 0; - int const v0 = CCTK_FirstVarIndexI(gi); - int const nv = CCTK_NumVarsInGroupI(gi); - for (int vi=v0; vi& v); - template ostream& output (ostream& os, const vector& v); - template ostream& output (ostream& os, const vector& v); - template ostream& output (ostream& os, const vector& v); - template ostream& output (ostream& os, const vector& v); - - - - //////////////////////////////////////////////////////////////////////////// - - - - extern "C" - void Carpet_Requirements_CheckReads(CCTK_POINTER_TO_CONST const cctkGH_, - CCTK_INT const nvars, - CCTK_INT const* const varinds, - char const* const clause) - { - cGH const* const cctkGH = static_cast(cctkGH_); - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - // TODO: come up with a scheme to avoid constructing and destroying clauses - cFunctionData const* const function_data = - CCTK_ScheduleQueryCurrentFunction(cctkGH); - int const reflevel = GetRefinementLevel(cctkGH); - int const map = GetMap(cctkGH); - int const timelevel = GetTimeLevel(cctkGH); - // TODO: design an interface to all_state.before_routine that operates - // on indices and claues directly - for (int v=0; v 0); - temp_function_data.n_WritesClauses = 0; - temp_function_data.WritesClauses = NULL; - temp_function_data.n_ReadsClauses = 1; - temp_function_data.ReadsClauses = (char const**)&reads; - all_clauses.get_clauses(&temp_function_data); - BeforeRoutine(&temp_function_data, reflevel, map, timelevel); - all_clauses.remove_clauses(&temp_function_data); - free(fullname); - free(reads); - } - } - } - - extern "C" - void Carpet_Requirements_NotifyWrites(CCTK_POINTER_TO_CONST const cctkGH_, - CCTK_INT const nvars, - CCTK_INT const* const varinds, - char const* const clause) - { - cGH const* const cctkGH = static_cast(cctkGH_); - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - // TODO: come up with a scheme to avoid constructing and destroying clauses - cFunctionData const* const function_data = - CCTK_ScheduleQueryCurrentFunction(cctkGH); - int const reflevel = GetRefinementLevel(cctkGH); - int const map = GetMap(cctkGH); - int const timelevel = GetTimeLevel(cctkGH); - // TODO: design an interface to all_state.before_routine that operates - // on indices and claues directly - for (int v=0; v 0); - temp_function_data.n_WritesClauses = 1; - temp_function_data.WritesClauses = (char const**)&writes; - temp_function_data.n_ReadsClauses = 0; - temp_function_data.ReadsClauses = NULL; - all_clauses.get_clauses(&temp_function_data); - AfterRoutine(&temp_function_data, reflevel, map, timelevel); - all_clauses.remove_clauses(&temp_function_data); - free(fullname); - free(writes); - } - } - } - - extern "C" - void Carpet_Requirements_Invalidate(CCTK_POINTER_TO_CONST const cctkGH_, - CCTK_INT const nvars, - CCTK_INT const* const varinds) - { - cGH const* const cctkGH = static_cast(cctkGH_); - DECLARE_CCTK_PARAMETERS; - if (check_requirements) { - vector vars(nvars); - for (int v=0; v const& vars1, - int const reflevel, int const map, - int const timelevel) - { - // Loop over all variables - for (vector::const_iterator - ivi = vars1.begin(); ivi != vars1.end(); ++ivi) - { - int const vi = *ivi; - reflevels_t& rls = vars.AT(vi); - maps_t& ms = rls.AT(reflevel); - timelevels_t& tls = ms.AT(map); - // This time level is uninitialised - tls.AT(timelevel) = gridpoint_t(); - } - } - - - - //////////////////////////////////////////////////////////////////////////// - - - - // scheduled routines to handle boundary and symmetry conditions - extern "C" - void CarpetCheckReadsBeforeBoundary(CCTK_ARGUMENTS) - { - DECLARE_CCTK_ARGUMENTS; - int num_vars, err; - vector vars, faces, widths, tables; - - num_vars = Boundary_SelectedGVs(cctkGH, 0, NULL, NULL, NULL, NULL, NULL); - if (num_vars < 0) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Error retrieving number of selected GVs: %d", num_vars); - } - vars.resize(num_vars); - faces.resize(num_vars); - widths.resize(num_vars); - tables.resize(num_vars); - - /* get selected vars for all bc */ - err = Boundary_SelectedGVs(cctkGH, num_vars, &vars[0], &faces[0], &widths[0], &tables[0], - NULL); - if (err<0) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Error in Boundary_SelectedGVs for all boundary conditions"); - } else if (err != num_vars) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Boundary_SelectedGVs returned %d selected variables for " - "all boundary conditions, but %d expected\n", err, - num_vars); - } - - Requirements_CheckReads(cctkGH, num_vars, &vars[0], "interior"); - } - - extern "C" - void CarpetNotifyWritesAfterBoundary(CCTK_ARGUMENTS) - { - DECLARE_CCTK_ARGUMENTS; - int num_vars, err; - vector vars, faces, widths, tables; - - num_vars = Boundary_SelectedGVs(cctkGH, 0, NULL, NULL, NULL, NULL, NULL); - if (num_vars < 0) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Error retrieving number of selected GVs: %d", num_vars); - } - vars.resize(num_vars); - faces.resize(num_vars); - widths.resize(num_vars); - tables.resize(num_vars); - - /* get selected vars for all bc */ - err = Boundary_SelectedGVs(cctkGH, num_vars, &vars[0], &faces[0], &widths[0], &tables[0], - NULL); - if (err<0) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Error in Boundary_SelectedGVs for all boundary conditions"); - } else if (err != num_vars) { - CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Boundary_SelectedGVs returned %d selected variables for " - "all boundary conditions, but %d expected\n", err, - num_vars); - } - - Requirements_NotifyWrites(cctkGH, num_vars, &vars[0], "boundary;boundary_ghostzones"); - } - - - - } // namespace Carpet -} // namespace Requirements diff --git a/Carpet/Carpet/src/Requirements.hh b/Carpet/Carpet/src/Requirements.hh deleted file mode 100644 index 4a458918e..000000000 --- a/Carpet/Carpet/src/Requirements.hh +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef REQUIREMENTS_HH -#define REQUIREMENTS_HH - -#include -#include - -#include - -namespace Carpet { - namespace Requirements { - - using namespace std; - - namespace valid { - enum valid_t { nowhere, interior, everywhere }; - } - - // Set up basic grid structure - void Setup(int maps); - // Change number of allocated time levels - void ChangeStorage(vector const& groups, vector const& timelevels, - int reflevel); - // Regrid, set new number of refinement levels, mark all levels as - // invalid - void Regrid(int reflevels); - // Recompose, ensures valid data on one level, indicating whether - // boundaries are valid (e.g. if recomposing was a no-op) - void Recompose(int reflevel, valid::valid_t where); - // Free data structures after regridding - void RegridFree(); - // Cycle time levels - void Cycle(int reflevel); - // Before calling a routine: ensure all reads clauses are - // satisfied - // TODO: Either combine these "before" and "after" routines, or - // split the other routines as well - void BeforeRoutine(cFunctionData const* function_data, - int reflevel, int map, int timelevel); - // After calling a routine: update according to writes clauses - void AfterRoutine(cFunctionData const* function_data, - int reflevel, int map, int timelevel); - // Synchronise and prolongate - // TODO: This does not handle variables that are not prolongated - // TODO: This does not handle buffer zones - void Sync(cFunctionData const* function_data, - vector const& groups, int reflevel, int timelevel); - // Restrict - void Restrict(vector const& groups, int reflevel); - - } // namespace Requirements -} // namespace Carpet - -#endif // #ifndef REQUIREMENTS_HH diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc index 4334b2a4e..628e414d5 100644 --- a/Carpet/Carpet/src/Restrict.cc +++ b/Carpet/Carpet/src/Restrict.cc @@ -5,14 +5,14 @@ #include #include +#include + #include #include #include #include -#include "Requirements.hh" - namespace Carpet { @@ -69,7 +69,9 @@ namespace Carpet { timer.stop(); } +#ifdef REQUIREMENTS_HH Requirements::Restrict(groups, reflevel); +#endif } diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc index 842f8b4f1..47f9dcad1 100644 --- a/Carpet/Carpet/src/SetupGH.cc +++ b/Carpet/Carpet/src/SetupGH.cc @@ -25,6 +25,8 @@ #endif #include +#include + #include #include #include @@ -35,7 +37,6 @@ #include #include -#include "Requirements.hh" #include "Timers.hh" @@ -720,7 +721,9 @@ namespace Carpet { maps = num_maps; } carpetGH.maps = maps; +#ifdef REQUIREMENTS_HH Requirements::Setup (maps); +#endif } diff --git a/Carpet/Carpet/src/Storage.cc b/Carpet/Carpet/src/Storage.cc index e95728858..d6ec2260a 100644 --- a/Carpet/Carpet/src/Storage.cc +++ b/Carpet/Carpet/src/Storage.cc @@ -6,6 +6,8 @@ #include #include +#include + #include #include #include @@ -14,8 +16,6 @@ #include -#include "Requirements.hh" - namespace Carpet { @@ -219,6 +219,7 @@ namespace Carpet { min_num_timelevels = 0; } +#ifdef REQUIREMENTS_HH vector vgroups(n_groups), vtimelevels(n_groups); for (int i=0; i