diff options
72 files changed, 4079 insertions, 4108 deletions
diff --git a/Carpet/Carpet/configuration.ccl b/Carpet/Carpet/configuration.ccl index 52de009c5..87604b0e0 100644 --- a/Carpet/Carpet/configuration.ccl +++ b/Carpet/Carpet/configuration.ccl @@ -1,4 +1,4 @@ # Configuration definitions for thorn Carpet -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/configuration.ccl,v 1.2 2003/09/19 16:02:56 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/configuration.ccl,v 1.3 2004/01/25 14:57:27 schnetter Exp $ -requires thorns: CarpetLib +requires thorns: IOUtil CarpetLib diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl index bb906378a..f87712f5c 100644 --- a/Carpet/Carpet/interface.ccl +++ b/Carpet/Carpet/interface.ccl @@ -1,7 +1,7 @@ # Interface definition for thorn Carpet -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/interface.ccl,v 1.6 2003/09/04 16:23:21 tradke Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/interface.ccl,v 1.7 2004/01/25 14:57:27 schnetter Exp $ -implements: driver +implements: Driver include header: carpet_public.hh in carpet.hh include header: carpet_public.h in carpet.h @@ -18,3 +18,71 @@ uses include header: gf.hh uses include header: ggf.hh uses include header: gh.hh uses include header: th.hh + +uses include header: operators.hh + + + +# The location of the boundary points +CCTK_INT FUNCTION GetBoundarySpecification \ + (CCTK_INT IN size, \ + CCTK_INT OUT ARRAY nboundaryzones, \ + CCTK_INT OUT ARRAY is_internal, \ + CCTK_INT OUT ARRAY is_staggered, \ + CCTK_INT OUT ARRAY shiftout) +USES FUNCTION GetBoundarySpecification + +# The overall size of the domain +CCTK_INT FUNCTION GetDomainSpecification \ + (CCTK_INT IN size, \ + CCTK_REAL OUT ARRAY physical_min, \ + CCTK_REAL OUT ARRAY physical_max, \ + CCTK_REAL OUT ARRAY interior_min, \ + CCTK_REAL OUT ARRAY interior_max, \ + CCTK_REAL OUT ARRAY exterior_min, \ + CCTK_REAL OUT ARRAY exterior_max, \ + CCTK_REAL OUT ARRAY spacing) +USES FUNCTION GetDomainSpecification + +# Conversion between boundary types +CCTK_INT FUNCTION ConvertFromPhysicalBoundary \ + (CCTK_INT INT size, \ + CCTK_REAL IN ARRAY physical_min, \ + CCTK_REAL IN ARRAY physical_max, \ + CCTK_REAL OUT ARRAY interior_min, \ + CCTK_REAL OUT ARRAY interior_max, \ + CCTK_REAL OUT ARRAY exterior_min, \ + CCTK_REAL OUT ARRAY exterior_max, \ + CCTK_REAL IN ARRAY spacing) +USES FUNCTION ConvertFromPhysicalBoundary + +CCTK_INT FUNCTION ConvertFromExteriorBoundary \ + (CCTK_INT INT size, \ + CCTK_REAL OUT ARRAY physical_min, \ + CCTK_REAL OUT ARRAY physical_max, \ + CCTK_REAL OUT ARRAY interior_min, \ + CCTK_REAL OUT ARRAY interior_max, \ + CCTK_REAL IN ARRAY exterior_min, \ + CCTK_REAL IN ARRAY exterior_max, \ + CCTK_REAL IN ARRAY spacing) +USES FUNCTION ConvertFromExteriorBoundary + + + +# The true prototype of the routine below: +# int Carpet_Regrid (const cGH * cctkGH, +# gh<dim>::rexts * bbsss, +# gh<dim>::rbnds * obss, +# gh<dim>::rprocs * pss); +CCTK_INT FUNCTION Carpet_Regrid (CCTK_POINTER_TO_CONST IN cctkGH, \ + CCTK_INT IN reflevel, \ + CCTK_INT IN map, \ + CCTK_INT IN size, \ + CCTK_INT IN ARRAY nboundaryzones, \ + CCTK_INT IN ARRAY is_internal, \ + CCTK_INT IN ARRAY is_staggered, \ + CCTK_INT IN ARRAY shiftout, \ + CCTK_POINTER IN bsss, \ + CCTK_POINTER IN obss, \ + CCTK_POINTER IN pss) +USES FUNCTION Carpet_Regrid diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl index 4c226ed5b..8897b5a31 100644 --- a/Carpet/Carpet/param.ccl +++ b/Carpet/Carpet/param.ccl @@ -1,5 +1,5 @@ # Parameter definitions for thorn Carpet -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/param.ccl,v 1.34 2003/11/05 16:18:37 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/param.ccl,v 1.35 2004/01/25 14:57:27 schnetter Exp $ shares: Cactus @@ -18,6 +18,12 @@ USES STRING out_dir +private: + +BOOLEAN domain_from_coordbase "Use the domain description from CoordBase" +{ +} "no" + restricted: CCTK_INT global_nx "Grid size in x direction" @@ -64,6 +70,22 @@ CCTK_INT ghost_size "Ghost zones in each spatial direction" -1 :: "use the per-dimension parameters" } -1 +BOOLEAN periodic "do not use this parameter" +{ +} "no" + +BOOLEAN periodic_x "do not use this parameter" +{ +} "no" + +BOOLEAN periodic_y "do not use this parameter" +{ +} "no" + +BOOLEAN periodic_z "do not use this parameter" +{ +} "no" + private: @@ -78,16 +100,32 @@ CCTK_INT refinement_factor "Refinement factor" 1:* :: "must be positive" } 2 -CCTK_INT multigrid_levels "Number of multigrid levels (including the base level)" + + +CCTK_INT convergence_level "Convergence level" +{ + *:* :: "negative for finer, positive for coarser resolutions" +} 0 + +CCTK_INT num_convergence_levels "Number of convergence levels (including the base level)" { 1:* :: "must be positive" } 1 -CCTK_INT multigrid_factor "Multigrid factor" +CCTK_INT convergence_factor "Multigrid factor" { 1:* :: "must be positive" } 2 + + +CCTK_INT num_maps "Number of maps" +{ + 1:* :: "" +} 1 + + + CCTK_INT prolongation_order_space "Order of prolongation operator in space" { 1 :: "first order (linear)" @@ -102,6 +140,8 @@ CCTK_INT prolongation_order_time "Order of prolongation operator in time" 2 :: "second order (quadratic)" } 1 + + CCTK_INT buffer_width "Width of the buffer zone inside the fine grid" { 0:* :: "Should be the radius of the numerical domain of dependence of the time integrator, minus the number of ghost zones" @@ -176,6 +216,12 @@ BOOLEAN checksum_timelevels "Try to catch unintentionally changed timelevels by +BOOLEAN suppress_restriction "Suppress the restriction operations. This makes the coarser refinement levels independent of the finer ones." +{ +} "no" + + + BOOLEAN verbose "Display info on the screen" { } "no" @@ -227,36 +273,18 @@ STRING grid_structure_filename "File name to output grid structure to (empty = n -# These parameters are here for historic reasons only. -# They will go away in the future. Do not use them. - -restricted: - -BOOLEAN periodic "do not use this parameter" -{ -} "no" -BOOLEAN periodic_x "do not use this parameter" -{ -} "yes" -BOOLEAN periodic_y "do not use this parameter" -{ -} "yes" -BOOLEAN periodic_z "do not use this parameter" -{ -} "yes" - - - private: BOOLEAN init_each_timelevel "Call initial data routines once for each timelevel" { } "no" -BOOLEAN init_3_timelevels "Set up 3 timelevels of initial data" +BOOLEAN prolongate_initial_data "Prolongate the refined regions during initial data generation" { } "no" -BOOLEAN prolongate_initial_data "Prolongate the refined regions during initial data generation" + + +BOOLEAN init_3_timelevels "Set up 3 timelevels of initial data" { } "no" diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc index f668be304..e06a8669d 100644 --- a/Carpet/Carpet/src/CallFunction.cc +++ b/Carpet/Carpet/src/CallFunction.cc @@ -11,7 +11,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/CallFunction.cc,v 1.13 2003/08/10 21:59:51 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/CallFunction.cc,v 1.14 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_CallFunction_cc); } @@ -27,51 +27,74 @@ namespace Carpet { cFunctionData* attribute, ///< attributes of the function void* data) ///< ??? { -// Checkpoint ("%*sStarting CallFunction...", 2*reflevel, ""); +// Checkpoint ("Starting CallFunction..."); cGH* cgh = (cGH*)data; - if (attribute->global || reflevel==-1) { +// TODO: disable temporarily +// if (attribute->meta || is_meta_mode()) { + if (is_meta_mode()) { + // Convtest operation + + if (do_meta_mode) { + assert (is_meta_mode()); + Checkpoint ("Meta mode call at %s to %s::%s", + attribute->where, attribute->thorn, attribute->routine); + const int res = CCTK_CallFunction (function, attribute, data); + assert (res==0); + } + + } else if (attribute->global || is_global_mode()) { // Global operation: call once if (do_global_mode) { - assert (component == -1); - const int saved_mglevel = mglevel; - if (mglevel!=-1) set_mglevel (cgh, -1); - const int saved_reflevel = reflevel; - if (reflevel!=-1) set_reflevel (cgh, -1); Checkpoint ("Global mode call at %s to %s::%s", attribute->where, attribute->thorn, attribute->routine); - const int res = CCTK_CallFunction (function, attribute, data); - assert (res==0); - if (reflevel!=saved_reflevel) set_reflevel (cgh, saved_reflevel); - if (mglevel!=saved_mglevel) set_mglevel (cgh, saved_mglevel); + BEGIN_GLOBAL_MODE(cgh) { + const int res = CCTK_CallFunction (function, attribute, data); + assert (res==0); + } END_GLOBAL_MODE; } } else if (attribute->level) { // Level operation: call once per refinement level - Checkpoint ("%*sLevel mode call at %s to %s::%s", 2*reflevel, "", + Checkpoint ("Level mode call at %s to %s::%s", attribute->where, attribute->thorn, attribute->routine); const int res = CCTK_CallFunction (function, attribute, data); assert (res==0); +// TODO: disable temporarily +// } else if (attribute->singlemap) { + } else if (false) { + // Single map operation: call once per refinement level and map + + BEGIN_MAP_LOOP(cgh, CCTK_GF) { + + Checkpoint ("Singlemap mode call at %s to %s::%s", + attribute->where, attribute->thorn, attribute->routine); + const int res = CCTK_CallFunction (function, attribute, data); + assert (res==0); + + } END_MAP_LOOP; + } else { // Local operation: call once per component - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - - Checkpoint ("%*sLocal mode call on component %d at %s to %s::%s", - 2*reflevel, "", component, - attribute->where, attribute->thorn, attribute->routine); - const int res = CCTK_CallFunction (function, attribute, data); - assert (res==0); - - } END_LOCAL_COMPONENT_LOOP; + BEGIN_MAP_LOOP(cgh, CCTK_GF) { + BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { + + Checkpoint ("Local mode call at %s to %s::%s", + attribute->where, attribute->thorn, attribute->routine); + const int res = CCTK_CallFunction (function, attribute, data); + assert (res==0); + + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; } -// Checkpoint ("%*sdone with CallFunction.", 2*reflevel, ""); +// Checkpoint ("done with CallFunction."); // The return value indicates whether the grid functions have been // synchronised. diff --git a/Carpet/Carpet/src/Checksum.cc b/Carpet/Carpet/src/Checksum.cc index 70dbc46fc..13981d7ae 100644 --- a/Carpet/Carpet/src/Checksum.cc +++ b/Carpet/Carpet/src/Checksum.cc @@ -9,7 +9,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Checksum.cc,v 1.12 2003/08/10 21:59:51 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Checksum.cc,v 1.13 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Checksum_cc); } @@ -27,59 +27,61 @@ namespace Carpet { if (! checksum_timelevels) return; - Checkpoint ("%*sCalculateChecksums", 2*reflevel, ""); + Checkpoint ("CalculateChecksums"); checksums.resize(maxreflevels); - checksums[reflevel].resize(hh->components(reflevel)); - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - checksums[reflevel][component].resize(CCTK_NumGroups()); - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_QueryGroupStorageI(cgh, group)) { - - const int nvar = CCTK_NumVarsInGroupI(group); - - checksums[reflevel][component][group].resize(nvar); - - if (reflevel<arrdata[group].hh->reflevels() - && component<arrdata[group].hh->components(reflevel) - && arrdata[group].hh->is_local(reflevel, component)) { - - const int n0 = CCTK_FirstVarIndexI(group); - const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0)); - assert (sz>0); - - vect<int,dim> size(1); - const int gpdim = arrdata[group].info.dim; - for (int d=0; d<gpdim; ++d) { - size[d] = arrdata[group].info.lsh[d]; - } - const int np = prod(size); - - const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); - assert (num_tl>0); - const int min_tl = mintl(where, num_tl); - const int max_tl = maxtl(where, num_tl); - - for (int var=0; var<nvar; ++var) { - checksums[reflevel][component][group][var].resize(num_tl); - for (int tl=min_tl; tl<=max_tl; ++tl) { + checksums[reflevel].resize(mglevels); + checksums[reflevel][mglevel].resize(CCTK_NumGroups()); + for (int group=0; group<CCTK_NumGroups(); ++group) { + if (CCTK_QueryGroupStorageI(cgh, group)) { + const int grouptype = CCTK_GroupTypeI(group); + if (reflevel == 0 || grouptype == CCTK_GF) { + checksums[reflevel][mglevel][group].resize(arrdata[group].size()); + BEGIN_MAP_LOOP(cgh, grouptype) { + checksums[reflevel][mglevel][group][map].resize(arrdata[group][map].hh->components(reflevel)); + BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { + const int nvars = CCTK_NumVarsInGroupI(group); + checksums[reflevel][mglevel][group][map][component].resize(nvars); + if (nvars > 0) { - const int n = n0 + var; - const void* data = cgh->data[n][tl]; - unsigned int chk = 0; - for (int i=0; i<np*sz/(int)sizeof chk; ++i) { - chk += ((const unsigned int*)data)[i]; + const int n0 = CCTK_FirstVarIndexI(group); + const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0)); + assert (sz>0); + + ivect size(1); + const int gpdim = groupdata[group].info.dim; + for (int d=0; d<gpdim; ++d) { + size[d] = groupdata[group].info.lsh[d]; } + const int np = prod(size); - checksums[reflevel][component][group][var][tl].sum = chk; - checksums[reflevel][component][group][var][tl].valid = true; + const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); + assert (num_tl>0); + const int min_tl = mintl(where, num_tl); + const int max_tl = maxtl(where, num_tl); - } // for tl - } // for var - } // if local - } // if storage - } // for group - } END_LOCAL_COMPONENT_LOOP; + for (int var=0; var<nvars; ++var) { + checksums[reflevel][mglevel][group][map][component][var].resize(num_tl); + for (int tl=min_tl; tl<=max_tl; ++tl) { + + const int n = n0 + var; + const void* data = cgh->data[n][tl]; + unsigned int chk = 0; + for (int i=0; i<np*sz/(int)sizeof chk; ++i) { + chk += ((const unsigned int*)data)[i]; + } + + checksums[reflevel][mglevel][group][map][component][var][tl].sum = chk; + checksums[reflevel][mglevel][group][map][component][var][tl].valid = true; + + } // for tl + } // for var + } // if group has vars + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; + } // if grouptype fits + } // if storage + } // for group } @@ -91,70 +93,71 @@ namespace Carpet { if (! checksum_timelevels) return; - Checkpoint ("%*sCheckChecksums", 2*reflevel, ""); + Checkpoint ("CheckChecksums"); assert ((int)checksums.size()==maxreflevels); - assert ((int)checksums[reflevel].size()==hh->components(reflevel)); - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - assert ((int)checksums[reflevel][component].size()==CCTK_NumGroups()); - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_QueryGroupStorageI(cgh, group)) { - - const int nvar = CCTK_NumVarsInGroupI(group); - - assert ((int)checksums[reflevel][component][group].size()==nvar); - - if (reflevel<arrdata[group].hh->reflevels() - && component<arrdata[group].hh->components(reflevel) - && arrdata[group].hh->is_local(reflevel, component)) { - - const int n0 = CCTK_FirstVarIndexI(group); - const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0)); - assert (sz>0); - - vect<int,dim> size(1); - const int gpdim = arrdata[group].info.dim; - for (int d=0; d<gpdim; ++d) { - size[d] = arrdata[group].info.lsh[d]; - } - const int np = prod(size); - - const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); - assert (num_tl>0); - const int min_tl = mintl(where, num_tl); - const int max_tl = maxtl(where, num_tl); - - for (int var=0; var<nvar; ++var) { - assert ((int)checksums[reflevel][component][group][var].size()==num_tl); - for (int tl=min_tl; tl<=max_tl; ++tl) { + assert ((int)checksums[reflevel].size()==mglevels); + assert ((int)checksums[reflevel][mglevel].size()==CCTK_NumGroups()); + for (int group=0; group<CCTK_NumGroups(); ++group) { + if (CCTK_QueryGroupStorageI(cgh, group)) { + const int grouptype = CCTK_GroupTypeI(group); + if (reflevel == 0 || grouptype == CCTK_GF) { + assert (checksums[reflevel][mglevel][group].size()==arrdata[group].size()); + BEGIN_MAP_LOOP(cgh, grouptype) { + assert ((int)checksums[reflevel][mglevel][group][map].size()==arrdata[group][map].hh->components(reflevel)); + BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { + const int nvars = CCTK_NumVarsInGroupI(group); + assert ((int)checksums[reflevel][mglevel][group][map][component].size()==nvars); + if (nvars > 0) { + + const int n0 = CCTK_FirstVarIndexI(group); + const int sz = CCTK_VarTypeSize(CCTK_VarTypeI(n0)); + assert (sz>0); + + ivect size(1); + const int gpdim = groupdata[group].info.dim; + for (int d=0; d<gpdim; ++d) { + size[d] = groupdata[group].info.lsh[d]; + } + const int np = prod(size); + + const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); + assert (num_tl>0); + const int min_tl = mintl(where, num_tl); + const int max_tl = maxtl(where, num_tl); - assert (checksums[reflevel][component][group][var][tl].valid); - if (checksums[reflevel][component][group][var][tl].valid) { - - const int n = n0 + var; - const void* data = cgh->data[n][tl]; - unsigned int chk = 0; - for (int i=0; i<np*sz/(int)sizeof chk; ++i) { - chk += ((const unsigned int*)data)[i]; - } - const bool unexpected_change = - chk != checksums[reflevel][component][group][var][tl].sum; - - if (unexpected_change) { - char* fullname = CCTK_FullName(n); - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Timelevel %d, component %d, refinement level %d of the variable \"%s\" has changed unexpectedly", - tl, component, reflevel, fullname); - free (fullname); - } - - } // if valid - } // for tl - } // for var - } // if local - } // if storage - } // for group - } END_LOCAL_COMPONENT_LOOP; + for (int var=0; var<nvars; ++var) { + assert ((int)checksums[reflevel][mglevel][group][map][component][var].size()==num_tl); + for (int tl=min_tl; tl<=max_tl; ++tl) { + if (checksums[reflevel][mglevel][group][map][component][var][tl].valid) { + + const int n = n0 + var; + const void* data = cgh->data[n][tl]; + unsigned int chk = 0; + for (int i=0; i<np*sz/(int)sizeof chk; ++i) { + chk += ((const unsigned int*)data)[i]; + } + const bool unexpected_change = + chk != checksums[reflevel][mglevel][group][map][component][var][tl].sum; + + if (unexpected_change) { + char* fullname = CCTK_FullName(n); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Timelevel %d, component %d, refinement level %d of the variable \"%s\" has changed unexpectedly", + tl, component, reflevel, fullname); + free (fullname); + } + + } // if valid + } // for tl + } // for var + } // if group has vars + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; + } // if grouptype fits + } // if storage + } // for group + } } // namespace Carpet diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc index e110bcbc1..dea882485 100644 --- a/Carpet/Carpet/src/Comm.cc +++ b/Carpet/Carpet/src/Comm.cc @@ -10,7 +10,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Comm.cc,v 1.23 2003/11/05 16:18:37 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Comm.cc,v 1.24 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Comm_cc); } @@ -28,26 +28,43 @@ namespace Carpet { const int group = CCTK_GroupIndex(groupname); assert (group>=0 && group<CCTK_NumGroups()); + assert (group<(int)arrdata.size()); - Checkpoint ("%*sSyncGroup %s time=%g", 2*reflevel, "", - groupname, cgh->cctk_time); + Checkpoint ("SyncGroup %s time=%g", groupname, (double)cgh->cctk_time); const int grouptype = CCTK_GroupTypeI(group); if (grouptype == CCTK_GF) { if (reflevel == -1) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot synchronise grid functions in global mode " + "Cannot synchronise in global mode " "(Tried to synchronise group \"%s\")", groupname); } - if (hh->local_components(reflevel) != 1 && component != -1) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot synchronise grid functions in local mode " - "(Tried to synchronise group \"%s\")", - groupname); + if (map != -1) { + if (maps == 1) { + CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, + "Synchronising group \"%s\" in singlemap mode", + groupname); + } else { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot synchronise in singlemap mode " + "(Tried to synchronise group \"%s\")", + groupname); + } + } + if (component != -1) { + if (maps == 1 && vhh.at(map)->local_components(reflevel) == 1) { + CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, + "Synchronising group \"%s\" in local mode", + groupname); + } else { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot synchronise in local mode " + "(Tried to synchronise group \"%s\")", + groupname); + } } - if (hh->local_components(reflevel) != 1) assert (component == -1); } if (! CCTK_QueryGroupStorageI(cgh, group)) { @@ -57,7 +74,7 @@ namespace Carpet { return -1; } - if (CCTK_NumVarsInGroupI(group)==0) return 0; + if (CCTK_NumVarsInGroupI(group) == 0) return 0; const int n0 = CCTK_FirstVarIndexI(group); assert (n0>=0); @@ -66,47 +83,70 @@ namespace Carpet { const int tl = 0; // Prolongate the boundaries - if (reflevel>0) { - if (grouptype == CCTK_GF) { - if (do_prolongate) { - if (arrdata[group].do_transfer) { + if (do_prolongate) { + switch (grouptype) { + + case CCTK_GF: + assert (reflevel>=0 && reflevel<reflevels); + if (reflevel > 0) { + if (groupdata.at(group).transport_operator != op_none) { + + // use the current time here (which may be modified by the + // user) + const CCTK_REAL time = ((cgh->cctk_time - cctk_initial_time) + / (delta_time * mglevelfact)); + for (comm_state<dim> state; !state.done(); state.step()) { - assert (group<(int)arrdata.size()); - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - // use the current time here (which may be modified by - // the user) - const CCTK_REAL time = (cgh->cctk_time - cctk_initial_time) / delta_time; -#if 0 - const CCTK_REAL time1 = tt->time (tl, reflevel, mglevel); - const CCTK_REAL time2 = (cgh->cctk_time - cctk_initial_time) / delta_time; - assert (fabs((time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time))) < 1e-12); -#endif - for (int c=0; c<arrdata[group].hh->components(reflevel); ++c) { - arrdata[group].data[var]->ref_bnd_prolongate - (state, tl, reflevel, c, mglevel, time); + for (int m=0; m<maps; ++m) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + for (int c=0; c<vhh.at(m)->components(reflevel); ++c) { + arrdata.at(group).at(m).data.at(var)->ref_bnd_prolongate + (state, tl, reflevel, c, mglevel, time); + } } - } // for var + } } // for state } else { - Checkpoint ("%*s(no prolongating for group %s)", - 2*reflevel, "", groupname); + Checkpoint ("(no prolongating for group %s)", groupname); } // if ! do_transfer - } // if do_prolongate - } // if grouptype == CCTK_GF - } // if reflevel>0 + } // if reflevel>0 + break; + + case CCTK_SCALAR: + case CCTK_ARRAY: + // do nothing + break; + + default: + assert (0); + } // switch grouptype + } // if do_prolongate // Sync for (comm_state<dim> state; !state.done(); state.step()) { - assert (group<(int)arrdata.size()); - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - if (grouptype == CCTK_GF) { - for (int c=0; c<arrdata[group].hh->components(reflevel); ++c) { - arrdata[group].data[var]->sync (state, tl, reflevel, c, mglevel); + switch (CCTK_GroupTypeI(group)) { + + case CCTK_GF: + for (int m=0; m<maps; ++m) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + for (int c=0; c<vhh.at(m)->components(reflevel); ++c) { + arrdata.at(group).at(m).data.at(var)->sync + (state, tl, reflevel, c, mglevel); + } } - } else { // grouptype != CCTK_GF - arrdata[group].data[var]->sync (state, 0, 0, 0, 0); - } // grouptype != CCTK_GF - } // for var + } + break; + + case CCTK_SCALAR: + case CCTK_ARRAY: + for (int var=0; var<(int)arrdata.at(group).at(0).data.size(); ++var) { + arrdata.at(group).at(0).data.at(var)->sync (state, 0, 0, 0, 0); + } + break; + + default: + assert (0); + } // switch grouptype } // for state return 0; diff --git a/Carpet/Carpet/src/Cycle.cc b/Carpet/Carpet/src/Cycle.cc index 932aab288..4d14d5a60 100644 --- a/Carpet/Carpet/src/Cycle.cc +++ b/Carpet/Carpet/src/Cycle.cc @@ -9,7 +9,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Cycle.cc,v 1.15 2003/11/05 16:18:37 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Cycle.cc,v 1.16 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Cycle_cc); } @@ -23,22 +23,44 @@ namespace Carpet { void CycleTimeLevels (const cGH* cgh) { - Checkpoint ("%*sCycleTimeLevels", 2*reflevel, ""); + Checkpoint ("CycleTimeLevels"); + assert (is_level_mode()); for (int group=0; group<CCTK_NumGroups(); ++group) { - if (reflevel<arrdata[group].hh->reflevels() - && CCTK_QueryGroupStorageI(cgh, group)) { - for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { - - assert (group<(int)arrdata.size()); - assert (var<(int)arrdata[group].data.size()); - for (int c=0; c<arrdata[group].hh->components(reflevel); ++c) { - arrdata[group].data[var]->cycle (reflevel, c, mglevel); - } - - } - } - } + assert (group<(int)arrdata.size()); + if (CCTK_QueryGroupStorageI(cgh, group)) { + switch (CCTK_GroupTypeI(group)) { + + case CCTK_GF: + assert (reflevel>=0 && reflevel<reflevels); + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + for (int m=0; m<maps; ++m) { + assert (m<(int)arrdata[group].size()); + assert (var<(int)arrdata[group][m].data.size()); + for (int c=0; c<arrdata[group][m].hh->components(reflevel); ++c) { + arrdata[group][m].data[var]->cycle (reflevel, c, mglevel); + } + } + } + break; + + case CCTK_SCALAR: + case CCTK_ARRAY: + if (do_global_mode) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata[group][0].data.size()); + for (int c=0; c<arrdata[group][0].hh->components(0); ++c) { + arrdata[group][0].data[var]->cycle (0, c, mglevel); + } + } + } + break; + + default: + assert (0); + } // switch grouptype + } // if storage + } // for group } @@ -46,38 +68,60 @@ namespace Carpet { void FlipTimeLevels (const cGH* cgh) { Checkpoint ("FlipTimeLevels"); + assert (is_level_mode()); for (int group=0; group<CCTK_NumGroups(); ++group) { + assert (group<(int)arrdata.size()); if (CCTK_QueryGroupStorageI(cgh, group)) { - - const int var0 = CCTK_FirstVarIndexI(group); - assert (var0>=0); - const int num_tl = CCTK_NumTimeLevelsFromVarI(var0); - switch (num_tl) { - case 1: - // Do nothing - break; - case 3: - // Flip - for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { - - assert (group<(int)arrdata.size()); - assert (var<(int)arrdata[group].data.size()); - for (int rl=0; rl<arrdata[group].hh->reflevels(); ++rl) { - for (int c=0; c<arrdata[group].hh->components(rl); ++c) { - arrdata[group].data[var]->flip (rl, c, mglevel); - } - } - - } - break; - default: - // Error - assert (0); - } // switch - - } - } + const int num_vars = CCTK_NumVarsInGroupI(group); + if (num_vars>0) { + const int var0 = CCTK_FirstVarIndexI(group); + assert (var0>=0); + const int num_tl = CCTK_NumTimeLevelsFromVarI(var0); + switch (num_tl) { + case 1: + // Do nothing + break; + case 3: + // Flip + switch (CCTK_GroupTypeI(group)) { + + case CCTK_GF: + for (int m=0; m<(int)arrdata[group].size(); ++m) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata[group][m].data.size()); + for (int c=0; c<arrdata[group][m].hh->components(reflevel); ++c) { + arrdata[group][m].data[var]->flip (reflevel, c, mglevel); + } + } + } + break; + + case CCTK_SCALAR: + case CCTK_ARRAY: + if (do_global_mode) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata[group][0].data.size()); + for (int c=0; c<arrdata[group][0].hh->components(0); ++c) { + arrdata[group][0].data[var]->flip (0, c, mglevel); + } + } + } + break; + + default: + assert (0); + } // switch grouptype + break; + + default: + // Error + assert (0); + } // switch num_tl + + } // if num_vars>0 + } // if storage + } // for group } } // namespace Carpet diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc index fd8fb3019..b9a9d4bcf 100644 --- a/Carpet/Carpet/src/Evolve.cc +++ b/Carpet/Carpet/src/Evolve.cc @@ -31,7 +31,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Evolve.cc,v 1.30 2004/01/13 13:51:19 hawke Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Evolve.cc,v 1.31 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Evolve_cc); } @@ -43,10 +43,6 @@ namespace Carpet { - static double initial_time; - - - static bool do_terminate (const cGH *cgh, const CCTK_REAL time, const int iteration) { @@ -66,17 +62,25 @@ namespace Carpet { ? time >= cctk_final_time : time <= cctk_final_time) && iteration % maxreflevelfact == 0); - double runtime; #ifdef HAVE_TIME_GETTIMEOFDAY // get the current time struct timeval tv; gettimeofday (&tv, 0); - runtime = (tv.tv_sec + tv.tv_usec / 1e6) - initial_time; -#else - runtime = 0; -#endif + const double thetime = tv.tv_sec + tv.tv_usec / 1e6; + + static bool firsttime = true; + static double initial_runtime; + if (firsttime) { + firsttime = false; + initial_runtime = thetime; + } + + const double runtime = thetime - initial_runtime; const bool term_runtime = (max_runtime > 0 && runtime >= 60.0 * max_runtime); +#else + const bool term_runtime = false; +#endif if (CCTK_Equals(terminate, "never")) { term = false; @@ -116,159 +120,116 @@ namespace Carpet { { DECLARE_CCTK_PARAMETERS; - Waypoint ("starting Evolve..."); + Waypoint ("Starting evolution loop"); const int convlev = 0; cGH* cgh = fc->GH[convlev]; -#ifdef HAVE_TIME_GETTIMEOFDAY - // get the starting time - struct timeval tv; - gettimeofday (&tv, 0); - initial_time = tv.tv_sec + tv.tv_usec / 1e6; -#else - initial_time = 0; -#endif - - int next_global_mode_iter_loop1 = 0; - int next_global_mode_iter_loop2 = 0; - int next_global_mode_iter_loop3 = 0; - // Main loop - while (! do_terminate(cgh, refleveltimes[0], cgh->cctk_iteration)) { + while (! do_terminate(cgh, cgh->cctk_time, cgh->cctk_iteration)) { // Advance time ++cgh->cctk_iteration; + global_time += delta_time / maxreflevelfact; + cgh->cctk_time = global_time; + Waypoint ("Evolving iteration %d at t=%g", + cgh->cctk_iteration, (double)cgh->cctk_time); - Waypoint ("Evolving iteration %d...", cgh->cctk_iteration); - - BEGIN_REFLEVEL_LOOP(cgh) { - const int do_every = maxreflevelfact/reflevelfact; - if ((cgh->cctk_iteration-1) % do_every == 0) { - - BEGIN_MGLEVEL_LOOP(cgh) { - const int do_every = mglevelfact * (maxreflevelfact/reflevelfact); - if ((cgh->cctk_iteration-1) % do_every == 0) { - - do_global_mode - = cgh->cctk_iteration >= next_global_mode_iter_loop1; - next_global_mode_iter_loop1 = cgh->cctk_iteration + 1; - - // Advance level times - tt->advance_time (reflevel, mglevel); - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - - // Cycle time levels - CycleTimeLevels (cgh); - - // Checking - CalculateChecksums (cgh, allbutcurrenttime); - Poison (cgh, currenttimebutnotifonly); - - // Evolve - Waypoint ("%*sScheduling PRESTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); - Waypoint ("%*sScheduling EVOL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTSTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); - - // Checking - PoisonCheck (cgh, currenttime); - - } - } END_MGLEVEL_LOOP; - - } - } END_REFLEVEL_LOOP; - - - - BEGIN_REVERSE_REFLEVEL_LOOP(cgh) { - const int do_every = maxreflevelfact/reflevelfact; - if (cgh->cctk_iteration % do_every == 0) { - - BEGIN_MGLEVEL_LOOP(cgh) { - const int do_every = mglevelfact * (maxreflevelfact/reflevelfact); - if (cgh->cctk_iteration % do_every == 0) { - - do_global_mode - = cgh->cctk_iteration >= next_global_mode_iter_loop2; - next_global_mode_iter_loop2 = cgh->cctk_iteration + 1; - - // Restrict - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - Restrict (cgh); - - Waypoint ("%*sScheduling POSTRESTRICT", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction); - - // Checking - CalculateChecksums (cgh, currenttime); - - } - } END_MGLEVEL_LOOP; - - } - } END_REVERSE_REFLEVEL_LOOP; + for (int rl=0; rl<reflevels; ++rl) { + + // Regrid + Checkpoint ("Regrid"); + Regrid (cgh, rl, rl+1, true); + + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + const int do_every = mglevelfact * (maxreflevelfact/reflevelfact); + if ((cgh->cctk_iteration-1) % do_every == 0) { + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + // Advance times + for (int m=0; m<maps; ++m) { + vtt[m]->advance_time (reflevel, mglevel); + } + cgh->cctk_time = (global_time + - delta_time * mglevelfact / maxreflevelfact + + delta_time * mglevelfact / reflevelfact); + CycleTimeLevels (cgh); + + Checkpoint ("Evolution I at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Checking + CalculateChecksums (cgh, allbutcurrenttime); + Poison (cgh, currenttimebutnotifonly); + + // Evolve + Checkpoint ("Scheduling PRESTEP"); + CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); + Checkpoint ("Scheduling EVOL"); + CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); + + // Checking + PoisonCheck (cgh, currenttime); + + } // if do_every + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl - BEGIN_REFLEVEL_LOOP(cgh) { - const int do_every = maxreflevelfact/reflevelfact; - if (cgh->cctk_iteration % do_every == 0) { - - // Regrid - Waypoint ("%*sRegrid", 2*reflevel, ""); - Regrid (cgh, reflevel+1, true); - - BEGIN_MGLEVEL_LOOP(cgh) { - const int do_every = mglevelfact * (maxreflevelfact/reflevelfact); - if (cgh->cctk_iteration % do_every == 0) { - - do_global_mode - = cgh->cctk_iteration >= next_global_mode_iter_loop3; - next_global_mode_iter_loop3 = cgh->cctk_iteration + 1; - - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - - // Checkpoint - Waypoint ("%*sScheduling CHECKPOINT", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_CHECKPOINT", cgh, CallFunction); - - // Analysis - Waypoint ("%*sScheduling ANALYSIS", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction); - - // Output - Waypoint ("%*sOutputGH", 2*reflevel, ""); - CCTK_OutputGH (cgh); - - // Checking - CheckChecksums (cgh, alltimes); - - } - } END_MGLEVEL_LOOP; - - } - } END_REFLEVEL_LOOP; + for (int rl=reflevels-1; rl>=0; --rl) { + BEGIN_REVERSE_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + const int do_every = mglevelfact * (maxreflevelfact/reflevelfact); + if (cgh->cctk_iteration % do_every == 0) { + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + Checkpoint ("Evolution II at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Restrict + Restrict (cgh); + + Checkpoint ("Scheduling POSTRESTRICT"); + CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction); + Checkpoint ("Scheduling POSTSTEP"); + CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); + + // Checking + PoisonCheck (cgh, currenttime); + CalculateChecksums (cgh, currenttime); + + // Checkpoint + Checkpoint ("Scheduling CHECKPOINT"); + CCTK_ScheduleTraverse ("CCTK_CHECKPOINT", cgh, CallFunction); + + // Analysis + Checkpoint ("Scheduling ANALYSIS"); + CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction); + + // Output + Checkpoint ("OutputGH"); + CCTK_OutputGH (cgh); + + // Checking + CheckChecksums (cgh, alltimes); + + } // if do_every + leave_level_mode (cgh); + } END_REVERSE_MGLEVEL_LOOP; + } // for rl } // main loop - Waypoint ("done with Evolve."); + Waypoint ("Done with evolution loop"); return 0; } diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc index 231bbf5d1..1c752860b 100644 --- a/Carpet/Carpet/src/Initialise.cc +++ b/Carpet/Carpet/src/Initialise.cc @@ -12,7 +12,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Initialise.cc,v 1.34 2003/11/05 16:18:37 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Initialise.cc,v 1.35 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Initialise_cc); } @@ -34,386 +34,372 @@ namespace Carpet { CCTKi_AddGH (fc, convlev, cgh); // Delay checkpoint until MPI has been initialised - Waypoint ("starting Initialise..."); + Waypoint ("Starting initialisation"); // Initialise stuff cgh->cctk_iteration = 0; + global_time = cctk_initial_time; + delta_time = 1.0; + cgh->cctk_time = global_time; + cgh->cctk_delta_time = delta_time; + do_global_mode = true; + do_meta_mode = true; // Enable storage and communtication CCTKi_ScheduleGHInit (cgh); // Initialise stuff CCTKi_InitGHExtensions (cgh); - - // Register coordinates - Waypoint ("Scheduling CCTK_WRAGH"); - CCTK_ScheduleTraverse ("CCTK_WRAGH", cgh, CallFunction); - - // Check parameters - Waypoint ("Current time is %g", cgh->cctk_time); - Waypoint ("Scheduling PARAMCHECK"); - CCTK_ScheduleTraverse ("CCTK_PARAMCHECK", cgh, CallFunction); - CCTKi_FinaliseParamWarn(); - - Waypoint ("Initialising iteration %d...", cgh->cctk_iteration); - + BEGIN_MGLEVEL_LOOP(cgh) { + do_global_mode = true; + do_meta_mode = mglevel==mglevels-1; + + // Register coordinates + Checkpoint ("Scheduling CCTK_WRAGH"); + CCTK_ScheduleTraverse ("CCTK_WRAGH", cgh, CallFunction); + + // Check parameters + Checkpoint ("Scheduling PARAMCHECK"); + CCTK_ScheduleTraverse ("CCTK_PARAMCHECK", cgh, CallFunction); + CCTKi_FinaliseParamWarn(); + } END_MGLEVEL_LOOP; - BEGIN_REFLEVEL_LOOP(cgh) { + if (fc->recovered) { + // if recovering - BEGIN_MGLEVEL_LOOP(cgh) { - - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - do_global_mode = reflevel == 0; - - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - - // Checking - Poison (cgh, alltimes); - - // Set up the grid - Waypoint ("%*sScheduling BASEGRID", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction); - - if (! init_each_timelevel) { + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; - // Set up the initial data - Waypoint ("%*sScheduling INITIAL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTINITIAL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction); + cgh->cctk_time = global_time; - } else { - // if init_each_timelevel + // Set up the grids + Checkpoint ("Scheduling BASEGRID"); + CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction); - bool const saved_do_global_mode = do_global_mode; + // Recover + Checkpoint ("Scheduling RECOVER_VARIABLES"); + CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction); + Checkpoint ("Scheduling POST_RECOVER_VARIABLES"); + CCTK_ScheduleTraverse + ("CCTK_POST_RECOVER_VARIABLES", cgh, CallFunction); - tt->set_delta - (reflevel, mglevel, - tt->get_delta (reflevel, mglevel)); - tt->advance_time (reflevel, mglevel); - tt->advance_time (reflevel, mglevel); - tt->advance_time (reflevel, mglevel); - tt->set_delta - (reflevel, mglevel, - tt->get_delta (reflevel, mglevel)); + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl + + } else { + // if not reconvering + + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; - for (int tl=-2; tl<=0; ++tl) { - - do_global_mode = saved_do_global_mode && tl==0; - - // Advance level times - tt->advance_time (reflevel, mglevel); - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); + cgh->cctk_time = global_time; + + Waypoint ("Initialisation at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Checking + Poison (cgh, alltimes); + + // Set up the grids + Checkpoint ("Scheduling BASEGRID"); + CCTK_ScheduleTraverse ("CCTK_BASEGRID", cgh, CallFunction); + + const int num_tl = init_each_timelevel ? 3 : 1; + + // Rewind + for (int m=0; m<maps; ++m) { + vtt[m]->set_delta + (reflevel, mglevel, - vtt[m]->get_delta (reflevel, mglevel)); + FlipTimeLevels (cgh); + for (int tl=0; tl<num_tl; ++tl) { + vtt[m]->advance_time (reflevel, mglevel); + CycleTimeLevels (cgh); + } + vtt[m]->set_delta + (reflevel, mglevel, - vtt[m]->get_delta (reflevel, mglevel)); + FlipTimeLevels (cgh); + } + + const bool outer_do_global_mode = do_global_mode; + for (int tl=num_tl-1; tl>=0; --tl) { + do_global_mode = outer_do_global_mode && tl==0; - // Cycle time levels + // Advance times + for (int m=0; m<maps; ++m) { + vtt[m]->advance_time (reflevel, mglevel); + } + cgh->cctk_time + = global_time - tl * delta_time * mglevelfact / reflevelfact; CycleTimeLevels (cgh); - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - // Set up the initial data - Waypoint ("%*sScheduling INITIAL", 2*reflevel, ""); + Checkpoint ("Scheduling INITIAL"); CCTK_ScheduleTraverse ("CCTK_INITIAL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTINITIAL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction); } // for tl - - do_global_mode = saved_do_global_mode; - - } // if init_each_timelevel - - // Poststep - Waypoint ("%*sScheduling POSTSTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); - - if (! init_each_timelevel) { + do_global_mode = outer_do_global_mode; // Checking PoisonCheck (cgh, currenttime); - } else { - // if init_each_timelevel - - // Checking - PoisonCheck (cgh, alltimes); - - } // if init_each_timelevel + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; - } END_MGLEVEL_LOOP; - - // Regrid - Waypoint ("%*sRegrid", 2*reflevel, ""); - Regrid (cgh, reflevel+1, prolongate_initial_data); - - BEGIN_MGLEVEL_LOOP(cgh) { + // Regrid + Checkpoint ("Regrid"); + Regrid (cgh, rl, rl+1, prolongate_initial_data); - if (init_3_timelevels) { - // Use Scott Hawley's algorithm for getting two extra - // timelevels of data (this is part 1) - - cout << "Initialising three timelevels" << endl; - - // Advance level times - tt->advance_time (reflevel, mglevel); - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; - - // Cycle time levels (ignore arrays) - cout << "3TL rl=" << reflevel << " cycling" << endl; - CycleTimeLevels (cgh); - - // Checking - CalculateChecksums (cgh, allbutcurrenttime); - PoisonCheck (cgh, previoustime); - Poison (cgh, currenttimebutnotifonly); - - // Evolve forward - Waypoint ("%*sScheduling PRESTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); - Waypoint ("%*sScheduling EVOL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTSTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); - - // Checking - PoisonCheck (cgh, currenttime); + } // for rl + + for (int rl=reflevels-1; rl>=0; --rl) { + BEGIN_REVERSE_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); - // Flip time levels - cout << "3TL rl=" << reflevel << " flipping" << endl; - FlipTimeLevels (cgh); + // Restrict + Restrict (cgh); - // Invert level times - for (int rl=0; rl<hh->reflevels(); ++rl) { - tt->set_delta (rl, mglevel, - tt->get_delta (rl, mglevel)); - tt->advance_time (rl, mglevel); - tt->advance_time (rl, mglevel); - } - cgh->cctk_delta_time *= -1; - delta_time *= -1; - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; + Checkpoint ("Scheduling POSTRESTRICTINITIAL"); + CCTK_ScheduleTraverse + ("CCTK_POSTRESTRICTINITIAL", cgh, CallFunction); - // Checking - CalculateChecksums (cgh, allbutcurrenttime); - PoisonCheck (cgh, allbutcurrenttime); - Poison (cgh, currenttimebutnotifonly); - - // Evolve backward - Waypoint ("%*sScheduling PRESTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); - Waypoint ("%*sScheduling EVOL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTSTEP", 2*reflevel, ""); + // Poststep + Checkpoint ("Scheduling POSTINITIAL"); + CCTK_ScheduleTraverse ("CCTK_POSTINITIAL", cgh, CallFunction); + Checkpoint ("Scheduling POSTSTEP"); CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); // Checking PoisonCheck (cgh, alltimes); + CalculateChecksums (cgh, allbutcurrenttime); - // Flip time levels back - cout << "3TL rl=" << reflevel << " flipping back" << endl; - FlipTimeLevels (cgh); - - // Invert level times back - for (int rl=0; rl<hh->reflevels(); ++rl) { - tt->set_delta (rl, mglevel, - tt->get_delta (rl, mglevel)); - tt->advance_time (rl, mglevel); - tt->advance_time (rl, mglevel); - } - cgh->cctk_delta_time *= -1; - delta_time *= -1; - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; - - } // if init_3_timelevels - - } END_MGLEVEL_LOOP; + leave_level_mode (cgh); + } END_REVERSE_MGLEVEL_LOOP; + } // for rl - } END_REFLEVEL_LOOP; - - if (init_3_timelevels) { - // Use Scott Hawley's algorithm for getting two extra timelevels - // of data (here comes part 2) + if (init_3_timelevels) { + // Use Scott Hawley's algorithm for getting two extra + // timelevels of data + Waypoint ("Initialising three timelevels"); - cout << "Hourglass structure in place" << endl; - - // Evolve each level "backwards" one more timestep - // Starting with the finest level and proceeding to the coarsest - BEGIN_REVERSE_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - - do_global_mode = reflevel == 0; - - // Flip time levels - cout << "3TL rl=" << reflevel << " flipping" << endl; - FlipTimeLevels (cgh); - - // Invert level times - for (int rl=0; rl<hh->reflevels(); ++rl) { - tt->set_delta (rl, mglevel, - tt->get_delta (rl, mglevel)); - tt->advance_time (rl, mglevel); - tt->advance_time (rl, mglevel); - } - cgh->cctk_delta_time *= -1; - delta_time *= -1; - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; - - // Checking - CalculateChecksums (cgh, currenttime); - PoisonCheck (cgh, alltimes); - - // Restrict - cout << "3TL rl=" << reflevel << " restricting" << endl; - Restrict (cgh); - - Waypoint ("%*sScheduling POSTRESTRICT", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction); - - // Checking - CalculateChecksums (cgh, currenttime); - PoisonCheck (cgh, alltimes); - - // Advance level times - tt->advance_time (reflevel, mglevel); - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + // Advance times + for (int m=0; m<maps; ++m) { + vtt[m]->advance_time (reflevel, mglevel); + } + cgh->cctk_time + = global_time + delta_time * mglevelfact / reflevelfact; + CycleTimeLevels (cgh); - // Cycle time levels - cout << "3TL rl=" << reflevel << " cycling" << endl; - CycleTimeLevels (cgh); + Waypoint ("Initialisation 3TL evolution I (a) (forwards) at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); - // Checking - CalculateChecksums (cgh, allbutcurrenttime); - Poison (cgh, currenttimebutnotifonly); - - // Evolve backward - Waypoint ("%*sScheduling PRESTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); - Waypoint ("%*sScheduling EVOL", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); - Waypoint ("%*sScheduling POSTSTEP", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); + // Checking + CalculateChecksums (cgh, allbutcurrenttime); + Poison (cgh, currenttimebutnotifonly); - // Checking - PoisonCheck (cgh, alltimes); + // Evolve forward + Checkpoint ("Scheduling PRESTEP"); + CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); + Checkpoint ("Scheduling EVOL"); + CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); - // Flip time levels back - cout << "3TL rl=" << reflevel << " flipping back" << endl; - FlipTimeLevels (cgh); + // Checking + PoisonCheck (cgh, currenttime); - // Invert level times back - for (int rl=0; rl<hh->reflevels(); ++rl) { - tt->set_delta (rl, mglevel, - tt->get_delta (rl, mglevel)); - tt->advance_time (rl, mglevel); - tt->advance_time (rl, mglevel); - } - cgh->cctk_delta_time *= -1; - delta_time *= -1; - cgh->cctk_time = (cctk_initial_time - + (tt->time (0, reflevel, mglevel) - * cgh->cctk_delta_time)); - cout << "3TL rl=" << reflevel << " ml=" << mglevel - << " time=" << tt->get_time (reflevel, mglevel) - << " time=" << cgh->cctk_time / cgh->cctk_delta_time << endl; + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl + + delta_time *= -1; + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + // Flip time levels + Waypoint ("Flipping timelevels"); + FlipTimeLevels (cgh); + + cgh->cctk_time + = global_time + delta_time * mglevelfact / reflevelfact; + + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl + + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + Waypoint ("Initialisation 3TL evolution I (b) (backwards) at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Checking + CalculateChecksums (cgh, allbutcurrenttime); + Poison (cgh, currenttimebutnotifonly); + + // Evolve backward + Checkpoint ("Scheduling PRESTEP"); + CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); + Checkpoint ("Scheduling EVOL"); + CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); + + // Checking + PoisonCheck (cgh, alltimes); + + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl + + Waypoint ("Hourglass structure in place"); + + // Evolve each level "backwards" one more timestep + // Starting with the finest level and proceeding to the coarsest + for (int rl=reflevels-1; rl>=0; --rl) { + BEGIN_REVERSE_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + Waypoint ("Initialisation 3TL evolution II (b) (backwards) at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Restrict + Restrict (cgh); + + Checkpoint ("Scheduling POSTRESTRICT"); + CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction); + Checkpoint ("Scheduling POSTSTEP"); + CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); + + // Checking + PoisonCheck (cgh, alltimes); + + // Advance times + for (int m=0; m<maps; ++m) { + vtt[m]->advance_time (reflevel, mglevel); + } + cgh->cctk_time + = global_time + 2 * delta_time * mglevelfact / reflevelfact; + CycleTimeLevels (cgh); - } END_MGLEVEL_LOOP; - } END_REVERSE_REFLEVEL_LOOP; + Waypoint ("Initialisation 3TL evolution I (c) (backwards) at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); + + // Checking + CalculateChecksums (cgh, allbutcurrenttime); + Poison (cgh, currenttimebutnotifonly); + + // Evolve backward + Checkpoint ("Scheduling PRESTEP"); + CCTK_ScheduleTraverse ("CCTK_PRESTEP", cgh, CallFunction); + Checkpoint ("Scheduling EVOL"); + CCTK_ScheduleTraverse ("CCTK_EVOL", cgh, CallFunction); + Checkpoint ("Scheduling POSTSTEP"); + CCTK_ScheduleTraverse ("CCTK_POSTSTEP", cgh, CallFunction); + + // Checking + PoisonCheck (cgh, alltimes); + + leave_level_mode (cgh); + } END_REVERSE_MGLEVEL_LOOP; + } // for rl - cout << "Finished initialising three timelevels" << endl; + delta_time *= -1; + for (int rl=0; rl<reflevels; ++rl) { + BEGIN_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; + + // Flip time levels back + Waypoint ("Flipping timelevels back"); + FlipTimeLevels (cgh); + + // Invert level times back + for (int m=0; m<maps; ++m) { + vtt[m]->set_delta + (reflevel, mglevel, - vtt[m]->get_delta (reflevel, mglevel)); + vtt[m]->advance_time (reflevel, mglevel); + vtt[m]->advance_time (reflevel, mglevel); + vtt[m]->set_delta + (reflevel, mglevel, - vtt[m]->get_delta (reflevel, mglevel)); + } + cgh->cctk_time = global_time; + + leave_level_mode (cgh); + } END_MGLEVEL_LOOP; + } // for rl - } // if init_3_timelevels - - - - BEGIN_REVERSE_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - - do_global_mode = reflevel == 0; - - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); - - // Restrict - Restrict (cgh); - - Waypoint ("%*sScheduling POSTRESTRICT", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_POSTRESTRICT", cgh, CallFunction); - - } END_MGLEVEL_LOOP; - } END_REVERSE_REFLEVEL_LOOP; - + Waypoint ("Finished initialising three timelevels"); + + } // if init_3_timelevels + } // if not recovering + + - BEGIN_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - - do_global_mode = reflevel == 0; + for (int rl=reflevels-1; rl>=0; --rl) { + BEGIN_REVERSE_MGLEVEL_LOOP(cgh) { + enter_level_mode (cgh, rl); + do_global_mode = reflevel==0; + do_meta_mode = do_global_mode && mglevel==mglevels-1; - Waypoint ("%*sCurrent time is %g, delta is %g%s", 2*reflevel, "", - cgh->cctk_time, - cgh->cctk_delta_time / cgh->cctk_timefac, - do_global_mode ? " (global time)" : ""); + Waypoint ("Initialisation II at iteration %d time %g%s%s", + cgh->cctk_iteration, (double)cgh->cctk_time, + (do_global_mode ? " (global)" : ""), + (do_meta_mode ? " (meta)" : "")); - // Checking - PoisonCheck (cgh, alltimes); - CalculateChecksums (cgh, allbutcurrenttime); - - // Recover - Waypoint ("%*sScheduling RECOVER_VARIABLES", 2*reflevel, ""); - CCTK_ScheduleTraverse ("CCTK_RECOVER_VARIABLES", cgh, CallFunction); - Waypoint ("%*sScheduling CPINITIAL", 2*reflevel, ""); + // Checkpoint + Checkpoint ("Scheduling CPINITIAL"); CCTK_ScheduleTraverse ("CCTK_CPINITIAL", cgh, CallFunction); // Analysis - Waypoint ("%*sScheduling ANALYSIS", 2*reflevel, ""); + Checkpoint ("Scheduling ANALYSIS"); CCTK_ScheduleTraverse ("CCTK_ANALYSIS", cgh, CallFunction); // Output - Waypoint ("%*sOutputGH", 2*reflevel, ""); + Checkpoint ("OutputGH"); CCTK_OutputGH (cgh); - + // Checking PoisonCheck (cgh, alltimes); CheckChecksums (cgh, allbutcurrenttime); - - } END_MGLEVEL_LOOP; - } END_REFLEVEL_LOOP; + + leave_level_mode (cgh); + } END_REVERSE_MGLEVEL_LOOP; + } // for rl - Waypoint ("done with Initialise."); + Waypoint ("Done with initialisation"); return 0; } diff --git a/Carpet/Carpet/src/Poison.cc b/Carpet/Carpet/src/Poison.cc index a151db081..3fe7cbd4f 100644 --- a/Carpet/Carpet/src/Poison.cc +++ b/Carpet/Carpet/src/Poison.cc @@ -8,7 +8,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Poison.cc,v 1.14 2003/08/10 21:59:51 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Poison.cc,v 1.15 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Poison_cc); } @@ -44,9 +44,11 @@ namespace Carpet { if (! poison_new_timelevels) return; if (! CCTK_QueryGroupStorageI(cgh, group)) { + char * const groupname = CCTK_GroupName(group); CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, "Cannot poison group \"%s\" because it has no storage", - CCTK_GroupName(group)); + groupname); + free (groupname); return; } @@ -63,27 +65,33 @@ namespace Carpet { if (min_tl <= max_tl) { - Checkpoint ("%*sPoisonGroup %s", 2*reflevel, "", CCTK_GroupName(group)); + { + char * const groupname = CCTK_GroupName(group); + Checkpoint ("PoisonGroup %s", groupname); + free (groupname); + } const int grouptype = CCTK_GroupTypeI(group); - BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { - - vect<int,dim> size(1); - const int gpdim = arrdata[group].info.dim; - for (int d=0; d<gpdim; ++d) { - size[d] = arrdata[group].info.lsh[d]; - } - const int np = prod(size); - - for (int var=0; var<nvar; ++var) { - const int n = n0 + var; - for (int tl=min_tl; tl<=max_tl; ++tl) { - memset (cgh->data[n][tl], poison_value, np*sz); - } // for tl - } // for var - - } END_LOCAL_COMPONENT_LOOP; + BEGIN_MAP_LOOP(cgh, grouptype) { + BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { + + ivect size(1); + const int gpdim = groupdata[group].info.dim; + for (int d=0; d<gpdim; ++d) { + size[d] = groupdata[group].info.lsh[d]; + } + const int np = prod(size); + + for (int var=0; var<nvar; ++var) { + const int n = n0 + var; + for (int tl=min_tl; tl<=max_tl; ++tl) { + memset (cgh->data[n][tl], poison_value, np*sz); + } // for tl + } // for var + + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; } // if tl } @@ -96,7 +104,7 @@ namespace Carpet { if (! check_for_poison) return; - Checkpoint ("%*sPoisonCheck", 2*reflevel, ""); + Checkpoint ("PoisonCheck"); for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_QueryGroupStorageI(cgh, group)) { @@ -106,76 +114,78 @@ namespace Carpet { assert (n0>=0); const int nvar = CCTK_NumVarsInGroupI(group); const int tp = CCTK_VarTypeI(n0); - const int gpdim = arrdata[group].info.dim; + const int gpdim = groupdata[group].info.dim; const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); assert (num_tl>0); const int min_tl = mintl(where, num_tl); const int max_tl = maxtl(where, num_tl); - BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { - - vect<int,dim> size(1); - for (int d=0; d<gpdim; ++d) { - size[d] = arrdata[group].info.lsh[d]; - } - const int np = prod(size); - - for (int var=0; var<nvar; ++var) { - const int n = n0 + var; + BEGIN_MAP_LOOP(cgh, grouptype) { + BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) { - for (int tl=min_tl; tl<=max_tl; ++tl) { + ivect size(1); + for (int d=0; d<gpdim; ++d) { + size[d] = groupdata[group].info.lsh[d]; + } + const int np = prod(size); + + for (int var=0; var<nvar; ++var) { + const int n = n0 + var; - const void* const data = cgh->data[n][tl]; - int numpoison=0; - for (int k=0; k<size[2]; ++k) { - for (int j=0; j<size[1]; ++j) { - for (int i=0; i<size[0]; ++i) { - const int idx = i + size[0] * (j + size[1] * k); - bool poisoned=false; - switch (tp) { -#define TYPECASE(N,T) \ - case N: { \ - T worm; \ - memset (&worm, poison_value, sizeof worm); \ - const T & val = ((const T*)data)[idx]; \ - poisoned = memcmp (&worm, &val, sizeof worm) == 0; \ - break; \ - } + for (int tl=min_tl; tl<=max_tl; ++tl) { + + const void* const data = cgh->data[n][tl]; + int numpoison=0; + for (int k=0; k<size[2]; ++k) { + for (int j=0; j<size[1]; ++j) { + for (int i=0; i<size[0]; ++i) { + const int idx = i + size[0] * (j + size[1] * k); + bool poisoned=false; + switch (tp) { +#define TYPECASE(N,T) \ + case N: { \ + T worm; \ + memset (&worm, poison_value, sizeof worm); \ + const T & val = ((const T*)data)[idx]; \ + poisoned = memcmp (&worm, &val, sizeof worm) == 0; \ + break; \ + } #include "typecase" #undef TYPECASE - default: - UnsupportedVarType(n); - } - if (poisoned) { - ++numpoison; - if (max_poison_locations==-1 - || numpoison<=max_poison_locations) { - char* fullname = CCTK_FullName(n); - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Timelevel %d, component %d, refinement level %d of the variable \"%s\" contains poison at [%d,%d,%d]", - tl, component, reflevel, fullname, i,j,k); - free (fullname); + default: + UnsupportedVarType(n); } - } // if poisoned - } // for i - } // for j - } // for k - if (max_poison_locations!=-1 && numpoison>max_poison_locations) { - char* fullname = CCTK_FullName(n); - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Timelevel %d, component %d, refinement level %d of the variable \"%s\" contains poison at %d of %d locations; not all locations were printed", - tl, component, reflevel, fullname, numpoison, np); - free (fullname); - } else if (numpoison>0) { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Found poison at %d of %d locations", - numpoison, np); - } - - } // for tl - } // for var - } END_LOCAL_COMPONENT_LOOP; + if (poisoned) { + ++numpoison; + if (max_poison_locations==-1 + || numpoison<=max_poison_locations) { + char* fullname = CCTK_FullName(n); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Timelevel %d, component %d, map %d, refinement level %d of the variable \"%s\" contains poison at [%d,%d,%d]", + tl, component, map, reflevel, fullname, i,j,k); + free (fullname); + } + } // if poisoned + } // for i + } // for j + } // for k + if (max_poison_locations!=-1 && numpoison>max_poison_locations) { + char* fullname = CCTK_FullName(n); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Timelevel %d, component %d, map %d, refinement level %d of the variable \"%s\" contains poison at %d of %d locations; not all locations were printed", + tl, component, map, reflevel, fullname, numpoison, np); + free (fullname); + } else if (numpoison>0) { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Found poison at %d of %d locations", + numpoison, np); + } + + } // for tl + } // for var + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; } // if has storage } // for group diff --git a/Carpet/Carpet/src/Recompose.cc b/Carpet/Carpet/src/Recompose.cc index 24b8dbed8..aea4c2a82 100644 --- a/Carpet/Carpet/src/Recompose.cc +++ b/Carpet/Carpet/src/Recompose.cc @@ -24,9 +24,10 @@ #include "vect.hh" #include "carpet.hh" +#include "modes.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Recompose.cc,v 1.48 2003/11/13 10:49:17 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Recompose.cc,v 1.49 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Recompose_cc); } @@ -40,70 +41,53 @@ namespace Carpet { - typedef vect<bool,dim> bvect; - - typedef vect<int,dim> ivect; - typedef bbox<int,dim> ibbox; - - typedef vect<double,dim> dvect; - - typedef vect<vect<bool,2>,dim> bbvect; - - - - static int (*regrid_routine) (const cGH * cckgGH, - gh<dim>::rexts& bbsss, - gh<dim>::rbnds& obss, - gh<dim>::rprocs& pss) = 0; + // Reduction operator + template<typename iter, typename func> + static typename func::result_type + reduce (iter const first, iter const last, + typename func::result_type const & init) + { + typename func::result_type res (init); + for (iter it (first); it != last; ++it) { + res = func::operator() (res, *it); + } + return res; + } - static void CheckRegions (const gh<dim>::rexts& bbsss, - const gh<dim>::rbnds& obss, + static void CheckRegions (const gh<dim>::rexts & bbsss, + const gh<dim>::rbnds & obss, const gh<dim>::rprocs& pss); - static void Adapt (const cGH* cgh, int reflevels, gh<dim>* hh); - static void Output (const cGH* cgh, const gh<dim>* hh); + static void Output (const cGH* cgh, const int m, const gh<dim>& hh); static void OutputGridStructure (const cGH *cgh, - const gh<dim>::rexts& bbsss, - const gh<dim>::rbnds& obss, + const int m, + const gh<dim>::rexts & bbsss, + const gh<dim>::rbnds & obss, const gh<dim>::rprocs& pss); - void SplitRegions (const cGH* cgh, - vector<ibbox>& bbs, - vector<bbvect>& obs); - void SplitRegions_AlongZ (const cGH* cgh, - vector<ibbox>& bbs, - vector<bbvect>& obs); - void SplitRegions_AlongDir (const cGH* cgh, - vector<ibbox>& bbs, - vector<bbvect>& obs, - const int dir); static void SplitRegions_Automatic_Recursively (bvect const & dims, int const nprocs, - dvect const dshape, + rvect const rshape, ibbox const & bb, bbvect const & ob, vector<ibbox> & bbs, - vector<bbvect> & obs); - void SplitRegions_Automatic (const cGH* cgh, - vector<ibbox>& bbs, - vector<bbvect>& obs); + vector<bbvect> & obs, + vector<int> & ps); static void SplitRegions_AsSpecified (const cGH* cgh, vector<ibbox>& bbs, - vector<bbvect>& obs); - - static void MakeProcessors_RoundRobin (const cGH* cgh, - const gh<dim>::rexts& bbss, - gh<dim>::rprocs& pss); + vector<bbvect>& obs, + vector<int>& ps); - void CheckRegions (const gh<dim>::rexts& bbsss, const gh<dim>::rbnds& obss, - const gh<dim>::rprocs& pss) + void CheckRegions (const gh<dim>::rexts & bbsss, + const gh<dim>::rbnds & obss, + const gh<dim>::rprocs& pss) { // At least one level if (bbsss.size() == 0) { @@ -113,27 +97,27 @@ namespace Carpet { // At most maxreflevels levels if ((int)bbsss.size() > maxreflevels) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "I cannot set up a grid hierarchy with more than Carpet::max_refinement_levels refinement levels. I found Carpet::max_refinement_levels=%d, while %d levels were requested.", - (int)maxreflevels, (int)bbsss.size()); + "I cannot set up a grid hierarchy with more than Carpet::max_refinement_levels refinement levels. I found Carpet::max_refinement_levels=%d, while %d levels were requested.", + (int)maxreflevels, (int)bbsss.size()); } assert ((int)bbsss.size() <= maxreflevels); for (int rl=0; rl<(int)bbsss.size(); ++rl) { // No empty levels assert (bbsss[rl].size() > 0); for (int c=0; c<(int)bbsss[rl].size(); ++c) { - // At least one multigrid level - assert (bbsss[rl][c].size() > 0); - for (int ml=0; ml<(int)bbsss[rl][c].size(); ++ml) { - // Check sizes - // Do allow processors with zero grid points -// assert (all(bbsss[rl][c][ml].lower() <= bbsss[rl][c][ml].upper())); - // Check strides - const int str = ipow(reffact, maxreflevels-rl-1) * ipow(mgfact, ml); - assert (all(bbsss[rl][c][ml].stride() == str)); - // Check alignments - assert (all(bbsss[rl][c][ml].lower() % str == 0)); - assert (all(bbsss[rl][c][ml].upper() % str == 0)); - } + // At least one multigrid level + assert (bbsss[rl][c].size() > 0); + for (int ml=0; ml<(int)bbsss[rl][c].size(); ++ml) { + // Check sizes + // Do allow processors with zero grid points +// assert (all(bbsss[rl][c][ml].lower() <= bbsssi[rl][c][ml].upper())); + // Check strides + const int str = ipow(reffact, maxreflevels-rl-1) * ipow(mgfact, ml); + assert (all(bbsss[rl][c][ml].stride() == str)); + // Check alignments + assert (all(bbsss[rl][c][ml].lower() % str == 0)); + assert (all(bbsss[rl][c][ml].upper() % str == 0)); + } } } @@ -143,174 +127,100 @@ namespace Carpet { assert (obss[rl].size() == bbsss[rl].size()); assert (pss[rl].size() == bbsss[rl].size()); } + } - void RegisterRegridRoutine (int (*routine)(const cGH * cckgGH, - gh<dim>::rexts& bbsss, - gh<dim>::rbnds& obss, - gh<dim>::rprocs& pss)) - { - assert (!regrid_routine); - regrid_routine = routine; - } - - - - void Regrid (const cGH* cgh, + void Regrid (const cGH* cgh, const int rl, const int initialise_from, const bool do_prolongate) { - assert (mglevel == -1); - assert (component == -1); + assert (is_meta_mode()); - if (!regrid_routine) { + if (! CCTK_IsFunctionAliased ("Carpet_Regrid")) { static bool didtell = false; if (!didtell) { - CCTK_WARN (1, "No regridding routine has been registered. There will be no regridding. (Maybe you forgot to activate the regridding thorn?)"); + CCTK_WARN (1, "No regridding routine has been provided. There will be no regridding. Maybe you forgot to activate a regridding thorn?"); didtell = true; } return; } - // Check whether to recompose - gh<dim>::rexts bbsss; - gh<dim>::rbnds obss; - gh<dim>::rprocs pss; - int do_recompose = (*regrid_routine) (cgh, bbsss, obss, pss); - assert (do_recompose >= 0); - if (do_recompose == 0) return; - Recompose (cgh, bbsss, obss, pss, initialise_from, do_prolongate); - } - - - - void Recompose (const cGH* const cgh, - const gh<dim>::rexts& bbsss, - const gh<dim>::rbnds& obss, - const gh<dim>::rprocs& pss, - const int initialise_from, - const bool do_prolongate) - { - assert (mglevel == -1); - assert (component == -1); - - // Check the regions - CheckRegions (bbsss, obss, pss); - - // Write grid structure to file - OutputGridStructure (cgh, bbsss, obss, pss); - - // Recompose - hh->recompose (bbsss, obss, pss, initialise_from, do_prolongate); - - Output (cgh, hh); - } - - - - // This routine is a leftover. It determines "automatically" how - // scalars and arrays should be refined. The user really should - // have a possibility to define how arrays are to be refined. - static void Adapt (const cGH* cgh, const int reflevels, gh<dim>* hh) - { - const int nprocs = CCTK_nProcs(cgh); - vector<vector<ibbox> > bbss(reflevels); - // note: what this routine calls "ub" is "ub+str" elsewhere - ivect rstr = hh->baseextent.stride(); - ivect rlb = hh->baseextent.lower(); - ivect rub = hh->baseextent.upper() + rstr; - for (int rl=0; rl<reflevels; ++rl) { - if (rl>0) { - // save old values - const ivect oldrlb = rlb; - const ivect oldrub = rub; - // calculate extent - const ivect rextent = rub - rlb; - // calculate new extent - assert (all(rextent % hh->reffact == 0)); - const ivect newrextent = rextent / hh->reffact; - // refined boxes have smaller stride - assert (all(rstr%hh->reffact == 0)); - rstr /= hh->reffact; - // refine around the lower boundary only - rlb = rlb; - rub = rlb + newrextent; - // require rub<oldrub because we really want rub-rstr<=oldrub-oldstr - assert (all(rlb >= oldrlb && rub < oldrub)); - } - vector<ibbox> bbs(nprocs); - for (int c=0; c<nprocs; ++c) { - ivect cstr = rstr; - ivect clb = rlb; - ivect cub = rub; - // split the components along the z axis - const int glonpz = (rub[dim-1] - rlb[dim-1]) / cstr[dim-1]; - const int locnpz = (glonpz + nprocs - 1) / nprocs; - const int zstep = locnpz * cstr[dim-1]; - clb[dim-1] = rlb[dim-1] + zstep * c; - cub[dim-1] = rlb[dim-1] + zstep * (c+1); - if (clb[dim-1] > rub[dim-1]) clb[dim-1] = rub[dim-1]; - if (cub[dim-1] > rub[dim-1]) cub[dim-1] = rub[dim-1]; - assert (clb[dim-1] <= cub[dim-1]); - assert (cub[dim-1] <= rub[dim-1]); - bbs[c] = ibbox(clb, cub-cstr, cstr); - } -// bbss[rl] = bbs; - bbss[rl].clear(); - assert (Carpet::hh->components(rl) % nprocs == 0); - for (int cc=0; cc<(int)Carpet::hh->components(rl)/nprocs; ++cc) { - bbss[rl].insert(bbss[rl].end(), bbs.begin(), bbs.end()); + for (int m=0; m<maps; ++m) { + + jjvect nboundaryzones, is_internal, is_staggered, shiftout; + CCTK_INT const ierr = GetBoundarySpecification + (2*dim, &nboundaryzones[0][0], &is_internal[0][0], + &is_staggered[0][0], &shiftout[0][0]); + assert (!ierr); + + gh<dim>::rexts bbsss = vhh[m]->extents; + gh<dim>::rbnds obss = vhh[m]->outer_boundaries; + gh<dim>::rprocs pss = vhh[m]->processors; + + // Check whether to recompose + CCTK_INT const do_recompose = Carpet_Regrid + (cgh, rl, m, + 2*dim, &nboundaryzones[0][0], &is_internal[0][0], + &is_staggered[0][0], &shiftout[0][0], + &bbsss, &obss, &pss); + assert (do_recompose >= 0); + + if (do_recompose) { + + // Check the regions + CheckRegions (bbsss, obss, pss); + // TODO: check also that the current and all coarser levels + // did not change + + // Write grid structure to file + OutputGridStructure (cgh, m, bbsss, obss, pss); + + // Recompose + vhh[m]->recompose (bbsss, obss, pss, + initialise_from, do_prolongate); + + Output (cgh, m, *vhh[m]); + } - } + + } // for m - vector<vector<vector<ibbox> > > bbsss - = hh->make_multigrid_boxes(bbss, mglevels); - - vector<vector<int> > pss(bbss.size()); - vector<vector<bbvect> > obss(bbss.size()); - for (int rl=0; rl<reflevels; ++rl) { - pss[rl] = vector<int>(bbss[rl].size()); - obss[rl] = vector<bbvect>(bbss[rl].size()); - // make sure all processors have the same number of components - assert (bbss[rl].size() % nprocs == 0); - for (int c=0; c<(int)bbss[rl].size(); ++c) { - // distribute among processors - pss[rl][c] = c % nprocs; - for (int d=0; d<dim; ++d) { - // assume the components are split along the z axis - obss[rl][c][d][0] = d<dim-1 || c==0; - obss[rl][c][d][1] = d<dim-1 || c==(int)bbss[rl].size()-1; - } - } + // Calculate new number of levels + reflevels = vhh[0]->reflevels(); + for (int m=0; m<maps; ++m) { + assert (vhh[0]->reflevels() == reflevels); } - hh->recompose(bbsss, obss, pss, 0, true); + // One cannot switch off the current level + assert (reflevels>rl); } - static void Output (const cGH* cgh, const gh<dim>* hh) + static void Output (const cGH* cgh, const int m, const gh<dim>& hh) { DECLARE_CCTK_PARAMETERS; if (verbose) { cout << endl; cout << "New bounding boxes:" << endl; - for (int rl=0; rl<hh->reflevels(); ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - for (int ml=0; ml<hh->mglevels(rl,c); ++ml) { - cout << " rl " << rl << " c " << c << " ml " << ml - << " bbox " << hh->extents[rl][c][ml] << endl; - } + for (int rl=0; rl<hh.reflevels(); ++rl) { + for (int c=0; c<hh.components(rl); ++c) { + for (int ml=0; ml<hh.mglevels(rl,c); ++ml) { + cout << " m " << m << " rl " << rl << " c " << c + << " ml " << ml + << " bbox " << hh.extents[rl][c][ml] + << endl; + } } } cout << endl; cout << "New processor distribution:" << endl; - for (int rl=0; rl<hh->reflevels(); ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - cout << " rl " << rl << " c " << c - << " processor " << hh->processors[rl][c] << endl; + for (int rl=0; rl<hh.reflevels(); ++rl) { + for (int c=0; c<hh.components(rl); ++c) { + cout << " m " << m << " rl " << rl << " c " << c + << " processor " << hh.processors[rl][c] << endl; } } cout << endl; @@ -320,8 +230,9 @@ namespace Carpet { static void OutputGridStructure (const cGH * const cgh, - const gh<dim>::rexts& bbsss, - const gh<dim>::rbnds& obss, + const int m, + const gh<dim>::rexts & bbsss, + const gh<dim>::rbnds & obss, const gh<dim>::rprocs& pss) { DECLARE_CCTK_PARAMETERS; @@ -361,7 +272,7 @@ namespace Carpet { file.open (filename, ios::out | ios::trunc); assert (file.good()); file << "# grid structure" << endl - << "# format: reflevel component mglevel processor bounding-box is-outer-boundary" << endl; + << "# format: map reflevel component mglevel processor bounding-box is-outer-boundary" << endl; assert (file.good()); } } @@ -371,14 +282,15 @@ namespace Carpet { } file << "iteration " << cgh->cctk_iteration << endl; - file << "reflevels " << bbsss.size() << endl; - for (int rl=0; rl<(int)bbsss.size(); ++rl) { - file << rl << " components " << bbsss[rl].size() << endl; - for (int c=0; c<(int)bbsss[rl].size(); ++c) { - file << rl << " " << c << " mglevels " << bbsss[rl][c].size() << endl; - for (int ml=0; ml<(int)bbsss[rl][c].size(); ++ml) { - file << rl << " " << c << " " << ml << " " << pss[rl][c] << " " << bbsss[rl][c][ml] << obss[rl][c] << endl; - } + file << "maps " << maps << endl; + file << m << " reflevels " << bbsss.size() << endl; + for (size_t rl=0; rl<bbsss.size(); ++rl) { + file << m << " " << rl << " components " << bbsss[rl].size() << endl; + for (size_t c=0; c<bbsss[rl].size(); ++c) { + file << m << " " << rl << " " << c << " mglevels " << bbsss[rl][c].size() << endl; + for (size_t ml=0; ml<bbsss[rl][c].size(); ++ml) { + file << m << " " << rl << " " << c << " " << ml << " " << pss[rl][c] << " " << bbsss[rl][c][ml] << obss[rl][c] << endl; + } } } file << endl; @@ -391,18 +303,19 @@ namespace Carpet { // TODO: this routine should go into CarpetRegrid (except maybe // SplitRegions_AlongZ for grid arrays) - void SplitRegions (const cGH* cgh, vector<ibbox>& bbs, vector<bbvect>& obs) + void SplitRegions (const cGH* cgh, vector<ibbox>& bbs, vector<bbvect>& obs, + vector<int>& ps) { DECLARE_CCTK_PARAMETERS; if (CCTK_EQUALS (processor_topology, "along-z")) { - SplitRegions_AlongZ (cgh, bbs, obs); + SplitRegions_AlongZ (cgh, bbs, obs, ps); } else if (CCTK_EQUALS (processor_topology, "along-dir")) { - SplitRegions_AlongDir (cgh, bbs, obs, split_direction); + SplitRegions_AlongDir (cgh, bbs, obs, ps, split_direction); } else if (CCTK_EQUALS (processor_topology, "automatic")) { - SplitRegions_Automatic (cgh, bbs, obs); + SplitRegions_Automatic (cgh, bbs, obs, ps); } else if (CCTK_EQUALS (processor_topology, "manual")) { - SplitRegions_AsSpecified (cgh, bbs, obs); + SplitRegions_AsSpecified (cgh, bbs, obs, ps); } else { assert (0); } @@ -411,23 +324,30 @@ namespace Carpet { void SplitRegions_AlongZ (const cGH* cgh, vector<ibbox>& bbs, - vector<bbvect>& obs) + vector<bbvect>& obs, vector<int>& ps) { - SplitRegions_AlongDir (cgh, bbs, obs, 2); + SplitRegions_AlongDir (cgh, bbs, obs, ps, 2); } void SplitRegions_AlongDir (const cGH* cgh, vector<ibbox>& bbs, - vector<bbvect>& obs, + vector<bbvect>& obs, vector<int>& ps, const int dir) { // Something to do? - if (bbs.size() == 0) return; + if (bbs.size() == 0) { + ps.resize(0); + return; + } const int nprocs = CCTK_nProcs(cgh); - if (nprocs==1) return; + if (nprocs==1) { + ps.resize(1); + ps[0] = 0; + return; + } assert (bbs.size() == 1); @@ -440,6 +360,7 @@ namespace Carpet { bbs.resize(nprocs); obs.resize(nprocs); + ps.resize(nprocs); for (int c=0; c<nprocs; ++c) { ivect cstr = rstr; ivect clb = rlb; @@ -455,20 +376,27 @@ namespace Carpet { assert (cub[dir] <= rub[dir]); bbs[c] = ibbox(clb, cub-cstr, cstr); obs[c] = obnd; + ps[c] = c; if (c>0) obs[c][dir][0] = false; if (c<nprocs-1) obs[c][dir][1] = false; } + + for (size_t n=0; n<ps.size(); ++n) { + assert (ps[n] == n); + } } void SplitRegions_Automatic_Recursively (bvect const & dims, int const nprocs, - dvect const dshape, + rvect const rshape, ibbox const & bb, bbvect const & ob, + int const & p, vector<ibbox> & bbs, - vector<bbvect> & obs) + vector<bbvect> & obs, + vector<int> & ps) { if (DEBUG) cout << "SRAR enter" << endl; // check preconditions @@ -484,6 +412,7 @@ namespace Carpet { // return arguments bbs.assign (1, bb); obs.assign (1, ob); + ps.assign (1, p); // return if (DEBUG) cout << "SRAR exit" << endl; @@ -492,16 +421,16 @@ namespace Carpet { // choose a direction int mydim = -1; - double mysize = 0; + CCTK_REAL mysize = 0; int alldims = 0; - double allsizes = 1; + CCTK_REAL allsizes = 1; for (int d=0; d<dim; ++d) { if (! dims[d]) { ++ alldims; - allsizes *= dshape[d]; - if (dshape[d] >= mysize) { + allsizes *= rshape[d]; + if (rshape[d] >= mysize) { mydim = d; - mysize = dshape[d]; + mysize = rshape[d]; } } } @@ -517,23 +446,29 @@ namespace Carpet { // create the bboxes bbs.clear(); obs.clear(); + ps.clear(); bbs.reserve(nprocs); obs.reserve(nprocs); + ps.reserve(nprocs); // create a new bbox assert (bb.empty()); - bbvect const newob (vect<bool,2>(false)); + bbvect const newob (false); ibbox const newbb (bb); + int const newp (p); if (DEBUG) cout << "SRAR " << mydim << " newbb " << newbb << endl; if (DEBUG) cout << "SRAR " << mydim << " newob " << newob << endl; + if (DEBUG) cout << "SRAR " << mydim << " newp " << newp << endl; // store bbs.insert (bbs.end(), nprocs, newbb); obs.insert (obs.end(), nprocs, newob); + for (int p=0; p<nprocs; ++p) ps.insert (ps.end(), 1, newp+p); // check postconditions assert (bbs.size() == nprocs); assert (obs.size() == nprocs); + assert (ps.size() == nprocs); if (DEBUG) cout << "SRAR exit" << endl; return; } @@ -584,8 +519,10 @@ namespace Carpet { if (DEBUG) cout << "SRAR " << mydim << ": create bboxes" << endl; bbs.clear(); obs.clear(); + ps.clear(); bbs.reserve(nprocs); obs.reserve(nprocs); + ps.reserve(nprocs); ivect last_up; for (int n=0; n<nslices; ++n) { if (DEBUG) cout << "SRAR " << mydim << " n " << n << endl; @@ -605,36 +542,50 @@ namespace Carpet { last_up = up; } ibbox newbb(lo, up, str); + int newp(p + n * mynprocs_base + (n < mynprocs_left ? n : mynprocs_left)); if (DEBUG) cout << "SRAR " << mydim << " newbb " << newbb << endl; if (DEBUG) cout << "SRAR " << mydim << " newob " << newob << endl; + if (DEBUG) cout << "SRAR " << mydim << " newp " << newp << endl; // recurse vector<ibbox> newbbs; vector<bbvect> newobs; + vector<int> newps; SplitRegions_Automatic_Recursively - (newdims, mynprocs[n], dshape, newbb, newob, newbbs, newobs); + (newdims, mynprocs[n], rshape, + newbb, newob, newp, newbbs, newobs, newps); if (DEBUG) cout << "SRAR " << mydim << " newbbs " << newbbs << endl; if (DEBUG) cout << "SRAR " << mydim << " newobs " << newobs << endl; + if (DEBUG) cout << "SRAR " << mydim << " newps " << newps << endl; // store assert (newbbs.size() == mynprocs[n]); assert (newobs.size() == mynprocs[n]); + assert (newps.size() == mynprocs[n]); bbs.insert (bbs.end(), newbbs.begin(), newbbs.end()); obs.insert (obs.end(), newobs.begin(), newobs.end()); + ps.insert (ps.end(), newps.begin(), newps.end()); } // check postconditions assert (bbs.size() == nprocs); assert (obs.size() == nprocs); + assert (ps.size() == nprocs); + for (size_t n=0; n<ps.size(); ++n) { + assert (ps[n] == p+n); + } if (DEBUG) cout << "SRAR exit" << endl; } void SplitRegions_Automatic (const cGH* cgh, vector<ibbox>& bbs, - vector<bbvect>& obs) + vector<bbvect>& obs, vector<int>& ps) { if (DEBUG) cout << "SRA enter" << endl; // Something to do? - if (bbs.size() == 0) return; + if (bbs.size() == 0) { + ps.resize(0); + return; + } const int nprocs = CCTK_nProcs(cgh); if (DEBUG) cout << "SRA nprocs " << nprocs << endl; @@ -664,9 +615,9 @@ namespace Carpet { while (ncomps_left > 0) { if (DEBUG) cout << "SRA ncomps_left " << ncomps_left << endl; int maxc = -1; - double maxratio = -1; + CCTK_REAL maxratio = -1; for (int c=0; c<nslices; ++c) { - double const ratio = (double)mysize[c] / mynprocs[c]; + CCTK_REAL const ratio = (CCTK_REAL)mysize[c] / mynprocs[c]; if (ratio > maxratio) { maxc=c; maxratio=ratio; } } assert (maxc>=0 && maxc<nslices); @@ -681,12 +632,18 @@ namespace Carpet { vector<ibbox> allbbs; vector<bbvect> allobs; + vector<int> allps; if (DEBUG) cout << "SRA: splitting regions" << endl; for (int c=0; c<nslices; ++c) { const ibbox bb = bbs[c]; const bbvect ob = obs[c]; + int p = 0; + for (int cc=0; cc<c; ++cc) { + p += mynprocs[cc]; + } + assert (p>=0 && p<=nprocs); if (DEBUG) cout << "SRA c " << c << endl; if (DEBUG) cout << "SRA bb " << bb << endl; if (DEBUG) cout << "SRA ob " << ob << endl; @@ -696,37 +653,44 @@ namespace Carpet { const ivect rub = bb.upper() + rstr; // calculate real shape factors - dvect dshape; + rvect rshape; if (any(rub == rlb)) { // the bbox is empty - dshape = 0.0; + rshape = 0.0; } else { for (int d=0; d<dim; ++d) { - dshape[d] = (double)(rub[d]-rlb[d]) / (rub[0]-rlb[0]); + rshape[d] = (CCTK_REAL)(rub[d]-rlb[d]) / (rub[0]-rlb[0]); } - const double dfact = pow(nprocs / prod(dshape), 1.0/dim); - dshape *= dfact; - assert (abs(prod(dshape) - nprocs) < 1e-6); + const CCTK_REAL rfact = pow(nprocs / prod(rshape), 1.0/dim); + rshape *= rfact; + assert (abs(prod(rshape) - nprocs) < 1e-6); } - if (DEBUG) cout << "SRA shapes " << dshape << endl; + if (DEBUG) cout << "SRA shapes " << rshape << endl; bvect const dims = false; vector<ibbox> thebbs; vector<bbvect> theobs; + vector<int> theps; SplitRegions_Automatic_Recursively - (dims, mynprocs[c], dshape, bb, ob, thebbs, theobs); + (dims, mynprocs[c], rshape, bb, ob, p, thebbs, theobs, theps); if (DEBUG) cout << "SRA thebbs " << thebbs << endl; if (DEBUG) cout << "SRA theobs " << theobs << endl; + if (DEBUG) cout << "SRA theps " << theps << endl; allbbs.insert(allbbs.end(), thebbs.begin(), thebbs.end()); allobs.insert(allobs.end(), theobs.begin(), theobs.end()); + allps.insert(allps.end(), theps.begin(), theps.end()); } // for c bbs = allbbs; obs = allobs; + ps = allps; + for (size_t n=0; n<ps.size(); ++n) { + assert (ps[n] == n); + } if (DEBUG) cout << "SRA exit" << endl; } @@ -734,12 +698,15 @@ namespace Carpet { void SplitRegions_AsSpecified (const cGH* cgh, vector<ibbox>& bbs, - vector<bbvect>& obs) + vector<bbvect>& obs, vector<int>& ps) { DECLARE_CCTK_PARAMETERS; // Something to do? - if (bbs.size() == 0) return; + if (bbs.size() == 0) { + ps.resize(0); + return; + } const int nprocs = CCTK_nProcs(cgh); @@ -756,7 +723,7 @@ namespace Carpet { assert (all (nprocs_dir > 0)); if (prod(nprocs_dir) != nprocs) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "The specified processor topology [%d,%d,%d] is inconsistent with the number of processors, which is %d", nprocs_dir[0], nprocs_dir[1], nprocs_dir[2], nprocs); + "The specified processor topology [%d,%d,%d] requires %d processors, but there are %d processors", nprocs_dir[0], nprocs_dir[1], nprocs_dir[2], prod(nprocs_dir), nprocs); } assert (prod(nprocs_dir) == nprocs); @@ -764,6 +731,7 @@ namespace Carpet { bbs.resize(nprocs); obs.resize(nprocs); + ps.resize(nprocs); const ivect cstr = rstr; const ivect glonp = (rub - rlb) / cstr; // const ivect locnp = (glonp + nprocs_dir - 1) / nprocs_dir; @@ -796,6 +764,7 @@ namespace Carpet { assert (all (! (ipos==nprocs_dir-1) || cub==rub)); bbs[c] = ibbox(clb, cub-cstr, cstr); obs[c] = obnd; + ps[c] = c; if (i>0) obs[c][0][0] = false; if (j>0) obs[c][1][0] = false; if (k>0) obs[c][2][0] = false; @@ -805,34 +774,68 @@ namespace Carpet { } } } + + for (size_t n=0; n<ps.size(); ++n) { + assert (ps[n] == n); + } } - void MakeProcessors (const cGH* cgh, const gh<dim>::rexts& bbsss, - gh<dim>::rprocs& pss) + static void MakeMultigridBoxes (const cGH* cgh, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + ibbox const & bb, + bbvect const & ob, + vector<ibbox>& bbs) { - MakeProcessors_RoundRobin (cgh, bbsss, pss); + bbs.resize (mglevels); + bbs[0] = bb; + // boundary offsets + assert (size==2*dim); + // (distance in grid points between the exterior and the physical boundary) + iivect offset; + for (int d=0; d<dim; ++d) { + for (int f=0; f<2; ++f) { + assert (! is_staggered[d][f]); + offset[d][f] = (+ (is_internal[d][f] ? 0 : nboundaryzones[d][f] - 1) + + shiftout[d][f]); + } + } + for (int ml=1; ml<mglevels; ++ml) { + // next finer grid + ivect const flo = bbs[ml-1].lower(); + ivect const fhi = bbs[ml-1].upper(); + ivect const fstr = bbs[ml-1].stride(); + // this grid + ivect const str = fstr * mgfact; + ivect const modoffset = (xpose(offset)[0] - ivect(mgfact) * xpose(offset)[0] + ivect(mgfact) * xpose(offset)[0] * str) % str; + ivect const lo = flo + xpose(ob)[0].ifthen ( xpose(offset)[0] - ivect(mgfact) * xpose(offset)[0] , modoffset); + ivect const hi = fhi + xpose(ob)[1].ifthen ( - (xpose(offset)[1] - ivect(mgfact) * xpose(offset)[1]), modoffset + fstr - str); + bbs[ml] = ibbox(lo,hi,str); + } } - - - // This is a helpful helper routine. The user can use it to define - // how the hierarchy should be refined. But the result of this - // routine is rather arbitrary. - void MakeProcessors_RoundRobin (const cGH* cgh, const gh<dim>::rexts& bbsss, - gh<dim>::rprocs& pss) + void MakeMultigridBoxes (const cGH* cgh, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + vector<ibbox> const & bbs, + vector<bbvect> const & obs, + vector<vector<ibbox> >& bbss) { - const int nprocs = CCTK_nProcs(cgh); - - pss.resize(bbsss.size()); - for (int rl=0; rl<(int)bbsss.size(); ++rl) { - pss[rl] = vector<int>(bbsss[rl].size()); - // make sure all processors have the same number of components - assert (bbsss[rl].size() % nprocs == 0); - for (int c=0; c<(int)bbsss[rl].size(); ++c) { - pss[rl][c] = c % nprocs; // distribute among processors - } + assert (bbs.size() == obs.size()); + bbss.resize(bbs.size()); + for (size_t c=0; c<bbs.size(); ++c) { + MakeMultigridBoxes + (cgh, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs[c], obs[c], bbss[c]); } } diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc index 06d87ef1a..c200a22b3 100644 --- a/Carpet/Carpet/src/Restrict.cc +++ b/Carpet/Carpet/src/Restrict.cc @@ -3,6 +3,7 @@ #include <stdlib.h> #include "cctk.h" +#include "cctk_Parameters.h" #include "ggf.hh" #include "gh.hh" @@ -10,7 +11,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Restrict.cc,v 1.22 2003/11/05 16:18:37 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Restrict.cc,v 1.23 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Restrict_cc); } @@ -24,42 +25,51 @@ namespace Carpet { void Restrict (const cGH* cgh) { - assert (reflevel != -1); - assert (mglevel != -1); - assert (component == -1); + DECLARE_CCTK_PARAMETERS; - Checkpoint ("%*sRestrict", 2*reflevel, ""); + assert (is_level_mode()); + + if (suppress_restriction) { + Checkpoint ("Restriction suppressed"); + return; + } + + Checkpoint ("Restrict"); // Restrict - if (reflevel < hh->reflevels()-1) { + if (reflevel < reflevels-1) { for (comm_state<dim> state; !state.done(); state.step()) { for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { if (CCTK_QueryGroupStorageI(cgh, group)) { - if (arrdata[group].do_transfer) { - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - - const int tl = 0; + if (groupdata[group].transport_operator != op_none) { + + const int tl = 0; + + for (int m=0; m<maps; ++m) { + assert (m<(int)arrdata[group].size()); // use background time here (which may not be // modified by the user) - const CCTK_REAL time = tt->time (tl, reflevel, mglevel); - const CCTK_REAL time1 = tt->time (0, reflevel, mglevel); - const CCTK_REAL time2 = cgh->cctk_time / cgh->cctk_delta_time; - assert (fabs((time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time))) < 1e-12); + const CCTK_REAL time = vtt[m]->time (tl, reflevel, mglevel); + const CCTK_REAL time1 = vtt[m]->time (0, reflevel, mglevel); + const CCTK_REAL time2 = (cgh->cctk_time - cctk_initial_time) / cgh->cctk_delta_time; + assert (fabs(time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time)) < 1e-12); - for (int c=0; c<hh->components(reflevel); ++c) { - arrdata[group].data[var]->ref_restrict - (state, tl, reflevel, c, mglevel, time); + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata[group][m].data.size()); + for (int c=0; c<vhh[m]->components(reflevel); ++c) { + arrdata[group][m].data[var]->ref_restrict + (state, tl, reflevel, c, mglevel, time); + } } - - } // loop over variables + } + } else { if (state.thestate==state_recv) { - char * groupname = CCTK_GroupName(group); - Checkpoint ("%*s(no restricting for group %s)", - 2*reflevel, "", groupname); + char * const groupname = CCTK_GroupName(group); + Checkpoint ("(no restricting for group %s)", groupname); free (groupname); } } // if ! do_transfer @@ -72,22 +82,26 @@ namespace Carpet { // Sync - if (reflevel < hh->reflevels()-1) { + if (reflevel < reflevels-1) { for (comm_state<dim> state; !state.done(); state.step()) { for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { if (CCTK_QueryGroupStorageI(cgh, group)) { - if (arrdata[group].do_transfer) { - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - - const int tl = 0; - - for (int c=0; c<arrdata[group].hh->components(reflevel); ++c) { - arrdata[group].data[var]->sync - (state, tl, reflevel, c, mglevel); + if (groupdata[group].transport_operator != op_none) { + + const int tl = 0; + + for (int m=0; m<maps; ++m) { + assert (m<(int)arrdata[group].size()); + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata[group][m].data.size()); + for (int c=0; c<vhh[m]->components(reflevel); ++c) { + arrdata[group][m].data[var]->sync + (state, tl, reflevel, c, mglevel); + } } - - } // loop over variables + } + } // if do_transfer } // if group has storage } // if grouptype == CCTK_GF @@ -98,3 +112,4 @@ namespace Carpet { } } // namespace Carpet + diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc index d9416510c..bb801ca96 100644 --- a/Carpet/Carpet/src/SetupGH.cc +++ b/Carpet/Carpet/src/SetupGH.cc @@ -1,5 +1,6 @@ #include <assert.h> #include <limits.h> +#include <math.h> #include <stdlib.h> #include <string.h> @@ -10,6 +11,9 @@ #include "cctk.h" #include "cctk_Parameters.h" +#include "util_ErrorCodes.h" +#include "util_Table.h" + #include "bbox.hh" #include "defs.hh" #include "dist.hh" @@ -20,7 +24,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/SetupGH.cc,v 1.56 2003/11/11 14:23:41 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/SetupGH.cc,v 1.57 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_SetupGH_cc); } @@ -32,7 +36,7 @@ namespace Carpet { - static bool CanTransferVariableType (cGH* cgh, const int group) + static bool CanTransferVariableType (const cGH * const cgh, const int group) { // Find out which types correspond to the default types #if CCTK_INTEGER_PRECISION_1 @@ -135,6 +139,67 @@ namespace Carpet { + static operator_type GetTransportOperator (const cGH * const cgh, + const int group) + { + assert (group>=0 && group<CCTK_NumGroups()); + + const bool can_transfer = CanTransferVariableType (cgh, group); + + cGroup gp; + const int ierr = CCTK_GroupData (group, &gp); + assert (!ierr); + const int tagstable = CCTK_GroupTagsTableI (group); + assert (tagstable >= 0); + + char prolong_string[1000]; + const int length = Util_TableGetString + (tagstable, sizeof prolong_string, prolong_string, "Prolongation"); + if (length == UTIL_ERROR_TABLE_NO_SUCH_KEY) { + if (can_transfer) { + // Use the default + if (gp.numtimelevels == 1) { + // Only one time level: do not prolongate + char * const groupname = CCTK_GroupName (group); + const char * const vartypename = CCTK_VarTypeName(gp.vartype); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Group %s has only one time level; therefore it will not be prolongated or restricted", + groupname); + free (groupname); + return op_none; + } else { + // Several time levels: use the default + return op_Lagrange; + } + } else { + if (gp.grouptype == CCTK_GF) { + char * const groupname = CCTK_GroupName (group); + const char * const vartypename = CCTK_VarTypeName(gp.vartype); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Group %s has the variable type %s which cannot be prolongated or restricted", + groupname, vartypename); + free (groupname); + return op_none; + } else { + return op_error; + } + } + } + assert (length >= 0); + if (CCTK_Equals(prolong_string, "None")) { + return op_none; + } else if (CCTK_Equals(prolong_string, "Lagrange")) { + return op_Lagrange; + } else if (CCTK_Equals(prolong_string, "TVD")) { + return op_TVD; + } else { + assert (0); + } + return op_error; + } + + + void* SetupGH (tFleshConfig* fc, int convLevel, cGH* cgh) { DECLARE_CCTK_PARAMETERS; @@ -148,82 +213,280 @@ namespace Carpet { dist::pseudoinit(); - CCTK_VInfo (CCTK_THORNSTRING, - "Carpet is running on %d processors", CCTK_nProcs(cgh)); + // Initialise current position + mglevel = -1; + reflevel = -1; + map = -1; + component = -1; + + + + Waypoint ("Setting up the grid hierarchy"); - Waypoint ("starting SetupGH..."); + // Processor information + Waypoint ("Carpet is running on %d processors", CCTK_nProcs(cgh)); + + // Multigrid information + basemglevel = convergence_level; + mglevels = num_convergence_levels; + mgfact = convergence_factor; + maxmglevelfact = ipow(mgfact, mglevels-1); +// TODO: disable temporarily +// cgh->cctk_convfac = mgfact; + assert (mgfact == 2); // Refinement information maxreflevels = max_refinement_levels; reffact = refinement_factor; maxreflevelfact = ipow(reffact, maxreflevels-1); - // Multigrid information - mglevels = multigrid_levels; - mgfact = multigrid_factor; - maxmglevelfact = ipow(mgfact, mglevels-1); + // Map information + maps = num_maps; +// TODO: disable temporarily +// cgh->cctk_nmaps = maps; + assert (maps == 1); - // Ghost zones - vect<int,dim> lghosts, ughosts; - if (ghost_size == -1) { - lghosts = vect<int,dim>(ghost_size_x, ghost_size_y, ghost_size_z); - ughosts = vect<int,dim>(ghost_size_x, ghost_size_y, ghost_size_z); - } else { - lghosts = vect<int,dim>(ghost_size, ghost_size, ghost_size); - ughosts = vect<int,dim>(ghost_size, ghost_size, ghost_size); - } - // Grid size - const int stride = maxreflevelfact; - vect<int,dim> npoints; - if (global_nsize == -1) { - npoints = vect<int,dim>(global_nx, global_ny, global_nz); - } else { - npoints = vect<int,dim>(global_nsize, global_nsize, global_nsize); - } - // Sanity check - // (if this fails, someone requested an insane amount of memory) - assert (all(npoints <= INT_MAX)); - { - int max = INT_MAX; - for (int d=0; d<dim; ++d) { - assert (npoints[d] <= max); - max /= npoints[d]; - } - } - const vect<int,dim> str(stride); - const vect<int,dim> lb(0); - const vect<int,dim> ub((npoints - 1) * str); - - const bbox<int,dim> baseext(lb, ub, str); - - // Allocate grid hierarchy - hh = new gh<dim>(refinement_factor, vertex_centered, - multigrid_factor, vertex_centered, - baseext); - - // Allocate time hierarchy - tt = new th<dim>(hh, 1.0); - - // Allocate data hierarchy - dd = new dh<dim>(*hh, lghosts, ughosts, - prolongation_order_space, buffer_width); - - if (max_refinement_levels > 1) { - const int prolongation_stencil_size = dd->prolongation_stencil_size(); - const int min_nghosts - = ((prolongation_stencil_size + refinement_factor - 1) - / (refinement_factor-1)); - if (any(min(lghosts,ughosts) < min_nghosts)) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "There are not enough ghost zones for the desired spatial prolongation order. With Carpet::prolongation_order_space=%d, you need at least %d ghost zones.", - prolongation_order_space, min_nghosts); + // Allocate space for groups + groupdata.resize(CCTK_NumGroups()); + arrdata.resize(CCTK_NumGroups()); + + vhh.resize(maps); + vdd.resize(maps); + vtt.resize(maps); + + // Loop over maps + for (int m=0; m<maps; ++m) { + + // Get boundary description + jjvect nboundaryzones, is_internal, is_staggered, shiftout; + ierr = GetBoundarySpecification + (2*dim, &nboundaryzones[0][0], &is_internal[0][0], + &is_staggered[0][0], &shiftout[0][0]); + assert (!ierr); + + { + ostringstream buf; + buf << "CoordBase boundary specification for map " << m << ":" << endl + << " nboundaryzones: " << nboundaryzones << endl + << " is_internal : " << is_internal << endl + << " is_staggered : " << is_staggered << endl + << " shiftout : " << shiftout; + Output (buf.str().c_str()); } + + // Ghost zones + ivect lghosts, ughosts; + if (ghost_size == -1) { + lghosts = ivect(ghost_size_x, ghost_size_y, ghost_size_z); + ughosts = ivect(ghost_size_x, ghost_size_y, ghost_size_z); + } else { + lghosts = ivect(ghost_size, ghost_size, ghost_size); + ughosts = ivect(ghost_size, ghost_size, ghost_size); + } + + // Grid size + rvect physical_min, physical_max; + rvect interior_min, interior_max; + rvect exterior_min, exterior_max; + rvect base_spacing; + + if (domain_from_coordbase) { + + ierr = GetDomainSpecification + (dim, &physical_min[0], &physical_max[0], + &interior_min[0], &interior_max[0], + &exterior_min[0], &exterior_max[0], &base_spacing[0]); + assert (!ierr); + + } else { + // Legacy code + + // specify global number of grid points + ivect npoints; + if (global_nsize == -1) { + npoints = ivect(global_nx, global_ny, global_nz); + } else { + npoints = ivect(global_nsize, global_nsize, global_nsize); + } + ostringstream buf; + buf << "Standard grid specification for map " << m << ":" << endl + << " number of grid points: " << npoints; + Output (buf.str().c_str()); + + // reduce to physical domain + exterior_min = 0.0; + exterior_max = rvect(npoints - 1); + base_spacing = 1.0; + ierr = ConvertFromExteriorBoundary + (dim, &physical_min[0], &physical_max[0], + &interior_min[0], &interior_max[0], + &exterior_min[0], &exterior_max[0], &base_spacing[0]); + assert (!ierr); + + } + + { + ostringstream buf; + buf << "CoordBase domain specification for map " << m << ":" << endl + << " physical extent: " << physical_min << " : " << physical_max << endl + << " interior extent: " << interior_min << " : " << interior_max << endl + << " exterior extent: " << exterior_min << " : " << exterior_max << endl + << " base_spacing : " << base_spacing; + Output (buf.str().c_str()); + } + + // Adapt for convergence level + rvect const spacing + = base_spacing * pow ((CCTK_REAL)convergence_factor, basemglevel); + + { + ostringstream buf; + buf << "Adapted domain specification for map " << m << ":" << endl + << " convergence factor: " << convergence_factor << endl + << " convergence level : " << basemglevel << endl + << " spacing : " << spacing; + Output (buf.str().c_str()); + } + + // Calculate global number of grid points + ierr = ConvertFromPhysicalBoundary + (dim, &physical_min[0], &physical_max[0], + &interior_min[0], &interior_max[0], + &exterior_min[0], &exterior_max[0], &spacing[0]); + assert (!ierr); + rvect const real_npoints = (exterior_max - exterior_min) / spacing + 1; + + { + ostringstream buf; + buf << "Base grid specification for map " << m << ":" << endl + << " number of grid points : " << real_npoints << endl + << " number of ghost points: " << lghosts; + Output (buf.str().c_str()); + } + + CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; + const ivect npoints = ::map(rfloor, real_npoints + 0.5); + if (any(abs(npoints - real_npoints) > 0.001)) { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "The domain size for map %d scaled for convergence level %d with convergence factor %d is not integer", + m, basemglevel, convergence_factor); + } + + // Sanity check + // (if this fails, someone requested an insane amount of memory) + assert (all(npoints <= INT_MAX)); + { + int max = INT_MAX; + for (int d=0; d<dim; ++d) { + assert (npoints[d] <= max); + max /= npoints[d]; + } + } + + + + // Base grid extent + const int stride = maxreflevelfact; + const ivect str(stride); + const ivect lb(0); + const ivect ub((npoints - 1) * str); + const ibbox baseext(lb, ub, str); + + // Allocate grid hierarchy + vhh[m] = new gh<dim>(refinement_factor, vertex_centered, + convergence_factor, vertex_centered, baseext); + + // Allocate data hierarchy + vdd[m] = new dh<dim>(*vhh[m], lghosts, ughosts, + prolongation_order_space, buffer_width); + + // Allocate time hierarchy + vtt[m] = new th<dim>(*vhh[m], 1.0); + + if (max_refinement_levels > 1) { + const int prolongation_stencil_size + = vdd[m]->prolongation_stencil_size(); + const int min_nghosts + = ((prolongation_stencil_size + refinement_factor - 1) + / (refinement_factor-1)); + if (any(min(lghosts,ughosts) < min_nghosts)) { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "There are not enough ghost zones for the desired spatial prolongation order on map %d. With Carpet::prolongation_order_space=%d, you need at least %d ghost zones.", + m, prolongation_order_space, min_nghosts); + } + + } + + + + // Set initial refinement structure + + vector<ibbox> bbs; + vector<bbvect> obs; + if (strcmp(base_extents, "") == 0) { + + // default: one grid component covering everything + bbs.push_back (vhh[m]->baseextent); + obs.push_back (bbvect(true)); + + } else { + + // explicit grid components + // TODO: invent something for the other convergence levels + istringstream ext_str(base_extents); + try { + ext_str >> bbs; + } catch (input_error) { + CCTK_WARN (0, "Could not parse parameter \"base_extents\""); + } + CCTK_VInfo (CCTK_THORNSTRING, "Using %d grid patches", bbs.size()); + cout << "grid-patches-are " << bbs << endl; + if (bbs.size()<=0) { + CCTK_WARN (0, "Cannot evolve with 0 grid patches"); + } + istringstream ob_str (base_outerbounds); + try { + ob_str >> obs; + } catch (input_error) { + CCTK_WARN (0, "Could not parse parameter \"base_outerbounds\""); + } + assert (obs.size() == bbs.size()); + + } + + // Distribute onto the processors + // (TODO: this should be done globally for all maps) + vector<int> ps; + SplitRegions (cgh, bbs, obs, ps); + + // Create all multigrid levels + vector<vector<ibbox> > bbss; + MakeMultigridBoxes + (cgh, + 2*dim, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); + + // Only one refinement level + vector<vector<vector<ibbox> > > bbsss(1); + vector<vector<bbvect> > obss(1); + vector<vector<int> > pss(1); + bbsss[0] = bbss; + obss[0] = obs; + pss[0] = ps; + + // Recompose grid hierarchy + vhh[m]->recompose (bbsss, obss, pss, maxreflevels, false); + + } // loop over maps + + reflevels = 1; + for (int m=0; m<maps; ++m) { + assert (vhh[m]->reflevels() == reflevels); } - // Allocate space for groups - arrdata.resize(CCTK_NumGroups()); + // Allocate space for variables in group (but don't enable storage // yet) @@ -232,28 +495,39 @@ namespace Carpet { cGroup gp; ierr = CCTK_GroupData (group, &gp); assert (!ierr); + const int tagstable = CCTK_GroupTagsTableI (group); + assert (tagstable >= 0); switch (gp.grouptype) { + case CCTK_GF: { + assert (gp.dim == dim); + arrdata[group].resize(maps); + for (int m=0; m<maps; ++m) { + arrdata[group][m].hh = vhh[m]; + arrdata[group][m].dd = vdd[m]; + arrdata[group][m].tt = vtt[m]; + } + break; + } + case CCTK_SCALAR: case CCTK_ARRAY: { - - int disttype; - vect<int,dim> sizes(1), ghostsizes(0); + + arrdata[group].resize(1); + + ivect sizes(1), ghostsizes(0); switch (gp.grouptype) { case CCTK_SCALAR: // treat scalars as DIM=0, DISTRIB=const arrays assert (gp.dim==0); - arrdata[group].info.dim = gp.dim; - disttype = CCTK_DISTRIB_CONSTANT; + assert (gp.disttype == CCTK_DISTRIB_CONSTANT); break; case CCTK_ARRAY: { assert (gp.dim>=1 || gp.dim<=dim); - arrdata[group].info.dim = gp.dim; - disttype = gp.disttype; const CCTK_INT * const * const sz = CCTK_GroupSizesI(group); const CCTK_INT * const * const gsz = CCTK_GroupGhostsizesI(group); for (int d=0; d<gp.dim; ++d) { @@ -262,88 +536,172 @@ namespace Carpet { } break; } - + default: assert (0); } - - assert (disttype==CCTK_DISTRIB_CONSTANT - || disttype==CCTK_DISTRIB_DEFAULT); - - if (disttype==CCTK_DISTRIB_CONSTANT) { - for (int d=0; d<dim; ++d) { - ghostsizes[d] = 0; + + ivect alghosts(0), aughosts(0); + for (int d=0; d<gp.dim; ++d) { + alghosts[d] = ghostsizes[d]; + aughosts[d] = ghostsizes[d]; + } + + + + // Adapt array sizes for convergence level + jvect convpowers (0); + jvect convoffsets (0); + + if (tagstable >= 0) { + int status; + + status = Util_TableGetIntArray + (tagstable, gp.dim, &convpowers[0], "convergence_power"); + if (status == UTIL_ERROR_TABLE_NO_SUCH_KEY) { + // keep default: independent of convergence level + } else if (status == 1) { + // a scalar was given + convpowers = convpowers[0]; + } else if (status == gp.dim) { + // do nothing + } else { + char * const groupname = CCTK_GroupName(group); + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "The key \"convergence_power\" in the tags table of group \"%s\" is wrong", + groupname); + free (groupname); } + assert (all (convpowers >= 0)); + + status = Util_TableGetIntArray + (tagstable, gp.dim, &convoffsets[0], "convergence_offset"); + if (status == UTIL_ERROR_TABLE_NO_SUCH_KEY) { + // keep default: offset is 0 + } else if (status == 1) { + // a scalar was given + convoffsets = convoffsets[0]; + } else if (status == gp.dim) { + // do nothing + } else { + char * const groupname = CCTK_GroupName(group); + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "The key \"convergence_offset\" in the tags table of group \"%s\" is wrong", + groupname); + free (groupname); + } + + } // if there is a group tags table + + CCTK_REAL (* const ripow) (CCTK_REAL const, int const) = pow; + CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; + const rvect real_sizes + = ((sizes - convoffsets) + / ::zip(ripow, rvect(convergence_factor), + convpowers * basemglevel) + + convoffsets); + sizes = ::map(rfloor, (real_sizes + 0.5)); + if (any (sizes < 0)) { + char * const groupname = CCTK_GroupName(group); + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "The shape of group \"%s\" scaled for convergence level %d with convergence factor %d is negative", + groupname, basemglevel, convergence_factor); + free (groupname); + } + if (any (sizes - real_sizes > 0.001)) { + char * const groupname = CCTK_GroupName(group); + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "The shape of group \"%s\" scaled for convergence level %d with convergence factor %d is not integer", + groupname, basemglevel, convergence_factor); + free (groupname); + } + + + + assert (gp.disttype==CCTK_DISTRIB_CONSTANT + || gp.disttype==CCTK_DISTRIB_DEFAULT); + + if (gp.disttype==CCTK_DISTRIB_CONSTANT) { + assert (all (ghostsizes == 0)); const int d = gp.dim==0 ? 0 : gp.dim-1; - sizes[d] = (sizes[d] - 2*ghostsizes[d]) * CCTK_nProcs(cgh) + 2*ghostsizes[d]; + sizes[d] = (sizes[d] - 2*ghostsizes[d]) * CCTK_nProcs(cgh) + 2*ghostsizes[d]; assert (sizes[d] >= 0); - } - - const vect<int,dim> alb(0); - const vect<int,dim> aub(sizes-1); - const vect<int,dim> astr(1); - const bbox<int,dim> arrext(alb, aub, astr); - - arrdata[group].hh = new gh<dim>(refinement_factor, vertex_centered, - multigrid_factor, vertex_centered, - arrext); - - arrdata[group].tt = new th<dim>(arrdata[group].hh, 1.0); - - vect<int,dim> alghosts(0), aughosts(0); - for (int d=0; d<gp.dim; ++d) { - alghosts[d] = ghostsizes[d]; - aughosts[d] = ghostsizes[d]; - } - - arrdata[group].dd - = new dh<dim>(*arrdata[group].hh, alghosts, aughosts, 0, 0); - - // Set refinement structure for scalars and arrays: - - // Set the basic extent - vector<bbox<int,dim> > bbs; - vector<vect<vect<bool,2>,dim> > obs; - bbs.push_back (arrdata[group].hh->baseextent); - obs.push_back (vect<vect<bool,2>,dim>(vect<bool,2>(true))); + } + + + + const ivect alb(0); + const ivect aub(sizes-1); + const ivect astr(1); + const ibbox abaseext(alb, aub, astr); + + arrdata[group][0].hh + = new gh<dim>(refinement_factor, vertex_centered, + convergence_factor, vertex_centered, + abaseext); + + arrdata[group][0].dd + = new dh<dim>(*arrdata[group][0].hh, alghosts, aughosts, 0, 0); + + arrdata[group][0].tt = new th<dim>(*arrdata[group][0].hh, 1.0); - // Split it into components, one for each processor - if (disttype==CCTK_DISTRIB_CONSTANT) { - SplitRegions_AlongDir (cgh, bbs, obs, gp.dim==0 ? 0 : gp.dim-1); + + + // Set refinement structure for scalars and arrays + vector<ibbox> bbs; + vector<bbvect> obs; + bbs.push_back (abaseext); + obs.push_back (bbvect(true)); + + // Split it into components, one for each processor + vector<int> ps; + if (gp.disttype==CCTK_DISTRIB_CONSTANT) { + SplitRegions_AlongDir (cgh, bbs, obs, ps, gp.dim==0 ? 0 : gp.dim-1); } else { - SplitRegions_Automatic (cgh, bbs, obs); + SplitRegions_Automatic (cgh, bbs, obs, ps); } - - // For all refinement levels (but there is only one) - vector<vector<bbox<int,dim> > > bbss(1); - vector<vector<vect<vect<bool,2>,dim> > > obss(1); - bbss[0] = bbs; + + // Create all multigrid levels + vector<vector<ibbox> > bbss (bbs.size()); + ivect amgfact; + iivect aoffset; + for (int d=0; d<dim; ++d) { + amgfact[d] = ipow(mgfact, convpowers[d]); + aoffset[d][0] = 0; + aoffset[d][1] = convoffsets[d]; + } + for (size_t c=0; c<bbs.size(); ++c) { + bbss[c].resize (mglevels); + bbss[c][0] = bbs[c]; + for (int ml=1; ml<mglevels; ++ml) { + // next finer grid + ivect const flo = bbss[c][ml-1].lower(); + ivect const fhi = bbss[c][ml-1].upper(); + ivect const fstr = bbss[c][ml-1].stride(); + // this grid + ivect const str = fstr * amgfact; + ivect const modoffset = (xpose(aoffset)[0] - amgfact * xpose(aoffset)[0] + amgfact * xpose(aoffset)[0] * str) % str; + ivect const lo = flo + xpose(obs[c])[0].ifthen ( xpose(aoffset)[0] - amgfact * xpose(aoffset)[0] , modoffset); + ivect const hi = fhi + xpose(obs[c])[1].ifthen ( - (xpose(aoffset)[1] - amgfact * xpose(aoffset)[1]), modoffset + fstr - str); + bbss[c][ml] = ibbox(lo,hi,str); + } + } + + // Only one refinement level + vector<vector<vector<ibbox> > > bbsss(1); + vector<vector<bbvect> > obss(1); + vector<vector<int> > pss(1); + bbsss[0] = bbss; obss[0] = obs; + pss[0] = ps; - // For all multigrid levels - gh<dim>::rexts bbsss; - bbsss = hh->make_multigrid_boxes(bbss, mglevels); - - // Distribute onto processors - vector<vector<int> > pss; - MakeProcessors (cgh, bbsss, pss); - - // And recompose. Done. - char * groupname = CCTK_GroupName (group); + // Recompose for this map + char * const groupname = CCTK_GroupName (group); assert (groupname); Checkpoint ("Recomposing grid array group %s", groupname); free (groupname); - arrdata[group].hh->recompose (bbsss, obss, pss, 1, false); - - break; - } + arrdata[group][0].hh->recompose (bbsss, obss, pss, 1, false); - case CCTK_GF: { - assert (gp.dim == dim); - arrdata[group].info.dim = dim; - arrdata[group].hh = hh; - arrdata[group].tt = tt; - arrdata[group].dd = dd; break; } @@ -351,186 +709,73 @@ namespace Carpet { assert (0); } - arrdata[group].info.gsh = new int [dim]; - arrdata[group].info.lsh = new int [dim]; - arrdata[group].info.lbnd = new int [dim]; - arrdata[group].info.ubnd = new int [dim]; - arrdata[group].info.bbox = new int [2*dim]; - arrdata[group].info.nghostzones = new int [dim]; - - arrdata[group].data.resize(CCTK_NumVarsInGroupI(group)); - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - arrdata[group].data[var] = 0; - } + // Initialise group information + groupdata[group].info.dim = gp.dim; + groupdata[group].info.gsh = new int [dim]; + groupdata[group].info.lsh = new int [dim]; + groupdata[group].info.lbnd = new int [dim]; + groupdata[group].info.ubnd = new int [dim]; + groupdata[group].info.bbox = new int [2*dim]; + groupdata[group].info.nghostzones = new int [dim]; - arrdata[group].do_transfer = CanTransferVariableType (cgh, group); + groupdata[group].transport_operator = GetTransportOperator (cgh, group); - } // for group - - - - // Initialise cgh - for (int d=0; d<dim; ++d) { - cgh->cctk_nghostzones[d] = dd->lghosts[d]; - } - for (int group=0; group<CCTK_NumGroups(); ++group) { - for (int d=0; d<dim; ++d) { - ((int*)arrdata[group].info.nghostzones)[d] = arrdata[group].dd->lghosts[d]; - } - } - - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_GroupTypeI(group) != CCTK_GF) { + // Initialise group variables + for (int m=0; m<arrdata[group].size(); ++m) { - const int rl = 0; - const int ml = 0; - const int c = CCTK_MyProc(cgh); - - const bbox<int,dim>& base = arrdata[group].hh->baseextent; - const vect<vect<bool,2>,dim>& obnds = arrdata[group].hh->outer_boundaries[rl][c]; - const bbox<int,dim>& ext = arrdata[group].dd->boxes[rl][c][ml].exterior; - - for (int d=0; d<dim; ++d) { - ((int*)arrdata[group].info.gsh )[d] = (base.shape() / base.stride())[d]; - ((int*)arrdata[group].info.lsh )[d] = (ext.shape() / ext.stride())[d]; - ((int*)arrdata[group].info.lbnd)[d] = ((ext.lower() - baseext.lower()) / ext.stride())[d]; - ((int*)arrdata[group].info.ubnd)[d] = ((ext.upper() - baseext.lower()) / ext.stride())[d]; - ((int*)arrdata[group].info.bbox)[2*d ] = obnds[d][0]; - ((int*)arrdata[group].info.bbox)[2*d+1] = obnds[d][1]; - - assert (arrdata[group].info.lsh[d]>=0); - assert (arrdata[group].info.lsh[d]<=arrdata[group].info.gsh[d]); - if (d>=arrdata[group].info.dim) - if (! (arrdata[group].info.lsh[d]==1)) { - cout << "group " << group << " " << CCTK_GroupName(group) << " " - << "dim " << arrdata[group].info.dim << " " - << "d " << d << endl; - cout << "gsh " << arrdata[group].info.gsh[0] << " " << arrdata[group].info.gsh[1] << " " << arrdata[group].info.gsh[2] << endl; - cout << "lsh " << arrdata[group].info.lsh[0] << " " << arrdata[group].info.lsh[1] << " " << arrdata[group].info.lsh[2] << endl; - cout << "lbnd " << arrdata[group].info.lbnd[0] << " " << arrdata[group].info.lbnd[1] << " " << arrdata[group].info.lbnd[2] << endl; - } - if (d>=arrdata[group].info.dim) - assert (arrdata[group].info.lsh[d]==1); - assert (arrdata[group].info.lbnd[d]>=0); - assert (arrdata[group].info.lbnd[d]<=arrdata[group].info.ubnd[d]+1); - assert (arrdata[group].info.ubnd[d]<arrdata[group].info.gsh[d]); - assert (arrdata[group].info.lbnd[d] + arrdata[group].info.lsh[d] - 1 - == arrdata[group].info.ubnd[d]); - assert (arrdata[group].info.lbnd[d]<=arrdata[group].info.ubnd[d]+1); + arrdata[group][m].data.resize(CCTK_NumVarsInGroupI(group)); + for (int var=0; var<(int)arrdata[group][m].data.size(); ++var) { + arrdata[group][m].data[var] = 0; } - const int numvars = CCTK_NumVarsInGroupI (group); - if (numvars>0) { - const int firstvar = CCTK_FirstVarIndexI (group); - assert (firstvar>=0); - - const int num_tl = CCTK_NumTimeLevelsFromVarI (firstvar); - // We don't know when to cycle arrays or scalars - if (num_tl!=1) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Carpet does not allow grid array groups with multiple time levels. The grid array group \"%s\" has %d time levels.", - CCTK_GroupName(group), num_tl); - } - assert (num_tl==1); - - assert (rl>=0 && rl<(int)arrdata[group].dd->boxes.size()); - assert (c>=0 && c<(int)arrdata[group].dd->boxes[rl].size()); - assert (ml>=0 && ml<(int)arrdata[group].dd->boxes[rl][c].size()); - assert (arrdata[group].hh->is_local(rl,c)); - - assert (group<(int)arrdata.size()); - for (int var=0; var<numvars; ++var) { - assert (var<(int)arrdata[group].data.size()); - for (int tl=0; tl<num_tl; ++tl) { - cgh->data[firstvar+var][tl] = 0; - } - } - } - - } // if grouptype + } + } // for group - // Initialise current position - reflevel = -1; - mglevel = -1; - component = -1; + // Allocate level times + leveltimes.resize (mglevels); + for (int ml=0; ml<mglevels; ++ml) { + leveltimes[ml].resize (maxreflevels); + } // Enable prolongating do_prolongate = true; - // Set initial refinement structure - vector<bbox<int,dim> > bbs; - vector<vect<vect<bool,2>,dim> > obs; - if (strcmp(base_extents, "") == 0) { - // default: one grid component covering everything - bbs.push_back (hh->baseextent); - obs.push_back (vect<vect<bool,2>,dim>(vect<bool,2>(true))); - } else { - // explicit grid components - istringstream ext_str(base_extents); - ext_str >> bbs; - CCTK_VInfo (CCTK_THORNSTRING, "Using %d grid patches", bbs.size()); - cout << "grid-patches-are " << bbs << endl; - if (bbs.size()<=0) { - CCTK_WARN (0, "Cannot evolve with 0 grid patches"); - } - istringstream ob_str (base_outerbounds); - ob_str >> obs; - cout << "outer-boundaries-are " << obs << endl; - assert (obs.size() == bbs.size()); - } - - SplitRegions (cgh, bbs, obs); - - vector<vector<bbox<int,dim> > > bbss(1); - vector<vector<vect<vect<bool,2>,dim> > > obss(1); - bbss[0] = bbs; - obss[0] = obs; - - gh<dim>::rexts bbsss; - bbsss = hh->make_multigrid_boxes(bbss, mglevels); - - gh<dim>::rprocs pss; - MakeProcessors (cgh, bbsss, pss); - - // Recompose grid hierarchy - Checkpoint ("Recomposing grid functions"); - Recompose (cgh, bbsss, obss, pss, maxreflevels, false); - - + // Finish initialisation + mglevelfact = 1; + cgh->cctk_time = 0; + cgh->cctk_delta_time = 1.0; - // Initialise time step on coarse grid - cgh->cctk_timefac = 1; - for (int d=0; d<dim; ++d) { - cgh->cctk_levoff[d] = 0; - cgh->cctk_levoffdenom[d] = 1; - } + mglevel = 0; + reflevel = 0; + map = 0; + component = 0; - refleveltimes.resize (maxreflevels); - cgh->cctk_time = 0.0; - delta_time = 1.0; - cgh->cctk_delta_time = 1.0; + leave_local_mode (cgh); + leave_singlemap_mode (cgh); + leave_level_mode (cgh); + leave_global_mode (cgh); // Enable storage for all groups if desired if (true || enable_all_storage) { - BEGIN_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { + BEGIN_MGLEVEL_LOOP(cgh) { + BEGIN_REFLEVEL_LOOP(cgh) { for (int group=0; group<CCTK_NumGroups(); ++group) { - char * groupname = CCTK_GroupName(group); + char * const groupname = CCTK_GroupName(group); EnableGroupStorage (cgh, groupname); free (groupname); } - } END_MGLEVEL_LOOP; - } END_REFLEVEL_LOOP; + } END_REFLEVEL_LOOP; + } END_MGLEVEL_LOOP; } - Waypoint ("done with SetupGH."); + Waypoint ("Done with setting up the grid hierarchy"); // We register only once, ergo we get only one handle, ergo there // is only one grid hierarchy for us. We store that statically, diff --git a/Carpet/Carpet/src/Storage.cc b/Carpet/Carpet/src/Storage.cc index ed51a4d15..e95c8a7ec 100644 --- a/Carpet/Carpet/src/Storage.cc +++ b/Carpet/Carpet/src/Storage.cc @@ -10,7 +10,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Storage.cc,v 1.29 2004/01/16 10:57:04 hawke Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/Storage.cc,v 1.30 2004/01/25 14:57:27 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_Storage_cc); } @@ -26,16 +26,12 @@ namespace Carpet { { DECLARE_CCTK_PARAMETERS; - Checkpoint ("%*sEnableGroupStorage %s", 2*reflevel, "", groupname); + Checkpoint ("EnableGroupStorage %s", groupname); // TODO: Enabling storage for one refinement level has to enable // it for all other refinement levels as well. Disabling must // wait until all refinement levels have been disabled. - // TODO: Invent a mode "reflevel==-1" that is global, i.e. has - // effect for all refinement levels. This mode is used during - // INITIAL, and en-/disabling storage in it is also global. - const int group = CCTK_GroupIndex(groupname); assert (group>=0 && group<CCTK_NumGroups()); @@ -45,8 +41,8 @@ namespace Carpet { // No storage change in local mode if (grouptype == CCTK_GF) { - assert (reflevel == -1 - || hh->local_components(reflevel) == 1 || component == -1); + assert ((reflevel == -1 && map == -1 && mglevel == -1 && component == -1) + || (maps == 1 && vhh.at(0)->local_components(reflevel) == 1)); } if (CCTK_QueryGroupStorageI(cgh, group)) { @@ -58,19 +54,6 @@ namespace Carpet { return num_tl; } - // Check whether this group has transfer operators - if (grouptype == CCTK_GF) { - if (! arrdata[group].do_transfer) { - const int var = CCTK_FirstVarIndexI(group); - assert (var>=0); - const int vartype = CCTK_VarTypeI(var); - const char * vartypename = CCTK_VarTypeName(vartype); - CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, - "(Allocating storage for Cactus group \"%s\".) Note: This group (which has the variable type %s) will be neither prolongated nor restricted.", - groupname, vartypename); - } - } - // There is a difference between the Cactus time levels and the // Carpet time levels. If there are n time levels, then the // Cactus time levels are numbered 0 ... n-1, with the current @@ -82,49 +65,52 @@ namespace Carpet { assert (num_tl>0); const int tmin = 1 - num_tl; const int tmax = 0; - const int my_prolongation_order_time - = num_tl==1 ? 0 : prolongation_order_time; if (grouptype == CCTK_GF) { if (max_refinement_levels > 1) { - if (num_tl <= my_prolongation_order_time) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "There are not enough time levels for the desired temporal prolongation order in the grid function group \"%s\". With Carpet::prolongation_order_time=%d, you need at least %d time levels.", - CCTK_GroupName(group), - prolongation_order_time, prolongation_order_time+1); + if (groupdata.at(group).transport_operator != op_none) { + if (num_tl <= prolongation_order_time) { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "There are not enough time levels for the desired temporal prolongation order in the grid function group \"%s\". With Carpet::prolongation_order_time=%d, you need at least %d time levels.", + CCTK_GroupName(group), + prolongation_order_time, prolongation_order_time+1); + } } } } // VGF: allocate - assert (arrdata[group].data.size()==0 - || arrdata[group].data[0] == 0); - assert ((int)arrdata[group].data.size() == CCTK_NumVarsInGroupI(group)); - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - const int n = n0 + var; - switch (CCTK_VarTypeI(n)) { -#define TYPECASE(N,T) \ - case N: \ - assert (! arrdata[group].data[var]); \ - /* VGF */ \ - arrdata[group].data[var] = new gf<T,dim> \ - (n, *arrdata[group].tt, *arrdata[group].dd, \ - tmin, tmax, my_prolongation_order_time); \ - break; + assert (arrdata.at(group).at(0).data.size()==0 + || arrdata.at(group).at(0).data.at(0) == 0); + for (int m=0; m<maps; ++m) { + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + const int n = n0 + var; + switch (CCTK_VarTypeI(n)) { +#define TYPECASE(N,T) \ + case N: \ + /* VGF */ \ + arrdata.at(group).at(m).data.at(var) = new gf<T,dim> \ + (n, groupdata.at(group).transport_operator, \ + *arrdata.at(group).at(m).tt, *arrdata.at(group).at(m).dd, \ + tmin, tmax, prolongation_order_time); \ + break; #include "typecase" #undef TYPECASE - default: - UnsupportedVarType(n); - } // switch - - if (grouptype != CCTK_GF) { - for (int tl=0; tl<num_tl; ++tl) { - int const c = CCTK_MyProc(cgh); - cgh->data[n][tl] = ((*arrdata[group].data[var]) (-tl, 0, c, 0)->storage()); + default: + UnsupportedVarType(n); + } // switch + + if (grouptype != CCTK_GF) { + for (int tl=0; tl<num_tl; ++tl) { + assert (m == 0); + int const c = CCTK_MyProc(cgh); + cgh->data[n][tl] = ((*arrdata.at(group).at(m).data.at(var)) + (-tl, 0, c, 0)->storage()); + } } - } - + + } // for } // for // PoisonGroup (cgh, group, alltimes); @@ -137,7 +123,7 @@ namespace Carpet { int DisableGroupStorage (const cGH* cgh, const char* groupname) { - Checkpoint ("%*sDisableGroupStorage %s", 2*reflevel, "", groupname); + Checkpoint ("DisableGroupStorage %s", groupname); const int group = CCTK_GroupIndex(groupname); assert (group>=0 && group<CCTK_NumGroups()); @@ -159,34 +145,39 @@ namespace Carpet { const int num_tl = CCTK_NumTimeLevelsFromVarI(n0); assert (num_tl>0); - assert (arrdata[group].data.size()); - assert (arrdata[group].data[0]); - for (int var=0; var<(int)arrdata[group].data.size(); ++var) { - const int n = n0 + var; - switch (CCTK_VarTypeI(n)) { -#define TYPECASE(N,T) \ - case N: \ - assert (arrdata[group].data[var]); \ - delete (gf<T,dim>*)arrdata[group].data[var]; \ - arrdata[group].data[var] = 0; \ - break; + assert (arrdata.at(group).at(0).data.size()); + assert (arrdata.at(group).at(0).data.at(0)); + for (int m=0; m<maps; ++m) { + assert (m<(int)arrdata.at(group).size()); + for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) { + assert (var<(int)arrdata.at(group).at(m).data.size()); + const int n = n0 + var; + switch (CCTK_VarTypeI(n)) { +#define TYPECASE(N,T) \ + case N: \ + assert (arrdata.at(group).at(m).data.at(var)); \ + delete (gf<T,dim>*)arrdata.at(group).at(m).data.at(var); \ + arrdata.at(group).at(m).data.at(var) = 0; \ + break; #include "typecase" #undef TYPECASE - default: - UnsupportedVarType(n); - } // switch - arrdata[group].data[var] = 0; - - if (CCTK_GroupTypeI(group) != CCTK_GF) { - for (int tl=0; tl<num_tl; ++tl) { - cgh->data[n][tl] = 0; + default: + UnsupportedVarType(n); + } // switch + arrdata.at(group).at(m).data.at(var) = 0; + + if (CCTK_GroupTypeI(group) != CCTK_GF) { + for (int tl=0; tl<num_tl; ++tl) { + cgh->data[n][tl] = 0; + } } - } + + } // for } // for // VGF: free - // storage was not disabled previously + // storage was enabled previously return 1; } @@ -206,8 +197,8 @@ namespace Carpet { const int var = 0; assert (group<(int)arrdata.size()); - assert (var<(int)arrdata[group].data.size()); - return arrdata[group].data[var] != 0; + assert (var<(int)arrdata.at(group).at(0).data.size()); + return arrdata.at(group).at(0).data.at(var) != 0; } @@ -216,22 +207,29 @@ namespace Carpet { const char* groupname) { static const int zero = 0; + static const int error = -1; if (groupname) { group = CCTK_GroupIndex(groupname); } assert (group>=0 && group<CCTK_NumGroups()); - const int gpdim = arrdata[group].info.dim; + if (mglevel == -1) { + return &error; + } + + const int gptype = CCTK_GroupTypeI (group); + if (gptype == CCTK_GF && map == -1) { + return &error; + } + + const int gpdim = groupdata.at(group).info.dim; assert (dir>=0 && dir<gpdim); if (CCTK_QueryGroupStorageI(cgh, group)) { - const int var = CCTK_FirstVarIndexI(group); - assert (var>=0 && var<CCTK_NumVars()); - - assert (group<(int)arrdata.size()); - return &arrdata[group].info.lsh[dir]; + assert (group>=0 && group<(int)arrdata.size()); + return &groupdata.at(group).info.lsh[dir]; } else { @@ -246,7 +244,8 @@ namespace Carpet { int GroupDynamicData (const cGH* cgh, int group, cGroupDynamicData* data) { assert (group>=0 && group<CCTK_NumGroups()); - *data = arrdata[group].info; + assert (group>=0 && group<(int)arrdata.size()); + *data = groupdata.at(group).info; return 0; } diff --git a/Carpet/Carpet/src/carpet.hh b/Carpet/Carpet/src/carpet.hh index 9d50f0eb0..034d02616 100644 --- a/Carpet/Carpet/src/carpet.hh +++ b/Carpet/Carpet/src/carpet.hh @@ -1,8 +1,14 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet.hh,v 1.26 2003/11/05 16:18:37 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet.hh,v 1.27 2004/01/25 14:57:27 schnetter Exp $ #ifndef CARPET_HH #define CARPET_HH +#include <vector> + +#include "cctk.h" +#include "cctk_Arguments.h" +#include "cctk_Functions.h" + #include "gh.hh" #include "carpet_public.hh" @@ -11,19 +17,30 @@ namespace Carpet { - void Regrid (const cGH* cgh, + using namespace std; + + // Scheduled functions + extern "C" { + void CarpetParamCheck (CCTK_ARGUMENTS); + void CarpetStartup (void); + } + + // Registered functions + void* SetupGH (tFleshConfig* fc, int convLevel, cGH* cgh); + + int Initialise (tFleshConfig* config); + int Evolve (tFleshConfig* config); + int Shutdown (tFleshConfig* config); + int CallFunction (void* function, cFunctionData* attribute, void* data); + + // Other functions + void Regrid (const cGH* cgh, const int rl, const int initialise_from, const bool do_prolongate); void CycleTimeLevels (const cGH* cgh); void FlipTimeLevels (const cGH* cgh); void Restrict (const cGH* cgh); - void Recompose (const cGH* cgh, - const gh<dim>::rexts& bbsss, - const gh<dim>::rbnds& obss, - const gh<dim>::rprocs& pss, - const int initialise_from, - const bool do_prolongate); - + // Sanity checks enum checktimes { currenttime, currenttimebutnotifonly, previoustime, @@ -42,6 +59,7 @@ namespace Carpet { void CheckChecksums (const cGH* cgh, checktimes where); // Debugging output + void Output (const char* fmt, ...); void Waypoint (const char* fmt, ...); void Checkpoint (const char* fmt, ...); diff --git a/Carpet/Carpet/src/carpet_public.h b/Carpet/Carpet/src/carpet_public.h index 3ee6261f1..0a91d7c12 100644 --- a/Carpet/Carpet/src/carpet_public.h +++ b/Carpet/Carpet/src/carpet_public.h @@ -1,11 +1,11 @@ -/* $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet_public.h,v 1.11 2003/09/19 16:04:31 schnetter Exp $ */ +/* $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/carpet_public.h,v 1.12 2004/01/25 14:57:27 schnetter Exp $ */ #ifndef CARPET_PUBLIC_H #define CARPET_PUBLIC_H #include <mpi.h> -#include "cctk_Arguments.h" +#include "cctk.h" @@ -19,12 +19,6 @@ namespace Carpet { extern "C" { #endif - /* Scheduled functions */ - void CarpetParamCheck (CCTK_ARGUMENTS); - void CarpetStartup (void); - - - /* Prolongation management */ int CarpetEnableProlongating (const int flag); @@ -36,10 +30,14 @@ namespace Carpet { /* Call a local function */ int CallLocalFunction (cGH * const cgh, void (* const function) (cGH * const cgh)); + int CallSinglemapFunction (cGH * const cgh, + void (* const function) (cGH * const cgh)); int CallLevelFunction (cGH * const cgh, void (* const function) (cGH * const cgh)); int CallGlobalFunction (cGH * const cgh, void (* const function) (cGH * const cgh)); + int CallMetaFunction (cGH * const cgh, + void (* const function) (cGH * const cgh)); diff --git a/Carpet/Carpet/src/defines.hh b/Carpet/Carpet/src/defines.hh index 006d925f3..4cac30a8a 100644 --- a/Carpet/Carpet/src/defines.hh +++ b/Carpet/Carpet/src/defines.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/defines.hh,v 1.2 2004/03/02 10:29:54 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/defines.hh,v 1.1 2004/01/25 14:57:28 schnetter Exp $ #ifndef DEFINES_HH #define DEFINES_HH @@ -6,7 +6,6 @@ #include "cctk.h" #include <bbox.hh> -#include <bboxset.hh> #include <vect.hh> @@ -22,7 +21,6 @@ namespace Carpet { typedef bbox<int,dim> ibbox; typedef bbox<CCTK_INT,dim> jbbox; typedef bbox<CCTK_REAL,dim> rbbox; - typedef bboxset<int,dim> ibset; typedef vect<vect<bool,2>,dim> bbvect; typedef vect<vect<int,2>,dim> iivect; diff --git a/Carpet/Carpet/src/functions.hh b/Carpet/Carpet/src/functions.hh index 56170dbed..18bcaf637 100644 --- a/Carpet/Carpet/src/functions.hh +++ b/Carpet/Carpet/src/functions.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/functions.hh,v 1.4 2004/04/18 13:29:43 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/functions.hh,v 1.1 2004/01/25 14:57:28 schnetter Exp $ #ifndef FUNCTIONS_HH #define FUNCTIONS_HH @@ -34,21 +34,6 @@ namespace Carpet { const char* groupname); int QueryGroupStorageB (const cGH* cgh, int group, const char* groupname); int GroupDynamicData (const cGH* cgh, int group, cGroupDynamicData* data); - - - - // Helpers for recomposing the grid hierarchy - void CheckRegions (const gh<dim>::rexts & bbsss, - const gh<dim>::rbnds & obss, - const gh<dim>::rprocs& pss); - - void OutputGrids (const cGH* cgh, const int m, const gh<dim>& hh); - - void OutputGridStructure (const cGH *cgh, - const int m, - const gh<dim>::rexts & bbsss, - const gh<dim>::rbnds & obss, - const gh<dim>::rprocs& pss); @@ -64,6 +49,11 @@ namespace Carpet { vector<bbvect>& obs, vector<int>& ps); void MakeMultigridBoxes (const cGH* cgh, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, vector<ibbox> const & bbs, vector<bbvect> const & obs, vector<vector<ibbox> > & bbss); diff --git a/Carpet/Carpet/src/helpers.cc b/Carpet/Carpet/src/helpers.cc index 2857c53ae..2f24910f0 100644 --- a/Carpet/Carpet/src/helpers.cc +++ b/Carpet/Carpet/src/helpers.cc @@ -1,6 +1,8 @@ #include <assert.h> #include <stdarg.h> +#include <stdio.h> #include <stdlib.h> +#include <string.h> #include <mpi.h> @@ -15,7 +17,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/helpers.cc,v 1.44 2003/11/19 14:06:07 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/helpers.cc,v 1.45 2004/01/25 14:57:28 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_helpers_cc); } @@ -27,6 +29,23 @@ namespace Carpet { + // Enable or disable prolongating + int CarpetEnableProlongating (const int flag) + { + assert (flag==0 || flag==1); + do_prolongate = flag; + if (do_prolongate) { + Checkpoint ("Prolongating enabled"); + } else { + Checkpoint ("Prolongating disabled"); + } + return 0; + } + + + + // Communication + int Barrier (const cGH* cgh) { MPI_Barrier (dist::comm); @@ -79,6 +98,8 @@ namespace Carpet { + // Datatypes + MPI_Datatype CarpetMPIDatatype (const int vartype) { switch (vartype) { @@ -99,6 +120,8 @@ namespace Carpet { + // Timelevels + int mintl (const checktimes where, const int num_tl) { assert (num_tl>0); @@ -147,14 +170,51 @@ namespace Carpet { + // Diagnostic output + + static void prepend_id (char* const msg, size_t const msglen) + { + if (mglevel!=-1) { + snprintf (msg+strlen(msg), msglen-strlen(msg), "[%d]", mglevel); + if (reflevel!=-1) { + snprintf (msg+strlen(msg), msglen-strlen(msg), "[%d]", reflevel); + if (map!=-1) { + snprintf (msg+strlen(msg), msglen-strlen(msg), "[%d]", map); + if (component!=-1) { + snprintf (msg+strlen(msg), msglen-strlen(msg), "[%d]", component); + } + } + } + snprintf (msg+strlen(msg), msglen-strlen(msg), " "); + } + } + + void Output (const char* fmt, ...) + { + DECLARE_CCTK_PARAMETERS; + va_list args; + char msg[1000]; + snprintf (msg, sizeof msg, ""); + prepend_id (msg + strlen(msg), sizeof msg - strlen(msg)); + va_start (args, fmt); + vsnprintf (msg + strlen(msg), sizeof msg - strlen(msg), fmt, args); + va_end (args); + CCTK_INFO (msg); + if (barriers) { + MPI_Barrier (dist::comm); + } + } + void Waypoint (const char* fmt, ...) { DECLARE_CCTK_PARAMETERS; if (verbose || veryverbose) { va_list args; char msg[1000]; + snprintf (msg, sizeof msg, ""); + prepend_id (msg + strlen(msg), sizeof msg - strlen(msg)); va_start (args, fmt); - vsnprintf (msg, sizeof msg, fmt, args); + vsnprintf (msg + strlen(msg), sizeof msg - strlen(msg), fmt, args); va_end (args); CCTK_INFO (msg); } @@ -169,8 +229,10 @@ namespace Carpet { if (veryverbose) { va_list args; char msg[1000]; + snprintf (msg, sizeof msg, ""); + prepend_id (msg + strlen(msg), sizeof msg - strlen(msg)); va_start (args, fmt); - vsnprintf (msg, sizeof msg, fmt, args); + vsnprintf (msg + strlen(msg), sizeof msg - strlen(msg), fmt, args); va_end (args); CCTK_INFO (msg); } @@ -191,327 +253,4 @@ namespace Carpet { "or change the type of this variable.", CCTK_FullName(vindex)); } - - - void set_reflevel (cGH* cgh, const int rl) - { - DECLARE_CCTK_PARAMETERS; - - // Check - assert (rl==-1 || (rl>=0 && rl<maxreflevels && rl<maxreflevels)); - assert (mglevel == -1); - assert (component == -1); - - // Save - if (reflevel != -1) { - refleveltimes[reflevel] = cgh->cctk_time; - delta_time = cgh->cctk_delta_time; - } - - // Change - reflevel = rl; - if (reflevel == -1) { - // global mode - reflevelfact = 0; - cgh->cctk_time = (cctk_initial_time - + cgh->cctk_iteration * delta_time / maxreflevelfact); - } else { - // level mode or local mode - reflevelfact = ipow(reffact, reflevel); - cgh->cctk_time = refleveltimes[reflevel]; - } - vect<int,dim>::ref(cgh->cctk_levfac) = reflevelfact; - } - - - - void set_mglevel (cGH* cgh, const int ml) - { - // Check - assert (ml==-1 || (ml>=0 && ml<mglevels)); - assert (reflevel>=0 && reflevel<hh->reflevels()); - assert (component == -1); - - // Change - mglevel = ml; - mglevelfact = ipow(mgfact, mglevel); - cgh->cctk_convlevel = mglevel; - - // Set gsh - if (mglevel == -1) { - - mglevelfact = 0xdead; - cgh->cctk_convlevel = 0xdead; - - cgh->cctk_timefac = 0; - for (int d=0; d<dim; ++d) { - cgh->cctk_levoff[d] = 0xdead; - cgh->cctk_levoffdenom[d] = 0xdead; - } - - vect<int,dim>::ref(cgh->cctk_gsh) = 0xdead; - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_GroupTypeI(group) == CCTK_GF) { - vect<int,dim>::ref((int*)arrdata[group].info.gsh) = 0xdead; - } - } - - } else { - - mglevelfact = ipow(mgfact, mglevel); - cgh->cctk_convlevel = mglevel; - - const bbox<int,dim>& baseext = dd->bases[reflevel][mglevel].exterior; - - assert (mglevelfact==1); - cgh->cctk_timefac = reflevelfact / mglevelfact; - for (int d=0; d<dim; ++d) { - assert (baseext.lower()[d] % (maxreflevelfact/reflevelfact) == 0); - cgh->cctk_levoff[d] = baseext.lower()[d] / (maxreflevelfact/reflevelfact); - cgh->cctk_levoffdenom[d] = 1; - } - - vect<int,dim>::ref(cgh->cctk_gsh) = baseext.shape() / baseext.stride(); - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_GroupTypeI(group) == CCTK_GF) { - const bbox<int,dim>& baseext = arrdata[group].dd->bases[reflevel][mglevel].exterior; - for (int d=0; d<dim; ++d) { - ((int*)arrdata[group].info.gsh)[d] = (baseext.shape() / baseext.stride())[d]; - } - } - } - - } // if mglevel != -1 - } - - - - void set_component (cGH* cgh, const int c) - { - assert (reflevel>=0 && reflevel<hh->reflevels()); - assert (c==-1 || (c>=0 && c<hh->components(reflevel))); - component = c; - assert (component==-1 - || (mglevel>=0 && mglevel<hh->mglevels(reflevel,component))); - - // Set Cactus parameters - if (component == -1 || ! hh->is_local(reflevel,component)) { - // Level mode -- no component is active - - for (int d=0; d<dim; ++d) { - cgh->cctk_lsh[d] = 0xdead; - cgh->cctk_bbox[2*d ] = 0xdead; - cgh->cctk_bbox[2*d+1] = 0xdead; - cgh->cctk_lbnd[d] = -0xdead; - cgh->cctk_ubnd[d] = 0xdead; - for (int stg=0; stg<CCTK_NSTAGGER; ++stg) { - cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = 0xdead; - } - } - - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_GroupTypeI(group) == CCTK_GF) { - - for (int d=0; d<dim; ++d) { - ((int*)arrdata[group].info.lsh)[d] = 0xdead; - ((int*)arrdata[group].info.bbox)[2*d ] = 0xdead; - ((int*)arrdata[group].info.bbox)[2*d+1] = 0xdead; - ((int*)arrdata[group].info.lbnd)[d] = -0xdead; - ((int*)arrdata[group].info.ubnd)[d] = 0xdead; - } - - const int numvars = CCTK_NumVarsInGroupI (group); - if (numvars>0) { - const int firstvar = CCTK_FirstVarIndexI (group); - assert (firstvar>=0); - const int num_tl = CCTK_NumTimeLevelsFromVarI (firstvar); - - assert (group<(int)arrdata.size()); - for (int var=0; var<numvars; ++var) { - assert (var<(int)arrdata[group].data.size()); - for (int tl=0; tl<num_tl; ++tl) { - cgh->data[firstvar+var][tl] = 0; - } - } - } - - } // if grouptype - } // for var - - } else { - // Local mode - - assert (reflevel>=0 && reflevel < (int)dd->boxes.size()); - assert (component>=0 && component < (int)dd->boxes[reflevel].size()); - assert (mglevel>=0 && mglevel < (int)dd->boxes[reflevel][component].size()); - - const bbox<int,dim>& baseext = dd->bases[reflevel][mglevel].exterior; - const vect<vect<bool,2>,dim>& obnds = hh->outer_boundaries[reflevel][component]; - const bbox<int,dim>& ext = dd->boxes[reflevel][component][mglevel].exterior; - - for (int d=0; d<dim; ++d) { - - cgh->cctk_lsh[d] = (ext.shape() / ext.stride())[d]; - cgh->cctk_lbnd[d] = ((ext.lower() - baseext.lower()) / ext.stride())[d]; - cgh->cctk_ubnd[d] = ((ext.upper() - baseext.lower()) / ext.stride())[d]; - cgh->cctk_bbox[2*d ] = obnds[d][0]; - cgh->cctk_bbox[2*d+1] = obnds[d][1]; - for (int stg=0; stg<CCTK_NSTAGGER; ++stg) { - // TODO: support staggering - cgh->cctk_lssh[CCTK_LSSH_IDX(stg,d)] = cgh->cctk_lsh[d]; - } - - assert (cgh->cctk_lsh[d] >= 0); - assert (cgh->cctk_lsh[d] <= cgh->cctk_gsh[d]); - assert (cgh->cctk_lbnd[d] >= 0); - assert (cgh->cctk_lbnd[d] <= cgh->cctk_ubnd[d] + 1); - assert (cgh->cctk_ubnd[d] < cgh->cctk_gsh[d]); - assert (cgh->cctk_lbnd[d] + cgh->cctk_lsh[d] - 1 == cgh->cctk_ubnd[d]); - assert (cgh->cctk_lbnd[d] <= cgh->cctk_ubnd[d]+1); - - } - - for (int group=0; group<CCTK_NumGroups(); ++group) { - if (CCTK_GroupTypeI(group) == CCTK_GF) { - - assert (reflevel>=0 && reflevel < (int)arrdata[group].dd->boxes.size()); - assert (component>=0 && component < (int)arrdata[group].dd->boxes[reflevel].size()); - assert (mglevel>=0 && mglevel < (int)arrdata[group].dd->boxes[reflevel][component].size()); - - const bbox<int,dim>& baseext = arrdata[group].dd->bases[reflevel][mglevel].exterior; - const vect<vect<bool,2>,dim>& obnds = arrdata[group].hh->outer_boundaries[reflevel][component]; - const bbox<int,dim>& ext = arrdata[group].dd->boxes[reflevel][component][mglevel].exterior; - - for (int d=0; d<dim; ++d) { - - ((int*)arrdata[group].info.lsh )[d] = (ext.shape() / ext.stride())[d]; - ((int*)arrdata[group].info.lbnd)[d] = ((ext.lower() - baseext.lower()) / ext.stride())[d]; - ((int*)arrdata[group].info.ubnd)[d] = ((ext.upper() - baseext.lower()) / ext.stride())[d]; - ((int*)arrdata[group].info.bbox)[2*d ] = obnds[d][0]; - ((int*)arrdata[group].info.bbox)[2*d+1] = obnds[d][1]; - - assert (arrdata[group].info.lsh[d]>=0); - assert (arrdata[group].info.lsh[d]<=arrdata[group].info.gsh[d]); - if (d>=arrdata[group].info.dim) - assert (arrdata[group].info.lsh[d]==1); - assert (arrdata[group].info.lbnd[d]>=0); - assert (arrdata[group].info.lbnd[d]<=arrdata[group].info.ubnd[d]+1); - assert (arrdata[group].info.ubnd[d]<arrdata[group].info.gsh[d]); - assert (arrdata[group].info.lbnd[d] + arrdata[group].info.lsh[d] - 1 - == arrdata[group].info.ubnd[d]); - assert (arrdata[group].info.lbnd[d]<=arrdata[group].info.ubnd[d]+1); - } - - const int numvars = CCTK_NumVarsInGroupI (group); - if (numvars>0) { - const int firstvar = CCTK_FirstVarIndexI (group); - assert (firstvar>=0); - const int num_tl = CCTK_NumTimeLevelsFromVarI (firstvar); - - assert (hh->is_local(reflevel,component)); - - assert (group<(int)arrdata.size()); - for (int var=0; var<numvars; ++var) { - assert (var<(int)arrdata[group].data.size()); - for (int tl=0; tl<num_tl; ++tl) { - ggf<dim> * const ff = arrdata[group].data[var]; - if (ff) { - cgh->data[firstvar+var][tl] - = (*ff) (-tl, reflevel, component, mglevel)->storage(); - } else { - cgh->data[firstvar+var][tl] = 0; - } - } - } - } - - } // if grouptype - } // for group - - } // if local mode - - } - - - - // Enable or disable prolongating - int CarpetEnableProlongating (const int flag) - { - assert (flag==0 || flag==1); - do_prolongate = flag; - if (do_prolongate) { - Checkpoint ("Prolongating enabled"); - } else { - Checkpoint ("Prolongating disabled"); - } - return 0; - } - - - - // This is a temporary measure to call a schedule group from a level - // mode function. - int CallScheduleGroup (cGH * const cgh, const char * const group) - { - assert (reflevel != -1); - assert (component == -1); - CCTK_ScheduleTraverse (group, cgh, CallFunction); - return 0; - } - - extern "C" void CCTK_FCALL CCTK_FNAME(CallScheduleGroup) - (int * const ierr, cGH * const * const cgh, ONE_FORTSTRING_ARG) - { - ONE_FORTSTRING_CREATE (group); - *ierr = CallScheduleGroup (*cgh, group); - free (group); - } - - - - // This is a temporary measure to call a local mode function from a - // global mode or level mode function. A more elegant way would be - // to reuse the CallFunction stuff, or function aliasing. Is there - // a way for the user to get at the cFunctionData structure? - int CallLocalFunction (cGH * const cgh, - void (* const function) (cGH * const cgh)) - { - if (reflevel == -1) { - // we are in global mode - BEGIN_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - function (cgh); - } END_LOCAL_COMPONENT_LOOP; - } END_MGLEVEL_LOOP; - } END_REFLEVEL_LOOP; - } else { - // we are in level mode - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - function (cgh); - } END_LOCAL_COMPONENT_LOOP; - } - return 0; - } - - int CallLevelFunction (cGH * const cgh, - void (* const function) (cGH * const cgh)) - { - assert (reflevel == -1); - BEGIN_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - function (cgh); - } END_MGLEVEL_LOOP; - } END_REFLEVEL_LOOP; - return 0; - } - - int CallGlobalFunction (cGH * const cgh, - void (* const function) (cGH * const cgh)) - { - assert (reflevel == -1); - function (cgh); - return 0; - } - } // namespace Carpet diff --git a/Carpet/Carpet/src/modes.cc b/Carpet/Carpet/src/modes.cc index e831b2dd8..891ba2085 100644 --- a/Carpet/Carpet/src/modes.cc +++ b/Carpet/Carpet/src/modes.cc @@ -12,7 +12,7 @@ #include "carpet.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/modes.cc,v 1.9 2004/08/07 20:07:27 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/modes.cc,v 1.1 2004/01/25 14:57:28 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_modes_cc); } @@ -57,7 +57,7 @@ namespace Carpet { // assert (mglevel>=0 && mglevel<mglevels); // assert (reflevel>=0 && reflevel<reflevels); // assert (map>=0 && map<maps); -// assert (vhh.at(map)->local_components(reflevel)==1 || component==-1); +// assert (vhh[map]->local_components(reflevel)==1 || component==-1); } @@ -76,55 +76,51 @@ namespace Carpet { mglevel = ml; mglevelfact = ipow(mgfact, mglevel); - // TODO: this could also just be "mglevel" instead cgh->cctk_convlevel = basemglevel + mglevel; - // Set time and space delta + // Set time delta cgh->cctk_delta_time = delta_time * mglevelfact; - for (int d=0; d<dim; ++d) { - cgh->cctk_origin_space[d] = origin_space.at(mglevel)[d]; - cgh->cctk_delta_space[d] = delta_space[d] * mglevelfact; - } // Set array information for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) != CCTK_GF) { + const int ml = mglevel; const int rl = 0; const int m = 0; const int c = CCTK_MyProc(cgh); - const ibbox& base = arrdata.at(group).at(m).hh->bases.at(rl).at(ml); - const bbvect& obnds = arrdata.at(group).at(m).hh->outer_boundaries.at(rl).at(c); - const ibbox& ext = arrdata.at(group).at(m).dd->boxes.at(rl).at(c).at(ml).exterior; + const ibbox& base = arrdata[group][m].hh->bases[rl][ml]; + const bbvect& obnds = arrdata[group][m].hh->outer_boundaries[rl][c]; + const ibbox& ext = arrdata[group][m].dd->boxes[rl][c][ml].exterior; - ivect::ref(const_cast<int*>(groupdata.at(group).info.nghostzones)) - = arrdata.at(group).at(m).dd->lghosts; - ivect::ref(const_cast<int*>(groupdata.at(group).info.gsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.nghostzones)) + = arrdata[group][m].dd->lghosts; + ivect::ref(const_cast<int*>(groupdata[group].info.gsh)) = base.shape() / base.stride(); - ivect::ref(const_cast<int*>(groupdata.at(group).info.lsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.lsh)) = ext.shape() / ext.stride(); - ivect::ref(const_cast<int*>(groupdata.at(group).info.lbnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.lbnd)) = (ext.lower() - base.lower()) / ext.stride(); - ivect::ref(const_cast<int*>(groupdata.at(group).info.ubnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.ubnd)) = (ext.upper() - base.lower()) / ext.stride(); for (int d=0; d<dim; ++d) { - const_cast<int*>(groupdata.at(group).info.bbox)[2*d ] = obnds[d][0]; - const_cast<int*>(groupdata.at(group).info.bbox)[2*d+1] = obnds[d][1]; + const_cast<int*>(groupdata[group].info.bbox)[2*d ] = obnds[d][0]; + const_cast<int*>(groupdata[group].info.bbox)[2*d+1] = obnds[d][1]; } for (int d=0; d<dim; ++d) { - assert (groupdata.at(group).info.lsh[d]>=0); - assert (groupdata.at(group).info.lsh[d]<=groupdata.at(group).info.gsh[d]); - if (d>=groupdata.at(group).info.dim) { - assert (groupdata.at(group).info.lsh[d]==1); + assert (groupdata[group].info.lsh[d]>=0); + assert (groupdata[group].info.lsh[d]<=groupdata[group].info.gsh[d]); + if (d>=groupdata[group].info.dim) { + assert (groupdata[group].info.lsh[d]==1); } - assert (groupdata.at(group).info.lbnd[d]>=0); - assert (groupdata.at(group).info.lbnd[d]<=groupdata.at(group).info.ubnd[d]+1); - assert (groupdata.at(group).info.ubnd[d]<groupdata.at(group).info.gsh[d]); - assert (groupdata.at(group).info.lbnd[d] + groupdata.at(group).info.lsh[d] - 1 - == groupdata.at(group).info.ubnd[d]); - assert (groupdata.at(group).info.lbnd[d]<=groupdata.at(group).info.ubnd[d]+1); + assert (groupdata[group].info.lbnd[d]>=0); + assert (groupdata[group].info.lbnd[d]<=groupdata[group].info.ubnd[d]+1); + assert (groupdata[group].info.ubnd[d]<groupdata[group].info.gsh[d]); + assert (groupdata[group].info.lbnd[d] + groupdata[group].info.lsh[d] - 1 + == groupdata[group].info.ubnd[d]); + assert (groupdata[group].info.lbnd[d]<=groupdata[group].info.ubnd[d]+1); } const int numvars = CCTK_NumVarsInGroupI (group); @@ -133,16 +129,21 @@ namespace Carpet { assert (firstvar>=0); const int num_tl = CCTK_NumTimeLevelsFromVarI (firstvar); - assert (arrdata.at(group).at(m).hh->is_local(rl,c)); + assert (rl>=0 && rl<(int)arrdata[group][m].dd->boxes.size()); + assert (c>=0 && c<(int)arrdata[group][m].dd->boxes[rl].size()); + assert (ml>=0 && ml<(int)arrdata[group][m].dd->boxes[rl][c].size()); + assert (arrdata[group][m].hh->is_local(rl,c)); + assert (group<(int)arrdata.size()); for (int var=0; var<numvars; ++var) { - assert (firstvar+var<CCTK_NumVars()); + assert (var<(int)arrdata[group][m].data.size()); for (int tl=0; tl<num_tl; ++tl) { - ggf<dim> * const ff = arrdata.at(group).at(m).data.at(var); + ggf<dim> * const ff = arrdata[group][m].data[var]; if (ff) { gdata<dim> * const data = (*ff) (-tl, rl, c, ml); assert (data); - cgh->data[firstvar+var][tl] = data->storage(); + cgh->data[firstvar+var][tl] + = (*ff) (-tl, rl, c, ml)->storage(); } else { cgh->data[firstvar+var][tl] = 0; } @@ -163,15 +164,8 @@ namespace Carpet { if (mglevel == -1) return; // early return - // Save and unset time and space delta + // Save and unset time delta delta_time = cgh->cctk_delta_time / mglevelfact; - cgh->cctk_delta_time = 0.0; - for (int d=0; d<dim; ++d) { - origin_space.at(mglevel)[d] = cgh->cctk_origin_space[d]; - delta_space[d] = cgh->cctk_delta_space[d] / mglevelfact; - cgh->cctk_origin_space[d] = -424242.0; - cgh->cctk_delta_space[d] = 0.0; - } // Set array information for (int group=0; group<CCTK_NumGroups(); ++group) { @@ -179,21 +173,21 @@ namespace Carpet { const int m = 0; -// ivect::ref(const_cast<int*>(groupdata.at(group).info.nghostzones)) +// ivect::ref(const_cast<int*>(groupdata[group].info.nghostzones)) // = 0xdead; - ivect::ref(const_cast<int*>(groupdata.at(group).info.nghostzones)) - = arrdata.at(group).at(m).dd->lghosts; - ivect::ref(const_cast<int*>(groupdata.at(group).info.gsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.nghostzones)) + = arrdata[group][m].dd->lghosts; + ivect::ref(const_cast<int*>(groupdata[group].info.gsh)) = 0xdead; - ivect::ref(const_cast<int*>(groupdata.at(group).info.lsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.lsh)) = 0xdead; - ivect::ref(const_cast<int*>(groupdata.at(group).info.lbnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.lbnd)) = -0xdead; - ivect::ref(const_cast<int*>(groupdata.at(group).info.ubnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.ubnd)) = 0xdead; for (int d=0; d<dim; ++d) { - const_cast<int*>(groupdata.at(group).info.bbox)[2*d ] = 0xdead; - const_cast<int*>(groupdata.at(group).info.bbox)[2*d+1] = 0xdead; + const_cast<int*>(groupdata[group].info.bbox)[2*d ] = 0xdead; + const_cast<int*>(groupdata[group].info.bbox)[2*d+1] = 0xdead; } const int numvars = CCTK_NumVarsInGroupI (group); @@ -204,7 +198,7 @@ namespace Carpet { assert (group<(int)arrdata.size()); for (int var=0; var<numvars; ++var) { - assert (firstvar+var<CCTK_NumVars()); + assert (var<(int)arrdata[group][m].data.size()); for (int tl=0; tl<num_tl; ++tl) { cgh->data[firstvar+var][tl] = 0; } @@ -237,9 +231,9 @@ namespace Carpet { cgh->cctk_timefac = reflevelfact; // Set current time - assert (mglevel>=0 && mglevel<(int)leveltimes.size()); - assert (reflevel>=0 && reflevel<(int)leveltimes.at(mglevel).size()); - cgh->cctk_time = leveltimes.at(mglevel).at(reflevel); + assert (mglevel>=0 && mglevel<leveltimes.size()); + assert (reflevel>=0 && reflevel<leveltimes[mglevel].size()); + cgh->cctk_time = leveltimes[mglevel][reflevel]; assert (is_level_mode()); } @@ -252,9 +246,9 @@ namespace Carpet { if (reflevel == -1) return; // early return // Save and unset current time - assert (mglevel>=0 && mglevel<(int)leveltimes.size()); - assert (reflevel>=0 && reflevel<(int)leveltimes.at(mglevel).size()); - leveltimes.at(mglevel).at(reflevel) = cgh->cctk_time; + assert (mglevel>=0 && mglevel<leveltimes.size()); + assert (reflevel>=0 && reflevel<leveltimes[mglevel].size()); + leveltimes[mglevel][reflevel] = cgh->cctk_time; cgh->cctk_time = global_time; reflevel = -1; @@ -275,24 +269,25 @@ namespace Carpet { assert (m>=0 && m<maps); Checkpoint ("Entering singlemap mode"); - carpetGH.map = map = m; + map = m; +// TODO: disable temporarily +// cgh->cctk_map = map; + assert (map==0); // Set grid shape - const ibbox& coarseext = vdd.at(map)->bases.at(0 ).at(mglevel).exterior; - const ibbox& baseext = vdd.at(map)->bases.at(reflevel).at(mglevel).exterior; + const ibbox& baseext = vdd[map]->bases[reflevel][mglevel].exterior; assert (all (baseext.lower() % baseext.stride() == 0)); - assert (all ((baseext.lower() - coarseext.lower()) % baseext.stride() == 0)); - ivect::ref(cgh->cctk_levoff) = (baseext.lower() - coarseext.lower()) / baseext.stride(); + ivect::ref(cgh->cctk_levoff) = baseext.lower() / baseext.stride(); ivect::ref(cgh->cctk_levoffdenom) = 1; ivect::ref(cgh->cctk_gsh) = baseext.shape() / baseext.stride(); - assert (all (vdd.at(map)->lghosts == vdd.at(map)->ughosts)); - ivect::ref(cgh->cctk_nghostzones) = vdd.at(map)->lghosts; + assert (all (vdd[map]->lghosts == vdd[map]->ughosts)); + ivect::ref(cgh->cctk_nghostzones) = vdd[map]->lghosts; for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { - ivect::ref(const_cast<int*>(groupdata.at(group).info.gsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.gsh)) = ivect::ref(cgh->cctk_gsh); - ivect::ref(const_cast<int*>(groupdata.at(group).info.nghostzones)) + ivect::ref(const_cast<int*>(groupdata[group].info.nghostzones)) = ivect::ref(cgh->cctk_nghostzones); } } @@ -312,18 +307,20 @@ namespace Carpet { ivect::ref(cgh->cctk_levoffdenom) = 0; ivect::ref(cgh->cctk_gsh) = 0xdead; // ivect::ref(cgh->cctk_nghostzones) = 0xdead; - ivect::ref(cgh->cctk_nghostzones) = vdd.at(map)->lghosts; + ivect::ref(cgh->cctk_nghostzones) = vdd[map]->lghosts; for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { - ivect::ref(const_cast<int*>(groupdata.at(group).info.gsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.gsh)) = ivect::ref(cgh->cctk_gsh); - ivect::ref(const_cast<int*>(groupdata.at(group).info.nghostzones)) + ivect::ref(const_cast<int*>(groupdata[group].info.nghostzones)) = ivect::ref(cgh->cctk_nghostzones); } } - carpetGH.map = map = -1; + map = -1; +// TODO: disable temporarily +// cgh->cctk_map = -0xdead; assert (is_level_mode()); } @@ -335,15 +332,15 @@ namespace Carpet { void enter_local_mode (cGH * const cgh, int const c) { assert (is_singlemap_mode()); - assert (c>=0 && c<vhh.at(map)->components(reflevel)); + assert (c>=0 && c<vhh[map]->components(reflevel)); Checkpoint ("Entering local mode"); component = c; // Set cGH fields - const ibbox& baseext = vdd.at(map)->bases.at(reflevel).at(mglevel).exterior; - const bbvect& obnds = vhh.at(map)->outer_boundaries.at(reflevel).at(component); - const ibbox& ext = vdd.at(map)->boxes.at(reflevel).at(component).at(mglevel).exterior; + const ibbox& baseext = vdd[map]->bases[reflevel][mglevel].exterior; + const bbvect& obnds = vhh[map]->outer_boundaries[reflevel][component]; + const ibbox& ext = vdd[map]->boxes[reflevel][component][mglevel].exterior; ivect::ref(cgh->cctk_lsh) = ext.shape() / ext.stride(); ivect::ref(cgh->cctk_lbnd) @@ -376,17 +373,17 @@ namespace Carpet { for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { - ivect::ref(const_cast<int*>(groupdata.at(group).info.lsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.lsh)) = ivect::ref(cgh->cctk_lsh); - ivect::ref(const_cast<int*>(groupdata.at(group).info.lbnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.lbnd)) = ivect::ref(cgh->cctk_lbnd); - ivect::ref(const_cast<int*>(groupdata.at(group).info.ubnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.ubnd)) = ivect::ref(cgh->cctk_ubnd); for (int d=0; d<dim; ++d) { - const_cast<int*>(groupdata.at(group).info.bbox)[2*d ] + const_cast<int*>(groupdata[group].info.bbox)[2*d ] = cgh->cctk_bbox[2*d ]; - const_cast<int*>(groupdata.at(group).info.bbox)[2*d+1] + const_cast<int*>(groupdata[group].info.bbox)[2*d+1] = cgh->cctk_bbox[2*d+1]; } @@ -396,18 +393,16 @@ namespace Carpet { assert (firstvar>=0); const int num_tl = CCTK_NumTimeLevelsFromVarI (firstvar); -// assert (vhh.at(map)->is_local(reflevel,component)); +// assert (vhh[map]->is_local(reflevel,component)); assert (group<(int)arrdata.size()); for (int var=0; var<numvars; ++var) { - assert (firstvar+var<CCTK_NumVars()); + assert (var<(int)arrdata[group][map].data.size()); for (int tl=0; tl<num_tl; ++tl) { - ggf<dim> * const ff = arrdata.at(group).at(map).data.at(var); + ggf<dim> * const ff = arrdata[group][map].data[var]; if (ff) { - gdata<dim> * const data - = (*ff) (-tl, reflevel, component, mglevel); - assert (data); - cgh->data[firstvar+var][tl] = data->storage(); + cgh->data[firstvar+var][tl] + = (*ff) (-tl, reflevel, component, mglevel)->storage(); } else { cgh->data[firstvar+var][tl] = 0; } @@ -448,17 +443,17 @@ namespace Carpet { for (int group=0; group<CCTK_NumGroups(); ++group) { if (CCTK_GroupTypeI(group) == CCTK_GF) { - ivect::ref(const_cast<int*>(groupdata.at(group).info.lsh)) + ivect::ref(const_cast<int*>(groupdata[group].info.lsh)) = ivect::ref(cgh->cctk_lsh); - ivect::ref(const_cast<int*>(groupdata.at(group).info.lbnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.lbnd)) = ivect::ref(cgh->cctk_lbnd); - ivect::ref(const_cast<int*>(groupdata.at(group).info.ubnd)) + ivect::ref(const_cast<int*>(groupdata[group].info.ubnd)) = ivect::ref(cgh->cctk_ubnd); for (int d=0; d<dim; ++d) { - const_cast<int*>(groupdata.at(group).info.bbox)[2*d ] + const_cast<int*>(groupdata[group].info.bbox)[2*d ] = cgh->cctk_bbox[2*d ]; - const_cast<int*>(groupdata.at(group).info.bbox)[2*d+1] + const_cast<int*>(groupdata[group].info.bbox)[2*d+1] = cgh->cctk_bbox[2*d+1]; } @@ -470,7 +465,7 @@ namespace Carpet { assert (group<(int)arrdata.size()); for (int var=0; var<numvars; ++var) { - assert (firstvar+var<CCTK_NumVars()); + assert (var<(int)arrdata[group][map].data.size()); for (int tl=0; tl<num_tl; ++tl) { cgh->data[firstvar+var][tl] = 0; } @@ -658,7 +653,7 @@ namespace Carpet { bool component_iterator::done () const { return (grouptype == CCTK_GF - ? c >= vhh.at(map)->components(reflevel) + ? c >= vhh[map]->components(reflevel) : c >= CCTK_nProcs(cgh)); } @@ -693,7 +688,7 @@ namespace Carpet { bool local_component_iterator::done () const { return c >= (grouptype == CCTK_GF - ? vhh.at(map)->components(reflevel) + ? vhh[map]->components(reflevel) : CCTK_nProcs(cgh)); } @@ -702,7 +697,7 @@ namespace Carpet { do { ++ c; } while (! done() && ! (grouptype == CCTK_GF - ? vhh.at(map)->is_local(reflevel, c) + ? vhh[map]->is_local(reflevel, c) : c == CCTK_MyProc(cgh))); if (! done()) { leave_local_mode (cgh); diff --git a/Carpet/Carpet/src/modes.hh b/Carpet/Carpet/src/modes.hh index b8cf65d92..618daddc1 100644 --- a/Carpet/Carpet/src/modes.hh +++ b/Carpet/Carpet/src/modes.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/modes.hh,v 1.2 2004/06/14 09:42:53 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/modes.hh,v 1.1 2004/01/25 14:57:28 schnetter Exp $ #ifndef MODES_HH #define MODES_HH @@ -146,21 +146,21 @@ namespace Carpet { #define BEGIN_MGLEVEL_LOOP(cgh) \ do { \ bool mglevel_loop_ = true; \ - for (mglevel_iterator mg_iter_(cgh); \ - !mg_iter_.done(); \ - mg_iter_.step()) { + for (mglevel_iterator iter_(cgh); \ + !iter_.done(); \ + iter_.step()) { #define END_MGLEVEL_LOOP \ } \ assert (mglevel_loop_); \ mglevel_loop_ = false; \ } while (false) -#define BEGIN_REVERSE_MGLEVEL_LOOP(cgh) \ - do { \ - bool reverse_mglevel_loop_ = true; \ - for (reverse_mglevel_iterator mg_iter_(cgh); \ - !mg_iter_.done(); \ - mg_iter_.step()) { +#define BEGIN_REVERSE_MGLEVEL_LOOP(cgh) \ + do { \ + bool reverse_mglevel_loop_ = true; \ + for (reverse_mglevel_iterator iter_(cgh); \ + !iter_.done(); \ + iter_.step()) { #define END_REVERSE_MGLEVEL_LOOP \ } \ assert (reverse_mglevel_loop_); \ @@ -170,33 +170,33 @@ namespace Carpet { #define BEGIN_REFLEVEL_LOOP(cgh) \ do { \ bool reflevel_loop_ = true; \ - for (reflevel_iterator ref_iter_(cgh); \ - !ref_iter_.done(); \ - ref_iter_.step()) { + for (reflevel_iterator iter_(cgh); \ + !iter_.done(); \ + iter_.step()) { #define END_REFLEVEL_LOOP \ } \ assert (reflevel_loop_); \ reflevel_loop_ = false; \ } while (false) -#define BEGIN_REVERSE_REFLEVEL_LOOP(cgh) \ - do { \ - bool reverse_reflevel_loop_ = true; \ - for (reverse_reflevel_iterator ref_iter_(cgh); \ - !ref_iter_.done(); \ - ref_iter_.step()) { +#define BEGIN_REVERSE_REFLEVEL_LOOP(cgh) \ + do { \ + bool reverse_reflevel_loop_ = true; \ + for (reverse_reflevel_iterator iter_(cgh); \ + !iter_.done(); \ + iter_.step()) { #define END_REVERSE_REFLEVEL_LOOP \ } \ assert (reverse_reflevel_loop_); \ reverse_reflevel_loop_ = false; \ } while (false) -#define BEGIN_MAP_LOOP(cgh, grouptype) \ - do { \ - bool map_loop_ = true; \ - for (map_iterator map_iter_(cgh, grouptype); \ - !map_iter_.done(); \ - map_iter_.step()) { +#define BEGIN_MAP_LOOP(cgh, grouptype) \ + do { \ + bool map_loop_ = true; \ + for (map_iterator iter_(cgh, grouptype); \ + !iter_.done(); \ + iter_.step()) { #define END_MAP_LOOP \ } \ assert (map_loop_); \ @@ -206,9 +206,9 @@ namespace Carpet { #define BEGIN_COMPONENT_LOOP(cgh, grouptype) \ do { \ bool component_loop_ = true; \ - for (component_iterator comp_iter_(cgh, grouptype); \ - !comp_iter_.done(); \ - comp_iter_.step()) { + for (component_iterator iter_(cgh, grouptype); \ + !iter_.done(); \ + iter_.step()) { #define END_COMPONENT_LOOP \ } \ assert (component_loop_); \ @@ -218,9 +218,9 @@ namespace Carpet { #define BEGIN_LOCAL_COMPONENT_LOOP(cgh, grouptype) \ do { \ bool local_component_loop_ = true; \ - for (local_component_iterator comp_iter_(cgh, grouptype); \ - !comp_iter_.done(); \ - comp_iter_.step()) { + for (local_component_iterator iter_(cgh, grouptype); \ + !iter_.done(); \ + iter_.step()) { #define END_LOCAL_COMPONENT_LOOP \ } \ assert (local_component_loop_); \ diff --git a/Carpet/Carpet/src/variables.cc b/Carpet/Carpet/src/variables.cc index 1df466026..d90eec8ee 100644 --- a/Carpet/Carpet/src/variables.cc +++ b/Carpet/Carpet/src/variables.cc @@ -1,17 +1,11 @@ -#include <vector> -#include "dh.hh" -#include "gh.hh" -#include "th.hh" -#include "carpet.hh" +#include "variables.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/variables.cc,v 1.18 2003/08/10 21:59:51 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/variables.cc,v 1.19 2004/01/25 14:57:28 schnetter Exp $"; CCTK_FILEVERSION(Carpet_Carpet_variables_cc); } - - namespace Carpet { using namespace std; @@ -24,12 +18,18 @@ namespace Carpet { // Maximum number of refinement levels int maxreflevels; + // Refinement levels + int reflevels; + // Refinement factor int reffact; // Refinement factor on finest grid int maxreflevelfact; + // Base multigrid level + int basemglevel; + // Multigrid levels int mglevels; @@ -39,11 +39,15 @@ namespace Carpet { // Multigrid factor on coarsest grid int maxmglevelfact; + // Maps + int maps; + // Current position on the grid hierarchy int reflevel; int mglevel; + int map; int component; // refinement factor of current level: ipow(refinement_factor, reflevel) @@ -52,30 +56,39 @@ namespace Carpet { // multigrid factor of current level: ipow(multigrid_factor, mglevel) int mglevelfact; + + + // Current times on the refinement levels + vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel] + CCTK_REAL global_time; + CCTK_REAL delta_time; + + + // Is this the time for a global mode call? + bool do_meta_mode; bool do_global_mode; // Is prolongation enabled? bool do_prolongate; - // Current times on the refinement levels - vector<CCTK_REAL> refleveltimes; - CCTK_REAL delta_time; - // Data for grid functions // The grid hierarchy - gh<dim>* hh; - th<dim>* tt; - dh<dim>* dd; + vector<gh<dim>*> vhh; // [map] + vector<dh<dim>*> vdd; // [map] + vector<th<dim>*> vtt; // [map] + // Data for the groups + vector<groupdesc> groupdata; // [group] + // Data for everything - vector<arrdesc> arrdata; // [group] + vector<vector<arrdesc> > arrdata; // [group][map] // Checksums - // [rl][c][group][var][tl] - vector<vector<vector<vector<vector<ckdesc> > > > > checksums; + // [rl][ml][group][m][c][var][tl] + vector<vector<vector<vector<vector<vector<vector<ckdesc> > > > > > > checksums; } // namespace Carpet diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh index 284f6d1ae..ba0b9aeed 100644 --- a/Carpet/Carpet/src/variables.hh +++ b/Carpet/Carpet/src/variables.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/variables.hh,v 1.5 2004/05/21 18:16:23 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/Carpet/src/variables.hh,v 1.1 2004/01/25 14:57:28 schnetter Exp $ // It is assumed that each group has at least one map. All arrays // have exactly one map. All maps have the same number of refinement @@ -24,9 +24,7 @@ #include "gh.hh" #include "operators.hh" #include "th.hh" -#include "vect.hh" -#include "carpet_public.h" #include "defines.hh" @@ -49,7 +47,7 @@ namespace Carpet { // Refinement factor extern int reffact; - // Refinement factor on finest possible grid + // Refinement factor on finest grid extern int maxreflevelfact; // Base multigrid level @@ -83,19 +81,11 @@ namespace Carpet { - // Carpet's GH - extern CarpetGH carpetGH; - - - - // Times and spaces on the refinement levels - extern CCTK_REAL global_time; + // Current times on the refinement levels extern vector<vector<CCTK_REAL> > leveltimes; // [mglevel][reflevel] + extern CCTK_REAL global_time; extern CCTK_REAL delta_time; - extern vector<vect<CCTK_REAL,dim> > origin_space; // [mglevel] - extern vect<CCTK_REAL,dim> delta_space; - // Is this the time for a global mode call? @@ -128,9 +118,20 @@ namespace Carpet { dh<dim>* dd; th<dim>* tt; vector<ggf<dim>*> data; // [var] + // VGF }; extern vector<vector<arrdesc> > arrdata; // [group][map] + + + // Checksums + struct ckdesc { + bool valid; + unsigned int sum; + }; + // [rl][ml][group][m][c][var][tl] + extern vector<vector<vector<vector<vector<vector<vector<ckdesc> > > > > > > checksums; + } // namespace Carpet #endif // !defined(VARIABLES_HH) diff --git a/Carpet/CarpetIOASCII/interface.ccl b/Carpet/CarpetIOASCII/interface.ccl index f76c83a95..875c27690 100644 --- a/Carpet/CarpetIOASCII/interface.ccl +++ b/Carpet/CarpetIOASCII/interface.ccl @@ -1,5 +1,5 @@ # Interface definition for thorn CarpetIOASCII -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/interface.ccl,v 1.7 2003/09/04 16:23:21 tradke Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/interface.ccl,v 1.8 2004/01/25 14:57:28 schnetter Exp $ implements: IOASCII @@ -14,3 +14,11 @@ uses include header: gdata.hh uses include header: gf.hh uses include header: ggf.hh + + + +# Vector groups don't work yet in Carpet +#CCTK_INT next_output_iteration[3] TYPE=scalar +#CCTK_REAL next_output_time[3] TYPE=scalar +CCTK_INT next_output_iteration TYPE=array DISTRIB=constant DIM=1 SIZE=3 +CCTK_REAL next_output_time TYPE=array DISTRIB=constant DIM=1 SIZE=3 diff --git a/Carpet/CarpetIOASCII/param.ccl b/Carpet/CarpetIOASCII/param.ccl index 959a56c69..1ec9d4b1c 100644 --- a/Carpet/CarpetIOASCII/param.ccl +++ b/Carpet/CarpetIOASCII/param.ccl @@ -1,5 +1,5 @@ # Parameter definitions for thorn CarpetIOASCII -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/param.ccl,v 1.10 2004/01/13 15:26:58 cott Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/param.ccl,v 1.11 2004/01/25 14:57:28 schnetter Exp $ @@ -7,7 +7,9 @@ shares: IO USES STRING out_dir +USES KEYWORD out_criterion USES CCTK_INT out_every +USES CCTK_REAL out_dt @@ -37,6 +39,10 @@ USES CCTK_REAL out_yzplane_x +USES BOOLEAN new_filename_scheme + + + private: @@ -101,6 +107,29 @@ CCTK_STRING out3D_vars "Variables to output in 3D ASCII file format" STEERABLE = +KEYWORD out1D_criterion "Criterion to select 1D ASCII output intervals, overrides out_every" STEERABLE = ALWAYS +{ + "never" :: "Never output" + "iteration" :: "Output every so many iterations" + "time" :: "Output every that much coordinate time" +} "iteration" + +KEYWORD out2D_criterion "Criterion to select 2D ASCII output intervals, overrides out_every" STEERABLE = ALWAYS +{ + "never" :: "Never output" + "iteration" :: "Output every so many iterations" + "time" :: "Output every that much coordinate time" +} "iteration" + +KEYWORD out3D_criterion "Criterion to select 3D ASCII output intervals, overrides out_every" STEERABLE = ALWAYS +{ + "never" :: "Never output" + "iteration" :: "Output every so many iterations" + "time" :: "Output every that much coordinate time" +} "iteration" + + + CCTK_INT out1D_every "How often to do 1D ASCII output, overrides out_every" STEERABLE = ALWAYS { -1:* :: "Values <= 0 disable 1D output" @@ -118,6 +147,30 @@ CCTK_INT out3D_every "How often to do 3D ASCII output, overrides out_every" STEE +REAL out1D_dt "How often to do 1D ASCII output, overrides out_dt" STEERABLE = ALWAYS +{ + (0:* :: "In intervals of that much coordinate time" + 0 :: "As often as possible" + -1 :: "Disable output" + -2 :: "Default to IO::out_dt" +} -2 + +REAL out2D_dt "How often to do 2D ASCII output, overrides out_dt" STEERABLE = ALWAYS +{ + (0:* :: "In intervals of that much coordinate time" + 0 :: "As often as possible" + -1 :: "Disable output" + -2 :: "Default to IO::out_dt" +} -2 + +REAL out3D_dt "How often to do 3D ASCII output, overrides out_dt" STEERABLE = ALWAYS +{ + (0:* :: "In intervals of that much coordinate time" + 0 :: "As often as possible" + -1 :: "Disable output" + -2 :: "Default to IO::out_dt" +} -2 + BOOLEAN out1D_x "Do 1D IOASCII output in the x-direction" STEERABLE = ALWAYS { } "yes" diff --git a/Carpet/CarpetIOASCII/schedule.ccl b/Carpet/CarpetIOASCII/schedule.ccl index 3c7f55c59..cee9dab8c 100644 --- a/Carpet/CarpetIOASCII/schedule.ccl +++ b/Carpet/CarpetIOASCII/schedule.ccl @@ -1,7 +1,16 @@ # Schedule definitions for thorn CarpetIOASCII -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/schedule.ccl,v 1.4 2003/11/05 16:18:37 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/schedule.ccl,v 1.5 2004/01/25 14:57:28 schnetter Exp $ + +storage: next_output_iteration next_output_time schedule CarpetIOASCIIStartup at STARTUP after IOUtil_Startup { LANG: C + OPTIONS: global } "Startup routine" + +schedule CarpetIOASCIIInit at INITIAL +{ + LANG: C + OPTIONS: global +} "Initialisation routine" diff --git a/Carpet/CarpetIOASCII/src/ioascii.cc b/Carpet/CarpetIOASCII/src/ioascii.cc index e8ef28719..d45163d96 100644 --- a/Carpet/CarpetIOASCII/src/ioascii.cc +++ b/Carpet/CarpetIOASCII/src/ioascii.cc @@ -30,7 +30,7 @@ #include "ioascii.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.cc,v 1.55 2003/11/13 16:03:38 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.cc,v 1.56 2004/01/25 14:57:28 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetIOASCII_ioascii_cc); } @@ -48,8 +48,6 @@ namespace CarpetIOASCII { using namespace std; using namespace Carpet; - bool CheckForVariable (const cGH* const cgh, - const char* const varlist, const int vindex); void SetFlag (int index, const char* optstring, void* arg); template<int D,int DD> @@ -60,23 +58,34 @@ namespace CarpetIOASCII { const int time, const vect<int,D>& org, const vect<int,DD>& dirs, - const int tl, const int rl, - const int c, const int ml, + const int m, + const int c, + const int tl, const CCTK_REAL coord_time, const vect<CCTK_REAL,D>& coord_lower, const vect<CCTK_REAL,D>& coord_upper); - int CarpetIOASCIIStartup() + void CarpetIOASCIIStartup() { IOASCII<1>::Startup(); IOASCII<2>::Startup(); IOASCII<3>::Startup(); + } + + + + void CarpetIOASCIIInit (CCTK_ARGUMENTS) + { + DECLARE_CCTK_ARGUMENTS; - return 0; + for (int d=0; d<3; ++d) { + next_output_iteration[d] = 0; + next_output_time[d] = cctkGH->cctk_time; + } } @@ -158,7 +167,14 @@ namespace CarpetIOASCII { { DECLARE_CCTK_PARAMETERS; + assert (is_level_mode()); + const int n = CCTK_VarIndex(varname); + if (n<0) { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Variable \"%s\" does not exist", varname); + return -1; + } assert (n>=0 && n<CCTK_NumVars()); const int group = CCTK_GroupIndexFromVarI (n); assert (group>=0 && group<(int)Carpet::arrdata.size()); @@ -178,21 +194,23 @@ namespace CarpetIOASCII { } const int grouptype = CCTK_GroupTypeI(group); - const int rl = grouptype==CCTK_GF ? reflevel : 0; + if (grouptype != CCTK_GF) assert (reflevel==0); + + const int groupdim = CCTK_GroupDimI(group); + assert (outdim <= groupdim); // Get grid hierarchy extentsion from IOUtil const ioGH * const iogh = (const ioGH *)CCTK_GHExtension (cgh, "IO"); assert (iogh); // Create the output directory - const char* myoutdir = GetStringParameter("out%dD_dir", out_dir); + const char* const myoutdir = GetStringParameter("out%dD_dir", out_dir); if (CCTK_MyProc(cgh)==0) { CCTK_CreateDirectory (0755, myoutdir); } // Loop over all direction combinations - vect<int,outdim> dirs; - for (int d=0; d<outdim; ++d) dirs[d] = 0; + vect<int,outdim> dirs (0); bool done; do { @@ -247,222 +265,254 @@ namespace CarpetIOASCII { // Skip output if not desired if (desired) { - - // Invent a file name - ostringstream filenamebuf; - filenamebuf << myoutdir << "/" << alias << "."; - for (int d=0; d<outdim; ++d) { - assert (dirs[d]>=0 && dirs[d]<3); - const char* const coords = "xyz"; - filenamebuf << coords[dirs[d]]; - } - const char* const suffixes = "lpv"; - filenamebuf << suffixes[outdim-1]; - // we need a persistent temporary here - string filenamestr = filenamebuf.str(); - const char* const filename = filenamestr.c_str(); - - ofstream file; - - if (CCTK_MyProc(cgh)==0) { - // If this is the first time, then write a nice header on - // the root processor - if (do_truncate[n]) { - struct stat fileinfo; - if (! iogh->recovered - || stat(filename, &fileinfo)!=0) { - file.open (filename, ios::out | ios::trunc); - if (! file.good()) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Could not open output file \"%s\" for variable \"%s\"", - filename, varname); - } - assert (file.good()); - file << "# " << varname; - for (int d=0; d<outdim; ++d) { - file << " " << "xyz"[dirs[d]]; - } - file << " (" << alias << ")" << endl; - file << "#" << endl; - assert (file.good()); - } - } - if (! file.is_open()) { - file.open (filename, ios::out | ios::app); - assert (file.good()); - } - file << setprecision(15); - assert (file.good()); - } - - assert (outdim <= CCTK_GroupDimI(group)); - - // Find the output offset - vect<int,dim> offset(0); - switch (outdim) { - case 1: - switch (dirs[0]) { - case 0: - offset[1] = GetGridOffset (cgh, 2, - "out%dD_xline_yi", "out_xline_yi", - "out%dD_xline_y", "out_xline_y", - out_xline_y); - offset[2] = GetGridOffset (cgh, 3, - "out%dD_xline_zi", "out_xline_zi", - "out%dD_xline_z", "out_xline_z", - out_xline_z); - break; - case 1: - offset[0] = GetGridOffset (cgh, 1, - "out%dD_yline_xi", "out_yline_xi", - "out%dD_yline_x", "out_yline_x", - out_yline_x); - offset[2] = GetGridOffset (cgh, 3, - "out%dD_yline_zi", "out_yline_zi", - "out%dD_yline_z", "out_yline_z", - out_yline_z); - break; - case 2: - offset[0] = GetGridOffset (cgh, 1, - "out%dD_zline_xi", "out_zline_xi", - "out%dD_zline_x", "out_zline_x", - out_zline_x); - offset[1] = GetGridOffset (cgh, 2, - "out%dD_zline_yi", "out_zline_yi", - "out%dD_zline_y", "out_zline_y", - out_zline_y); - break; - default: - assert (0); - } - break; - case 2: - if (dirs[0]==0 && dirs[1]==1) { - offset[2] = GetGridOffset - (cgh, 3, - "out%dD_xyplane_zi", "out_xyplane_zi", - "out%dD_xyplane_z", "out_xyplane_z", - out_xyplane_z); - } else if (dirs[0]==0 && dirs[1]==2) { - offset[1] = GetGridOffset - (cgh, 2, - "out%dD_xzplane_yi", "out_xzplane_yi", - "out%dD_xzplane_y", "out_xzplane_y", - out_xzplane_y); - } else if (dirs[0]==1 && dirs[1]==2) { - offset[0] = GetGridOffset - (cgh, 1, - "out%dD_yzplane_xi", "out_yzplane_xi", - "out%dD_yzplane_x", "out_yzplane_x", - out_yzplane_x); - } else { - assert (0); - } - break; - case 3: - // The offset doesn't matter in this case - break; - default: - assert (0); - } - - // Traverse all components on this refinement and multigrid - // level - BEGIN_COMPONENT_LOOP(cgh, grouptype) { - - const ggf<dim>* ff = 0; - - assert (var < (int)arrdata[group].data.size()); - ff = (ggf<dim>*)arrdata[group].data[var]; - - const int mintl = output_all_timelevels ? 1-num_tl : 0; - const int maxtl = 0; - for (int tl=mintl; tl<=maxtl; ++tl) { - - const gdata<dim>* const data - = (*ff) (tl, rl, component, mglevel); - bbox<int,dim> ext = data->extent(); - - vect<int,dim> lo = ext.lower(); - vect<int,dim> hi = ext.upper(); - vect<int,dim> str = ext.stride(); - - // Ignore ghost zones if desired - for (int d=0; d<dim; ++d) { - bool output_lower_ghosts - = cgh->cctk_bbox[2*d] ? out3D_outer_ghosts : out3D_ghosts; - bool output_upper_ghosts - = cgh->cctk_bbox[2*d+1] ? out3D_outer_ghosts : out3D_ghosts; - - if (! output_lower_ghosts) { - lo[d] += cgh->cctk_nghostzones[d] * str[d]; + + // Traverse all maps on this refinement and multigrid level + BEGIN_MAP_LOOP(cgh, grouptype) { + + // Find the output offset + ivect offset(0); + if (grouptype == CCTK_GF) { + switch (outdim) { + case 1: + switch (dirs[0]) { + case 0: + offset[1] = GetGridOffset (cgh, 2, + "out%dD_xline_yi", "out_xline_yi", + "out%dD_xline_y", "out_xline_y", + out_xline_y); + offset[2] = GetGridOffset (cgh, 3, + "out%dD_xline_zi", "out_xline_zi", + "out%dD_xline_z", "out_xline_z", + out_xline_z); + break; + case 1: + offset[0] = GetGridOffset (cgh, 1, + "out%dD_yline_xi", "out_yline_xi", + "out%dD_yline_x", "out_yline_x", + out_yline_x); + offset[2] = GetGridOffset (cgh, 3, + "out%dD_yline_zi", "out_yline_zi", + "out%dD_yline_z", "out_yline_z", + out_yline_z); + break; + case 2: + offset[0] = GetGridOffset (cgh, 1, + "out%dD_zline_xi", "out_zline_xi", + "out%dD_zline_x", "out_zline_x", + out_zline_x); + offset[1] = GetGridOffset (cgh, 2, + "out%dD_zline_yi", "out_zline_yi", + "out%dD_zline_y", "out_zline_y", + out_zline_y); + break; + default: + assert (0); } - if (! output_upper_ghosts) { - hi[d] -= cgh->cctk_nghostzones[d] * str[d]; + break; + case 2: + if (dirs[0]==0 && dirs[1]==1) { + offset[2] = GetGridOffset + (cgh, 3, + "out%dD_xyplane_zi", "out_xyplane_zi", + "out%dD_xyplane_z", "out_xyplane_z", + out_xyplane_z); + } else if (dirs[0]==0 && dirs[1]==2) { + offset[1] = GetGridOffset + (cgh, 2, + "out%dD_xzplane_yi", "out_xzplane_yi", + "out%dD_xzplane_y", "out_xzplane_y", + out_xzplane_y); + } else if (dirs[0]==1 && dirs[1]==2) { + offset[0] = GetGridOffset + (cgh, 1, + "out%dD_yzplane_xi", "out_yzplane_xi", + "out%dD_yzplane_x", "out_yzplane_x", + out_yzplane_x); + } else { + assert (0); } + break; + case 3: + // The offset doesn't matter in this case + break; + default: + assert (0); } - ext = bbox<int,dim>(lo,hi,str); + } // if grouptype is GF + + ofstream file; + if (CCTK_MyProc(cgh)==0) { - // coordinates - const CCTK_REAL coord_time = cgh->cctk_time; - vect<CCTK_REAL,dim> global_lower; - vect<CCTK_REAL,dim> coord_delta; - if (grouptype == CCTK_GF) { - for (int d=0; d<dim; ++d) { - global_lower[d] = cgh->cctk_origin_space[d]; - coord_delta[d] = cgh->cctk_delta_space[d] / maxreflevelfact; + // Invent a file name + ostringstream filenamebuf; + if (new_filename_scheme) { + filenamebuf << myoutdir << "/" << alias << "."; + if (maps > 1) { + cout << Carpet::map << "-"; + } + for (int d=0; d<outdim; ++d) { + const char* const coords = "xyz"; + filenamebuf << coords[dirs[d]]; } +// The offsets differ per level +// for (int dd=0; dd<groupdim; ++dd) { +// bool print_dir = true; +// for (int d=0; d<outdim; ++d) { +// print_dir = print_dir && dirs[d] != dd; +// } +// if (print_dir) { +// filenamebuf << "." << offset[dd]; +// } +// } + filenamebuf << ".asc"; } else { - for (int d=0; d<dim; ++d) { - global_lower[d] = 0.0; - coord_delta[d] = 1.0 / (cgh->cctk_gsh[d] - 1); + filenamebuf << myoutdir << "/" << alias << "."; + if (maps > 1) { + cout << Carpet::map << "-"; } + for (int d=0; d<outdim; ++d) { + assert (dirs[d]>=0 && dirs[d]<3); + const char* const coords = "xyz"; + filenamebuf << coords[dirs[d]]; + } + const char* const suffixes = "lpv"; + filenamebuf << suffixes[outdim-1]; } - // Note: don't permute the "coord_delta" and - // "data->extent().lower()" - // (it seems that for gcc 2.95 you then pick up the - // integer operator*) - const vect<CCTK_REAL,dim> coord_lower = global_lower + coord_delta * vect<CCTK_REAL,dim>(lo); - const vect<CCTK_REAL,dim> coord_upper = global_lower + coord_delta * vect<CCTK_REAL,dim>(hi); + // we need a persistent temporary here + string filenamestr = filenamebuf.str(); + const char* const filename = filenamestr.c_str(); - vect<int,dim> offset1; - for (int d=0; d<dim; ++d) { - assert (cgh->cctk_levoffdenom[d]==1); - offset1[d] = (cgh->cctk_levoff[d] + offset[d]) * ext.stride()[d]; + // If this is the first time, then write a nice header + if (do_truncate[n]) { + struct stat fileinfo; + if (! iogh->recovered + || stat(filename, &fileinfo)!=0) { + file.open (filename, ios::out | ios::trunc); + if (! file.good()) { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "Could not open output file \"%s\" for variable \"%s\"", + filename, varname); + } + assert (file.good()); + file << "# " << varname; + for (int d=0; d<outdim; ++d) { + file << " " << "xyz"[dirs[d]]; + } + file << " (" << alias << ")" << endl; + file << "#" << endl; + assert (file.good()); + } } - for (int d=0; d<outdim; ++d) { - offset1[dirs[d]] = ext.lower()[dirs[d]]; + + // Open the file + if (! file.is_open()) { + file.open (filename, ios::out | ios::app); + assert (file.good()); } - WriteASCII (file, data, ext, n, cgh->cctk_iteration, offset1, dirs, - tl, rl, component, mglevel, - coord_time, coord_lower, coord_upper); + file << setprecision(15); + assert (file.good()); - // Append EOL after every component - if (CCTK_MyProc(cgh)==0) { - if (separate_components) { - assert (file.good()); - file << endl; + } // if on the root processor + + // Traverse and components on this multigrid and + // refinement level and map + BEGIN_COMPONENT_LOOP(cgh, grouptype) { + + const ggf<dim>* ff = 0; + + assert (var < (int)arrdata[group][Carpet::map].data.size()); + ff = (ggf<dim>*)arrdata[group][Carpet::map].data[var]; + + const int mintl = output_all_timelevels ? 1-num_tl : 0; + const int maxtl = 0; + for (int tl=mintl; tl<=maxtl; ++tl) { + + const gdata<dim>* const data + = (*ff) (tl, reflevel, component, mglevel); + ibbox ext = data->extent(); + + ivect lo = ext.lower(); + ivect hi = ext.upper(); + ivect str = ext.stride(); + + // Ignore ghost zones if desired + for (int d=0; d<dim; ++d) { + bool output_lower_ghosts + = cgh->cctk_bbox[2*d] ? out3D_outer_ghosts : out3D_ghosts; + bool output_upper_ghosts + = cgh->cctk_bbox[2*d+1] ? out3D_outer_ghosts : out3D_ghosts; + + if (! output_lower_ghosts) { + lo[d] += cgh->cctk_nghostzones[d] * str[d]; + } + if (! output_upper_ghosts) { + hi[d] -= cgh->cctk_nghostzones[d] * str[d]; + } + } + ext = ibbox(lo,hi,str); + + // coordinates + const CCTK_REAL coord_time = cgh->cctk_time; + rvect global_lower; + rvect coord_delta; + if (grouptype == CCTK_GF) { + for (int d=0; d<dim; ++d) { + global_lower[d] = cgh->cctk_origin_space[d]; + coord_delta[d] = cgh->cctk_delta_space[d] / maxreflevelfact; + } + } else { + for (int d=0; d<dim; ++d) { + global_lower[d] = 0.0; + coord_delta[d] = 1.0 / (cgh->cctk_gsh[d] - 1); + } + } + const rvect coord_lower + = global_lower + coord_delta * rvect(lo); + const rvect coord_upper + = global_lower + coord_delta * rvect(hi); + + ivect offset1; + for (int d=0; d<dim; ++d) { + assert (cgh->cctk_levoffdenom[d]==1); + offset1[d] = (cgh->cctk_levoff[d] + offset[d]) * ext.stride()[d]; + } + for (int d=0; d<outdim; ++d) { + offset1[dirs[d]] = ext.lower()[dirs[d]]; + } + + WriteASCII (file, data, ext, n, cgh->cctk_iteration, + offset1, dirs, + reflevel, mglevel, Carpet::map, component, tl, + coord_time, coord_lower, coord_upper); + + // Append EOL after every component + if (CCTK_MyProc(cgh)==0) { + if (separate_components) { + assert (file.good()); + file << endl; + } } + assert (file.good()); + + } // for tl + + } END_COMPONENT_LOOP; + + // Append EOL after every complete set of components + if (CCTK_MyProc(cgh)==0) { + if (separate_grids) { + assert (file.good()); + file << endl; } + file.close(); assert (file.good()); - - } // for tl - - } END_COMPONENT_LOOP; - - // Append EOL after every complete set of components - if (CCTK_MyProc(cgh)==0) { - if (separate_grids) { - assert (file.good()); - file << endl; - } - file.close(); - assert (file.good()); - } - - assert (! file.is_open()); - + } + + assert (! file.is_open()); + + } END_MAP_LOOP; + } // if (desired) } // if (ascending) @@ -471,7 +521,7 @@ namespace CarpetIOASCII { done = true; for (int d=0; d<outdim; ++d) { ++dirs[d]; - if (dirs[d]<CCTK_GroupDimI(group)) { + if (dirs[d]<groupdim) { done = false; break; } @@ -490,30 +540,86 @@ namespace CarpetIOASCII { template<int outdim> int IOASCII<outdim> - ::TimeToOutput (const cGH* const cgh, const int vindex) + ::TimeToOutput (const cGH* const cctkGH, const int vindex) { + DECLARE_CCTK_ARGUMENTS; DECLARE_CCTK_PARAMETERS; assert (vindex>=0 && vindex<CCTK_NumVars()); - const int myoutevery = GetIntParameter("out%dD_every", out_every); + static int output_iteration = -1; + static bool output_this_iteration; + static vector<bool> output_variables; - if (myoutevery <= 0) { - // Nothing should be output at all - return 0; - } + assert (cctk_iteration >= output_iteration); + if (cctk_iteration > output_iteration) { + // A new iteration; check whether output should happen + + output_iteration = cctk_iteration; + + const char * const myoutcriterion + = GetStringParameter("out%dD_criterion", out_criterion); + if (CCTK_EQUALS (myoutcriterion, "never")) { + + // Never output + output_this_iteration = false; + + } else if (CCTK_EQUALS (myoutcriterion, "iteration")) { + + const int myoutevery = GetIntParameter("out%dD_every", out_every); + if (myoutevery <= 0) { + output_this_iteration = false; + } else { + output_this_iteration + = cctk_iteration >= next_output_iteration[outdim-1]; + if (output_this_iteration) { + next_output_iteration[outdim-1] += myoutevery; + } + } + + } else if (CCTK_EQUALS (myoutcriterion, "time")) { + + const CCTK_REAL myoutdt = GetRealParameter("out%dD_dt", out_dt); + if (myoutdt < 0) { + output_this_iteration = false; + } else if (myoutdt == 0) { + output_this_iteration = true; + } else { + output_this_iteration = cctk_time >= (next_output_time[outdim-1] + - 1.0e-12 * cctk_delta_time); + if (output_this_iteration) { + next_output_time[outdim-1] += myoutdt; + } + } + + } else { + + assert (0); + + } // select output criterion + + + + // check which variables to output + const int numvars = CCTK_NumVars(); + assert (vindex>=0 && vindex<numvars); + output_variables.resize (numvars); + + const char * const varlist = GetStringParameter("out%dD_vars", ""); + CCTK_TraverseString + (varlist, SetFlag, &output_variables, CCTK_GROUP_OR_VAR); + + } // check whether output should happen + + if (! output_this_iteration) return 0; + if (! output_variables[vindex]) return 0; + + const int grouptype = CCTK_GroupTypeFromVarI(vindex); + if (grouptype != CCTK_GF && reflevel > 0) return 0; - if (cgh->cctk_iteration % myoutevery != 0) { - // Nothing should be output during this iteration - return 0; - } - if (! CheckForVariable(cgh, GetStringParameter("out%dD_vars", ""), vindex)) { - // This variable should not be output - return 0; - } - if (last_output[reflevel][vindex] == cgh->cctk_iteration) { + if (last_output[reflevel][vindex] == cctk_iteration) { // Has already been output during this iteration char* varname = CCTK_FullName(vindex); CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING, @@ -525,7 +631,7 @@ namespace CarpetIOASCII { return 0; } - assert (last_output[reflevel][vindex] < cgh->cctk_iteration); + assert (last_output[reflevel][vindex] < cctk_iteration); // Should be output during this iteration return 1; @@ -559,7 +665,7 @@ namespace CarpetIOASCII { { // First choice: explicit coordinate char cparam[1000]; - sprintf (cparam, ctempl, outdim); + snprintf (cparam, sizeof cparam, ctempl, outdim); const int ncparam = CCTK_ParameterQueryTimesSet (cparam, CCTK_THORNSTRING); assert (ncparam >= 0); if (ncparam > 0) { @@ -575,7 +681,7 @@ namespace CarpetIOASCII { // Second choice: explicit index char iparam[1000]; - sprintf (iparam, itempl, outdim); + snprintf (iparam, sizeof iparam, itempl, outdim); const int niparam = CCTK_ParameterQueryTimesSet (iparam, CCTK_THORNSTRING); assert (niparam >= 0); if (niparam > 0) { @@ -629,7 +735,7 @@ namespace CarpetIOASCII { { assert (dir>=1 && dir<=dim); - assert (reflevel!=-1 && mglevel!=-1); + assert (mglevel!=-1 && reflevel!=-1 && Carpet::map!=-1); const int npoints = cgh->cctk_gsh[dir-1]; const CCTK_REAL delta = cgh->cctk_delta_space[dir-1] / cgh->cctk_levfac[dir-1]; @@ -640,10 +746,13 @@ namespace CarpetIOASCII { int cindex = (int)floor(rindex + 0.75); if (cindex<0 || cindex>=npoints) { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "The specified coordinate value %g is not within the grid range [%g,%g]", - coord, lower, upper); cindex = ifallback; + + assert (dir>=1 && dir<=3); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "The specified coordinate value %g for the %c-direction is not within the grid range [%g,%g] on convergence level %d, level %d, map %d; using %g instead", + coord, "xyz"[dir-1], lower, upper, + mglevel, reflevel, Carpet::map, lower + delta * cindex); } assert (cindex>=0 && cindex<npoints); @@ -659,7 +768,7 @@ namespace CarpetIOASCII { const char* const fallback) { char parametername[1000]; - sprintf (parametername, parametertemplate, outdim); + snprintf (parametername, sizeof parametername, parametertemplate, outdim); const int ntimes = CCTK_ParameterQueryTimesSet (parametername, CCTK_THORNSTRING); assert (ntimes >= 0); @@ -669,7 +778,7 @@ namespace CarpetIOASCII { (parametername, CCTK_THORNSTRING, &ptype); assert (ppval); const char* const pval = *ppval; - assert (ptype == PARAMETER_STRING); + assert (ptype == PARAMETER_STRING || ptype == PARAMETER_KEYWORD); return pval; } @@ -679,22 +788,23 @@ namespace CarpetIOASCII { template<int outdim> - int IOASCII<outdim> - ::GetIntParameter (const char* const parametertemplate, const int fallback) + CCTK_INT IOASCII<outdim> + ::GetIntParameter (const char* const parametertemplate, + const CCTK_INT fallback) { char parametername[1000]; - sprintf (parametername, parametertemplate, outdim); + snprintf (parametername, sizeof parametername, parametertemplate, outdim); const int ntimes = CCTK_ParameterQueryTimesSet (parametername, CCTK_THORNSTRING); assert (ntimes >= 0); if (ntimes > 0) { int ptype; - const int* const ppval - = (const int*)CCTK_ParameterGet + const CCTK_INT* const ppval + = (const CCTK_INT*)CCTK_ParameterGet (parametername, CCTK_THORNSTRING, &ptype); assert (ppval); - const int pval = *ppval; - assert (ptype == PARAMETER_INT); + const CCTK_INT pval = *ppval; + assert (ptype == PARAMETER_INT || ptype == PARAMETER_BOOLEAN); return pval; } @@ -703,19 +813,32 @@ namespace CarpetIOASCII { - bool CheckForVariable (const cGH* const cgh, - const char* const varlist, const int vindex) + template<int outdim> + CCTK_REAL IOASCII<outdim> + ::GetRealParameter (const char* const parametertemplate, + const CCTK_REAL fallback) { - const int numvars = CCTK_NumVars(); - assert (vindex>=0 && vindex<numvars); - - vector<bool> flags(numvars); - - CCTK_TraverseString (varlist, SetFlag, &flags, CCTK_GROUP_OR_VAR); + char parametername[1000]; + snprintf (parametername, sizeof parametername, parametertemplate, outdim); + const int ntimes = + CCTK_ParameterQueryTimesSet (parametername, CCTK_THORNSTRING); + assert (ntimes >= 0); + if (ntimes > 0) { + int ptype; + const CCTK_REAL* const ppval + = (const CCTK_REAL*)CCTK_ParameterGet + (parametername, CCTK_THORNSTRING, &ptype); + assert (ppval); + const CCTK_REAL pval = *ppval; + assert (ptype == PARAMETER_REAL); + return pval; + } - return flags[vindex]; + return fallback; } + + void SetFlag (int index, const char* optstring, void* arg) { vector<bool>& flags = *(vector<bool>*)arg; @@ -733,10 +856,11 @@ namespace CarpetIOASCII { const int time, const vect<int,D>& org, const vect<int,DD>& dirs, - const int tl, const int rl, - const int c, const int ml, + const int m, + const int c, + const int tl, const CCTK_REAL coord_time, const vect<CCTK_REAL,D>& coord_lower, const vect<CCTK_REAL,D>& coord_upper) @@ -753,8 +877,12 @@ namespace CarpetIOASCII { assert (os.good()); os << "# iteration " << time << endl - << "# time level " << tl << " refinement level " << rl - << " component " << c << " multigrid level " << ml << endl + << "# refinement level " << rl + << " multigrid level " << ml + << " map " << m + << " component " << c + << " time level " << tl + << endl << "# column format: it tl rl c ml "; assert (D>=1 && D<=3); const char* const coords = "xyz"; @@ -770,25 +898,25 @@ namespace CarpetIOASCII { // Check whether the output origin is contained in the extent // of the data that should be output - vect<int,dim> org1(org); + ivect org1(org); for (int d=0; d<DD; ++d) org1[dirs[d]] = ext.lower()[d]; if (gfext.contains(org1)) { for (typename bbox<int,DD>::iteratorT it=ext.beginT(); it!=ext.endT(); ++it) { - vect<int,dim> index(org); + ivect index(org); for (int d=0; d<DD; ++d) index[dirs[d]] = (*it)[d]; os << time << " " << tl << " " << rl << " " << c << " " << ml - << " "; - for (int d=0; d<D; ++d) os << index[d] << " "; - os << " " << coord_time << " "; + << " "; + for (int d=0; d<D; ++d) os << " " << index[d]; + os << " " << coord_time << " "; for (int d=0; d<D; ++d) { assert (gfext.upper()[d] - gfext.lower()[d] != 0); - os << (coord_lower[d] + (index[d] - gfext.lower()[d]) - * (coord_upper[d] - coord_lower[d]) - / (gfext.upper()[d] - gfext.lower()[d])) << " "; + os << " " << (coord_lower[d] + (index[d] - gfext.lower()[d]) + * (coord_upper[d] - coord_lower[d]) + / (gfext.upper()[d] - gfext.lower()[d])); } - os << " "; + os << " "; switch (CCTK_VarTypeI(vi)) { #define WANT_NO_COMPLEX #define TYPECASE(N,T) \ @@ -837,7 +965,7 @@ namespace CarpetIOASCII { for (comm_state<dim> state; !state.done(); state.step()) { tmp->copy_from (state, gfdata, gfdata->extent()); } - WriteASCII (os, tmp, gfext, vi, time, org, dirs, tl, rl, c, ml, + WriteASCII (os, tmp, gfext, vi, time, org, dirs, rl, ml, m, c, tl, coord_time, coord_lower, coord_upper); delete tmp; @@ -861,10 +989,11 @@ namespace CarpetIOASCII { const int time, const vect<int,3>& org, const vect<int,1>& dirs, - const int tl, const int rl, - const int c, const int ml, + const int m, + const int c, + const int tl, const CCTK_REAL coord_time, const vect<CCTK_REAL,3>& coord_lower, const vect<CCTK_REAL,3>& coord_upper); @@ -877,10 +1006,11 @@ namespace CarpetIOASCII { const int time, const vect<int,3>& org, const vect<int,2>& dirs, - const int tl, const int rl, - const int c, const int ml, + const int m, + const int c, + const int tl, const CCTK_REAL coord_time, const vect<CCTK_REAL,3>& coord_lower, const vect<CCTK_REAL,3>& coord_upper); @@ -893,10 +1023,11 @@ namespace CarpetIOASCII { const int time, const vect<int,3>& org, const vect<int,3>& dirs, - const int tl, const int rl, - const int c, const int ml, + const int m, + const int c, + const int tl, const CCTK_REAL coord_time, const vect<CCTK_REAL,3>& coord_lower, const vect<CCTK_REAL,3>& coord_upper); diff --git a/Carpet/CarpetIOASCII/src/ioascii.h b/Carpet/CarpetIOASCII/src/ioascii.h index d2e5c549b..acd3ffb80 100644 --- a/Carpet/CarpetIOASCII/src/ioascii.h +++ b/Carpet/CarpetIOASCII/src/ioascii.h @@ -1,15 +1,18 @@ -/* $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.h,v 1.5 2003/11/05 16:18:37 schnetter Exp $ */ +/* $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.h,v 1.6 2004/01/25 14:57:29 schnetter Exp $ */ #ifndef CARPETIOASCII_H #define CARPETIOASCII_H +#include "cctk_Arguments.h" + #ifdef __cplusplus namespace CarpetIOASCII { extern "C" { #endif /* Scheduled functions */ - int CarpetIOASCIIStartup (void); + void CarpetIOASCIIStartup (void); + void CarpetIOASCIIInit (CCTK_ARGUMENTS); #ifdef __cplusplus } /* extern "C" */ diff --git a/Carpet/CarpetIOASCII/src/ioascii.hh b/Carpet/CarpetIOASCII/src/ioascii.hh index 504e7586f..c95b3ad71 100644 --- a/Carpet/CarpetIOASCII/src/ioascii.hh +++ b/Carpet/CarpetIOASCII/src/ioascii.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.hh,v 1.13 2002/09/01 14:52:25 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetIOASCII/src/ioascii.hh,v 1.14 2004/01/25 14:57:29 schnetter Exp $ #ifndef CARPETIOASCII_HH #define CARPETIOASCII_HH @@ -64,9 +64,12 @@ namespace CarpetIOASCII { static const char* GetStringParameter (const char* parametertemplate, const char* fallback); - static int GetIntParameter (const char* parametertemplate, int fallback); + static CCTK_INT GetIntParameter + (const char* parametertemplate, CCTK_INT fallback); + static CCTK_REAL GetRealParameter + (const char* parametertemplate, CCTK_REAL fallback); - }; // struct IOASCII + }; // struct IOASCII } // namespace CarpetIOASCII diff --git a/Carpet/CarpetInterp/src/interp.cc b/Carpet/CarpetInterp/src/interp.cc index 2855b3ec3..35a006aae 100644 --- a/Carpet/CarpetInterp/src/interp.cc +++ b/Carpet/CarpetInterp/src/interp.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetInterp/src/interp.cc,v 1.17 2003/11/13 17:29:47 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetInterp/src/interp.cc,v 1.18 2004/01/25 14:57:29 schnetter Exp $ #include <assert.h> #include <math.h> @@ -21,7 +21,7 @@ #include "interp.hh" extern "C" { - static char const * const rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetInterp/src/interp.cc,v 1.17 2003/11/13 17:29:47 schnetter Exp $"; + static char const * const rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetInterp/src/interp.cc,v 1.18 2004/01/25 14:57:29 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetInterp_interp_cc); } @@ -40,33 +40,38 @@ namespace CarpetInterp { -#define ind_rc(rl,c) ind_rc_(rl,minrl,maxrl,c,maxncomps,hh) - static inline int ind_rc_(int const rl, int const minrl, int const maxrl, +#define ind_rc(m,rl,c) ind_rc_(m,rl,minrl,maxrl,c,maxncomps,vhh) + static inline int ind_rc_(int const m, + int const rl, int const minrl, int const maxrl, int const c, int const maxncomps, - gh<dim> const * const hh) + vector<gh<dim> *> const hh) { + assert (m>=0 && m<maps); assert (rl>=minrl && rl<maxrl); - assert (minrl>=0 && maxrl<=hh->reflevels()); + assert (minrl>=0 && maxrl<=hh.at(m)->reflevels()); assert (c>=0 && c<maxncomps); - assert (maxncomps>=0 && maxncomps<=hh->components(rl)); + assert (maxncomps>=0 && maxncomps<=hh.at(m)->components(rl)); int const ind = rl * maxncomps + c; assert (ind>=0 && ind < (maxrl-minrl) * maxncomps); return ind; } -#define ind_prc(p,rl,c) ind_prc_(p,nprocs,rl,minrl,maxrl,c,maxncomps,cgh,hh) +#define ind_prc(p,m,rl,c) \ + ind_prc_(p,nprocs,m,rl,minrl,maxrl,c,maxncomps,cgh,vhh) static inline int ind_prc_(int const p, int const nprocs, + int const m, int const rl, int const minrl, int const maxrl, int const c, int const maxncomps, cGH const * const cgh, - gh<dim> const * const hh) + vector<gh<dim> *> const hh) { assert (p>=0 && p<nprocs); assert (nprocs==CCTK_nProcs(cgh)); + assert (m>=0 && m<maps); assert (rl>=minrl && rl<maxrl); - assert (minrl>=0 && maxrl<=hh->reflevels()); + assert (minrl>=0 && maxrl<=hh.at(m)->reflevels()); assert (c>=0 && c<maxncomps); - assert (maxncomps>=0 && maxncomps<=hh->components(rl)); + assert (maxncomps>=0 && maxncomps<=hh.at(m)->components(rl)); int const ind = (p * (maxrl-minrl) + rl) * maxncomps + c; assert (ind>=0 && ind < nprocs * (maxrl-minrl) * maxncomps); return ind; @@ -102,22 +107,8 @@ namespace CarpetInterp { - // Find out about the coordinates - const char * coord_system_name - = CCTK_CoordSystemName (coord_system_handle); - assert (coord_system_name); - rvect lower, upper, delta; - for (int d=0; d<dim; ++d) { - ierr = CCTK_CoordRange - (cgh, &lower[d], &upper[d], d+1, 0, coord_system_name); - assert (!ierr); - delta[d] = (upper[d] - lower[d]) / (hh->baseextent.shape()[d] - hh->baseextent.stride()[d]); - } - - assert (N_interp_points >= 0); - assert (interp_coords); - for (int d=0; d<dim; ++d) { - assert (N_interp_points==0 || interp_coords[d]); + if (is_meta_mode()) { + CCTK_WARN (0, "It is not possible to interpolate in meta mode"); } @@ -128,52 +119,57 @@ namespace CarpetInterp { int const nprocs = CCTK_nProcs (cgh); assert (myproc>=0 && myproc<nprocs); - int const minrl = reflevel==-1 ? 0 : reflevel; - int const maxrl = reflevel==-1 ? hh->reflevels() : reflevel+1; + // Multiple maps are not supported + // (because we don't know how to select a map) + assert (maps == 1); + const int m = 0; + + int const minrl = reflevel==-1 ? 0 : reflevel; + int const maxrl = reflevel==-1 ? vhh.at(m)->reflevels() : reflevel+1; + + // Multiple convergence levels are not supported + assert (mglevels == 1); int const ml = 0; + int maxncomps = 0; for (int rl=minrl; rl<maxrl; ++rl) { - maxncomps = max(maxncomps, hh->components(rl)); + maxncomps = max(maxncomps, vhh.at(m)->components(rl)); } -#if 0 - // Assert that all refinement levels have the same time - // TODO: interpolate in time - { - bool can_interp = true; - for (int rl=minrl; rl<maxrl; ++rl) { - int const tl = 0; - CCTK_REAL const time1 = tt->time (tl, rl, ml); - CCTK_REAL const time2 = cgh->cctk_time / cgh->cctk_delta_time; -// assert (fabs((time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time))) < 1e-12); - can_interp = can_interp && (fabs((time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time))) < 1e-12); - } - if (! can_interp) { - if (reflevel == -1) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot interpolate in global mode at iteration %d (time %g), because not all refinement levels exist at this time. Interpolation in time is not yet implemented.", - cgh->cctk_iteration, (double)cgh->cctk_time); - } else { - // called in level mode at a time where the level does not - // exist - CCTK_WARN (0, "internal error"); - } - assert (0); - } - } -#endif - // Find the time interpolation order int partype; - void const * const parptr = CCTK_ParameterGet ("prolongation_order_time", "Carpet", &partype); + void const * const parptr + = CCTK_ParameterGet ("prolongation_order_time", "Carpet", &partype); assert (parptr); assert (partype == PARAMETER_INTEGER); int const prolongation_order_time = * (CCTK_INT const *) parptr; + // Find out about the coordinates + const char * coord_system_name + = CCTK_CoordSystemName (coord_system_handle); + assert (coord_system_name); + rvect lower, upper, delta; + for (int d=0; d<dim; ++d) { + ierr = CCTK_CoordRange + (cgh, &lower[d], &upper[d], d+1, 0, coord_system_name); + assert (!ierr); + const ibbox & baseext = vhh.at(m)->baseextent; + delta[d] + = (upper[d] - lower[d]) / (baseext.upper()[d] - baseext.lower()[d]); + } + + assert (N_interp_points >= 0); + assert (interp_coords); + for (int d=0; d<dim; ++d) { + assert (N_interp_points==0 || interp_coords[d]); + } + + + // Assign interpolation points to components vector<int> rlev (N_interp_points); // refinement level of point n vector<int> home (N_interp_points); // component of point n @@ -189,20 +185,20 @@ namespace CarpetInterp { } // Find the component that this grid point belongs to - rlev[n] = -1; - home[n] = -1; + rlev.at(n) = -1; + home.at(n) = -1; for (int rl=maxrl-1; rl>=minrl; --rl) { - const int fact = maxreflevelfact / ipow(reffact,rl); + const int fact = maxreflevelfact * ipow(mgfact, basemglevel + mglevel) / ipow(reffact, rl); CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; - ivect const ipos = ivect(map(rfloor, (pos - lower) / (delta * fact) + 0.5)) * fact; - assert (all(ipos % hh->bases[rl][ml].stride() == 0)); + ivect const ipos = ivect(::map(rfloor, (pos - lower) / (delta * fact) + 0.5)) * fact; + assert (all(ipos % vhh.at(m)->bases.at(rl).at(ml).stride() == 0)); // TODO: use something faster than a linear search - for (int c=0; c<hh->components(rl); ++c) { - if (hh->extents[rl][c][ml].contains(ipos)) { - rlev[n] = rl; - home[n] = c; + for (int c=0; c<vhh.at(m)->components(rl); ++c) { + if (vhh.at(m)->extents.at(rl).at(c).at(ml).contains(ipos)) { + rlev.at(n) = rl; + home.at(n) = c; goto found; } } @@ -211,31 +207,33 @@ namespace CarpetInterp { "Interpolation point #%d at [%g,%g,%g] is not on any grid patch", n, pos[0], pos[1], pos[2]); found: - assert (rlev[n]>=minrl && rlev[n]<maxrl); - assert (home[n]>=0 && home[n]<hh->components(rlev[n])); - ++ homecnts [ind_rc(rlev[n], home[n])]; + assert (rlev.at(n)>=minrl && rlev.at(n)<maxrl); + assert (home.at(n)>=0 && home.at(n)<vhh.at(m)->components(rlev.at(n))); + ++ homecnts.at(ind_rc(m, rlev.at(n), home.at(n))); } // for n // Communicate counts vector<int> allhomecnts(nprocs * (maxrl-minrl) * maxncomps); - MPI_Allgather (&homecnts [0], (maxrl-minrl) * maxncomps, MPI_INT, - &allhomecnts[0], (maxrl-minrl) * maxncomps, MPI_INT, comm); + MPI_Allgather + (&homecnts .at(0), (maxrl-minrl) * maxncomps, MPI_INT, + &allhomecnts.at(0), (maxrl-minrl) * maxncomps, MPI_INT, comm); // Create coordinate patches - vector<data<CCTK_REAL,dim> > allcoords (nprocs * (maxrl-minrl) * maxncomps, -1); + vector<data<CCTK_REAL,dim> > allcoords + (nprocs * (maxrl-minrl) * maxncomps); for (int p=0; p<nprocs; ++p) { for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { + for (int c=0; c<vhh.at(m)->components(rl); ++c) { ivect lo (0); ivect up (1); - up[0] = allhomecnts[ind_prc(p,rl,c)]; + up[0] = allhomecnts.at(ind_prc(p,m,rl,c)); up[1] = dim; ivect str (1); ibbox extent (lo, up-str, str); - allcoords [ind_prc(p,rl,c)].allocate (extent, p); + allcoords.at(ind_prc(p,m,rl,c)).allocate (extent, p); } } } @@ -244,22 +242,22 @@ namespace CarpetInterp { { vector<int> tmpcnts ((maxrl-minrl) * maxncomps); for (int n=0; n<N_interp_points; ++n) { - int const rl = rlev[n]; - int const c = home[n]; + int const rl = rlev.at(n); + int const c = home.at(n); assert (rl>=minrl && rl<maxrl); - assert (c>=0 && c<hh->components(rl)); - assert (tmpcnts[ind_rc(rl,c)] >= 0); - assert (tmpcnts[ind_rc(rl,c)] < homecnts[ind_rc(rl,c)]); + assert (c>=0 && c<vhh.at(m)->components(rl)); + assert (tmpcnts.at(ind_rc(m,rl,c)) >= 0); + assert (tmpcnts.at(ind_rc(m,rl,c)) < homecnts.at(ind_rc(m,rl,c))); assert (dim==3); for (int d=0; d<dim; ++d) { - allcoords[ind_prc(myproc,rl,c)][ivect(tmpcnts[ind_rc(rl,c)],d,0)] + allcoords.at(ind_prc(myproc,m,rl,c))[ivect(tmpcnts.at(ind_rc(m,rl,c)),d,0)] = static_cast<CCTK_REAL const *>(interp_coords[d])[n]; } - ++ tmpcnts[c + (rl-minrl)*maxncomps]; + ++ tmpcnts.at(c + (rl-minrl)*maxncomps); } for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - assert (tmpcnts[ind_rc(rl,c)] == homecnts[ind_rc(rl,c)]); + for (int c=0; c<vhh.at(m)->components(rl); ++c) { + assert (tmpcnts.at(ind_rc(m,rl,c)) == homecnts.at(ind_rc(m,rl,c))); } } } @@ -268,9 +266,9 @@ namespace CarpetInterp { for (comm_state<dim> state; !state.done(); state.step()) { for (int p=0; p<nprocs; ++p) { for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - allcoords[ind_prc(p,rl,c)].change_processor - (state, hh->processors[rl][c]); + for (int c=0; c<vhh.at(m)->components(rl); ++c) { + allcoords.at(ind_prc(p,m,rl,c)).change_processor + (state, vhh.at(m)->processors.at(rl).at(c)); } } } @@ -279,17 +277,19 @@ namespace CarpetInterp { // Create output patches - vector<data<CCTK_REAL,dim> > alloutputs (nprocs * (maxrl-minrl) * maxncomps, -1); + vector<data<CCTK_REAL,dim> > alloutputs + (nprocs * (maxrl-minrl) * maxncomps, -1); for (int p=0; p<nprocs; ++p) { for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { + for (int c=0; c<vhh.at(m)->components(rl); ++c) { ivect lo (0); ivect up (1); - up[0] = allhomecnts[ind_prc(p,rl,c)]; + up[0] = allhomecnts.at(ind_prc(p,m,rl,c)); up[1] = N_output_arrays; ivect str (1); ibbox extent (lo, up-str, str); - alloutputs[ind_prc(p,rl,c)].allocate (extent, hh->processors[rl][c]); + alloutputs.at(ind_prc(p,m,rl,c)).allocate + (extent, vhh.at(m)->processors.at(rl).at(c)); } } } @@ -300,188 +300,171 @@ namespace CarpetInterp { // Do the local interpolation // int overall_ierr = 0; - int const saved_reflevel = reflevel; - int const saved_mglevel = mglevel; - int const saved_component = component; - if (component!=-1) { - set_component ((cGH*)cgh, -1); - } - if (mglevel!=-1) { - set_mglevel ((cGH*)cgh, -1); - } - if (reflevel!=-1) { - set_reflevel ((cGH*)cgh, -1); - } - BEGIN_REFLEVEL_LOOP(cgh) { - if (reflevel>=minrl && reflevel<maxrl) { - BEGIN_MGLEVEL_LOOP(cgh) { - + BEGIN_GLOBAL_MODE(cgh) { + + BEGIN_REFLEVEL_LOOP(cgh) { + if (reflevel>=minrl && reflevel<maxrl) { + // Number of necessary time levels int const tl = 0; - CCTK_REAL const time1 = tt->time (tl, reflevel, mglevel); + CCTK_REAL const time1 + = vtt.at(m)->time (tl, reflevel, mglevel); CCTK_REAL const time2 = cgh->cctk_time / cgh->cctk_delta_time; bool const need_time_interp - = fabs((time1 - time2) / (fabs(time1) + fabs(time2) + fabs(cgh->cctk_delta_time))) > 1e-12; + = fabs((time1 - time2) / (fabs(time1) + fabs(time2) + + fabs(cgh->cctk_delta_time))) > 1e-12; int const num_tl = need_time_interp ? prolongation_order_time + 1 : 1; - - BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { + + BEGIN_MAP_LOOP(cgh, CCTK_GF) { + BEGIN_LOCAL_COMPONENT_LOOP(cgh, CCTK_GF) { - // Find out about the local geometry - ivect lsh; - rvect coord_origin, coord_delta; - for (int d=0; d<dim; ++d) { - lsh[d] = cgh->cctk_lsh[d]; - coord_delta[d] = cgh->cctk_delta_space[d] / cgh->cctk_levfac[d]; - coord_origin[d] = cgh->cctk_origin_space[d] + (1.0 * cgh->cctk_levoff[d] / cgh->cctk_levoffdenom[d] + cgh->cctk_lbnd[d]) * coord_delta[d]; - } + // Find out about the local geometry + ivect lsh; + rvect coord_origin, coord_delta; + for (int d=0; d<dim; ++d) { + lsh[d] = cgh->cctk_lsh[d]; + coord_delta[d] = cgh->cctk_delta_space[d] / cgh->cctk_levfac[d]; + coord_origin[d] = cgh->cctk_origin_space[d] + coord_delta[d] * (cgh->cctk_lbnd[d] + 1.0 * cgh->cctk_levoff[d] / cgh->cctk_levoffdenom[d]) * coord_delta[d]; + } - // Find out about the grid functions - vector<CCTK_INT> input_array_type_codes(N_input_arrays); - vector<const void *> input_arrays(N_input_arrays * num_tl); - for (int n=0; n<N_input_arrays; ++n) { - if (input_array_variable_indices[n] == -1) { + // Find out about the grid functions + vector<CCTK_INT> input_array_type_codes(N_input_arrays); + vector<const void *> input_arrays(N_input_arrays * num_tl); + for (int n=0; n<N_input_arrays; ++n) { + if (input_array_variable_indices[n] == -1) { - // Ignore this entry - input_array_type_codes[tl+n*num_tl] = -1; - input_arrays[tl+n*num_tl] = 0; + // Ignore this entry + input_array_type_codes.at(tl+n*num_tl) = -1; + input_arrays.at(tl+n*num_tl) = 0; - } else { + } else { - int const vi = input_array_variable_indices[n]; - assert (vi>=0 && vi<CCTK_NumVars()); + int const vi = input_array_variable_indices[n]; + assert (vi>=0 && vi<CCTK_NumVars()); - int const gi = CCTK_GroupIndexFromVarI (vi); - assert (gi>=0 && gi<CCTK_NumGroups()); + int const gi = CCTK_GroupIndexFromVarI (vi); + assert (gi>=0 && gi<CCTK_NumGroups()); - cGroup group; - ierr = CCTK_GroupData (gi, &group); - assert (!ierr); - assert (group.grouptype == CCTK_GF); - assert (group.dim == dim); - assert (group.disttype == CCTK_DISTRIB_DEFAULT); - assert (group.stagtype == 0); // not staggered + cGroup group; + ierr = CCTK_GroupData (gi, &group); + assert (!ierr); + assert (group.grouptype == CCTK_GF); + assert (group.dim == dim); + assert (group.disttype == CCTK_DISTRIB_DEFAULT); + assert (group.stagtype == 0); // not staggered - assert (group.numtimelevels >= num_tl); + assert (group.numtimelevels >= num_tl); - for (int tl=0; tl<num_tl; ++tl) { - input_array_type_codes[tl+n*num_tl] = group.vartype; - input_arrays[tl+n*num_tl] = CCTK_VarDataPtrI (cgh, tl, vi); - } + for (int tl=0; tl<num_tl; ++tl) { + input_array_type_codes.at(tl+n*num_tl) = group.vartype; + input_arrays.at(tl+n*num_tl) = CCTK_VarDataPtrI (cgh, tl, vi); + } - } - } // for input arrays + } + } // for input arrays - // Work on the data from all processors - for (int p=0; p<nprocs; ++p) { - assert (allcoords[ind_prc(p,reflevel,component)].owns_storage()); - assert (allhomecnts[ind_prc(p,reflevel,component)] - == allcoords[ind_prc(p,reflevel,component)].shape()[0]); - assert (allhomecnts[ind_prc(p,reflevel,component)] - == alloutputs[ind_prc(p,reflevel,component)].shape()[0]); - - int const npoints = allhomecnts[ind_prc(p,reflevel,component)]; - - // Do the processor-local interpolation - vector<const void *> tmp_interp_coords (dim); - for (int d=0; d<dim; ++d) { - tmp_interp_coords[d] - = &allcoords[ind_prc(p,reflevel,component)][ivect(0,d,0)]; - } - vector<void *> tmp_output_arrays (N_output_arrays * num_tl); - if (need_time_interp) { - for (int m=0; m<N_output_arrays; ++m) { - assert (output_array_type_codes[m] == CCTK_VARIABLE_REAL); - for (int tl=0; tl<num_tl; ++tl) { - tmp_output_arrays[tl+m*num_tl] = new CCTK_REAL [npoints]; - } + // Work on the data from all processors + for (int p=0; p<nprocs; ++p) { + assert (allcoords.at(ind_prc(p,m,reflevel,component)).owns_storage()); + assert (allhomecnts.at(ind_prc(p,m,reflevel,component)) == allcoords.at(ind_prc(p,m,reflevel,component)).shape()[0]); + assert (allhomecnts.at(ind_prc(p,m,reflevel,component)) == alloutputs.at(ind_prc(p,m,reflevel,component)).shape()[0]); + + int const npoints = allhomecnts.at(ind_prc(p,m,reflevel,component)); + + // Do the processor-local interpolation + vector<const void *> tmp_interp_coords (dim); + for (int d=0; d<dim; ++d) { + tmp_interp_coords.at(d) = &allcoords.at(ind_prc(p,m,reflevel,component))[ivect(0,d,0)]; } - } else { - for (int m=0; m<N_output_arrays; ++m) { - assert (output_array_type_codes[m] == CCTK_VARIABLE_REAL); - tmp_output_arrays[m] - = &alloutputs[ind_prc(p,reflevel,component)][ivect(0,m,0)]; + vector<void *> tmp_output_arrays (N_output_arrays * num_tl); + if (need_time_interp) { + for (int j=0; j<N_output_arrays; ++j) { + assert (output_array_type_codes[j] == CCTK_VARIABLE_REAL); + for (int tl=0; tl<num_tl; ++tl) { + tmp_output_arrays.at(tl+j*num_tl) + = new CCTK_REAL [npoints]; + } + } + } else { + for (int j=0; j<N_output_arrays; ++j) { + assert (output_array_type_codes[j] == CCTK_VARIABLE_REAL); + tmp_output_arrays.at(j) = &alloutputs.at(ind_prc(p,m,reflevel,component))[ivect(0,j,0)]; + } } - } - ierr = CCTK_InterpLocalUniform - (N_dims, local_interp_handle, param_table_handle, - &coord_origin[0], &coord_delta[0], - npoints, - interp_coords_type_code, &tmp_interp_coords[0], - N_input_arrays * num_tl, &lsh[0], - &input_array_type_codes[0], &input_arrays[0], - N_output_arrays * num_tl, - output_array_type_codes, &tmp_output_arrays[0]); - if (ierr) { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "The local interpolator returned the error code %d", ierr); - } - overall_ierr = min(overall_ierr, ierr); - - // Interpolate in time, if necessary - if (need_time_interp) { - CCTK_REAL const time = cgh->cctk_time / cgh->cctk_delta_time; - vector<CCTK_REAL> times(num_tl); - for (int tl=0; tl<num_tl; ++tl) { - times[tl] = tt->time (tl, reflevel, mglevel); - } - vector<CCTK_REAL> tfacs(num_tl); - switch (num_tl) { - case 1: - // no interpolation - assert (fabs((time - times[0]) / fabs(time + times[0] + cgh->cctk_delta_time)) < 1e-12); - tfacs[0] = 1.0; - break; - case 2: - // linear (2-point) interpolation - tfacs[0] = (time - times[1]) / (times[0] - times[1]); - tfacs[1] = (time - times[0]) / (times[1] - times[0]); - break; - case 3: - // quadratic (3-point) interpolation - tfacs[0] = (time - times[1]) * (time - times[2]) / ((times[0] - times[1]) * (times[0] - times[2])); - tfacs[1] = (time - times[0]) * (time - times[2]) / ((times[1] - times[0]) * (times[1] - times[2])); - tfacs[2] = (time - times[0]) * (time - times[1]) / ((times[2] - times[0]) * (times[2] - times[1])); - break; - default: - assert (0); + ierr = CCTK_InterpLocalUniform + (N_dims, local_interp_handle, param_table_handle, + &coord_origin[0], &coord_delta[0], + npoints, + interp_coords_type_code, &tmp_interp_coords[0], + N_input_arrays * num_tl, &lsh[0], + &input_array_type_codes[0], &input_arrays[0], + N_output_arrays * num_tl, + output_array_type_codes, &tmp_output_arrays[0]); + if (ierr) { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "The local interpolator returned the error code %d", ierr); } - for (int m=0; m<N_output_arrays; ++m) { - assert (output_array_type_codes[m] == CCTK_VARIABLE_REAL); - for (int k=0; k<npoints; ++k) { - CCTK_REAL & dest = alloutputs[ind_prc(p,reflevel,component)][ivect(k,m,0)]; - dest = 0; + overall_ierr = min(overall_ierr, ierr); + + // Interpolate in time, if necessary + if (need_time_interp) { + CCTK_REAL const time = cgh->cctk_time / cgh->cctk_delta_time; + vector<CCTK_REAL> times(num_tl); + for (int tl=0; tl<num_tl; ++tl) { + times.at(tl) = vtt.at(m)->time (tl, reflevel, mglevel); + } + vector<CCTK_REAL> tfacs(num_tl); + switch (num_tl) { + case 1: + // no interpolation + assert (fabs((time - times.at(0)) / fabs(time + times.at(0) + cgh->cctk_delta_time)) < 1e-12); + tfacs.at(0) = 1.0; + break; + case 2: + // linear (2-point) interpolation + tfacs.at(0) = (time - times.at(1)) / (times.at(0) - times.at(1)); + tfacs.at(1) = (time - times.at(0)) / (times.at(1) - times.at(0)); + break; + case 3: + // quadratic (3-point) interpolation + tfacs.at(0) = (time - times.at(1)) * (time - times.at(2)) / ((times.at(0) - times.at(1)) * (times.at(0) - times.at(2))); + tfacs.at(1) = (time - times.at(0)) * (time - times.at(2)) / ((times.at(1) - times.at(0)) * (times.at(1) - times.at(2))); + tfacs.at(2) = (time - times.at(0)) * (time - times.at(1)) / ((times.at(2) - times.at(0)) * (times.at(2) - times.at(1))); + break; + default: + assert (0); + } + for (int j=0; j<N_output_arrays; ++j) { + assert (output_array_type_codes[j] == CCTK_VARIABLE_REAL); + for (int k=0; k<npoints; ++k) { + CCTK_REAL & dest = alloutputs.at(ind_prc(p,m,reflevel,component))[ivect(k,j,0)]; + dest = 0; + for (int tl=0; tl<num_tl; ++tl) { + CCTK_REAL const src = ((CCTK_REAL const *)tmp_output_arrays.at(tl+j*num_tl))[k]; + dest += tfacs[tl] * src; + } + } for (int tl=0; tl<num_tl; ++tl) { - CCTK_REAL const src = ((CCTK_REAL const *)tmp_output_arrays[tl+m*num_tl])[k]; - dest += tfacs[tl] * src; + delete [] (CCTK_REAL *)tmp_output_arrays.at(tl+j*num_tl); } } - for (int tl=0; tl<num_tl; ++tl) { - delete [] (CCTK_REAL *)tmp_output_arrays[tl+m*num_tl]; - } } - } - } // for processors + } // for processors - } END_LOCAL_COMPONENT_LOOP; - } END_MGLEVEL_LOOP; - } // if reflevel active - } END_REFLEVEL_LOOP; - if (saved_reflevel!=-1) { - set_reflevel ((cGH*)cgh, saved_reflevel); - } - if (saved_mglevel!=-1) { - set_mglevel ((cGH*)cgh, saved_mglevel); - } - if (saved_component!=-1) { - set_component ((cGH*)cgh, saved_component); - } + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; + + } // if reflevel active + } END_REFLEVEL_LOOP; + + } END_GLOBAL_MODE; @@ -489,8 +472,8 @@ namespace CarpetInterp { for (comm_state<dim> state; !state.done(); state.step()) { for (int p=0; p<nprocs; ++p) { for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - alloutputs[ind_prc(p,rl,c)].change_processor (state, p); + for (int c=0; c<vhh.at(m)->components(rl); ++c) { + alloutputs.at(ind_prc(p,m,rl,c)).change_processor (state, p); } } } @@ -502,19 +485,18 @@ namespace CarpetInterp { { vector<int> tmpcnts ((maxrl-minrl) * maxncomps); for (int n=0; n<N_interp_points; ++n) { - int const rl = rlev[n]; - int const c = home[n]; - for (int m=0; m<N_output_arrays; ++m) { + int const rl = rlev.at(n); + int const c = home.at(n); + for (int j=0; j<N_output_arrays; ++j) { assert (interp_coords_type_code == CCTK_VARIABLE_REAL); - assert (alloutputs[ind_prc(myproc,rl,c)].owns_storage()); - static_cast<CCTK_REAL *>(output_arrays[m])[n] = - alloutputs[ind_prc(myproc,rl,c)][ivect(tmpcnts[ind_rc(rl,c)],m,0)]; + assert (alloutputs.at(ind_prc(myproc,m,rl,c)).owns_storage()); + static_cast<CCTK_REAL *>(output_arrays[j])[n] = alloutputs.at(ind_prc(myproc,m,rl,c))[ivect(tmpcnts.at(ind_rc(m,rl,c)),j,0)]; } - ++ tmpcnts[ind_rc(rl,c)]; + ++ tmpcnts.at(ind_rc(m,rl,c)); } for (int rl=minrl; rl<maxrl; ++rl) { - for (int c=0; c<hh->components(rl); ++c) { - assert (tmpcnts[ind_rc(rl,c)] == homecnts[ind_rc(rl,c)]); + for (int c=0; c<vhh.at(m)->components(rl); ++c) { + assert (tmpcnts.at(ind_rc(m,rl,c)) == homecnts.at(ind_rc(m,rl,c))); } } } @@ -522,8 +504,8 @@ namespace CarpetInterp { int global_overall_ierr; - MPI_Allreduce (&overall_ierr, &global_overall_ierr, 1, MPI_INT, - MPI_MIN, comm); + MPI_Allreduce + (&overall_ierr, &global_overall_ierr, 1, MPI_INT, MPI_MIN, comm); diff --git a/Carpet/CarpetLib/interface.ccl b/Carpet/CarpetLib/interface.ccl index b2c8cbde0..717c5ec06 100644 --- a/Carpet/CarpetLib/interface.ccl +++ b/Carpet/CarpetLib/interface.ccl @@ -1,5 +1,5 @@ # Interface definition for thorn CarpetLib -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/interface.ccl,v 1.3 2003/06/18 18:28:07 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/interface.ccl,v 1.4 2004/01/25 14:57:29 schnetter Exp $ implements: CarpetLib @@ -18,3 +18,5 @@ includes header: gf.hh in gf.hh includes header: ggf.hh in ggf.hh includes header: gh.hh in gh.hh includes header: th.hh in th.hh + +includes header: operators.hh in operators.hh diff --git a/Carpet/CarpetLib/src/bbox.cc b/Carpet/CarpetLib/src/bbox.cc index bbfcc3df5..6decccabe 100644 --- a/Carpet/CarpetLib/src/bbox.cc +++ b/Carpet/CarpetLib/src/bbox.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.cc,v 1.18 2003/11/13 16:03:58 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/bbox.cc,v 1.19 2004/01/25 14:57:29 schnetter Exp $ #include <assert.h> @@ -249,20 +249,16 @@ typename bbox<T,D>::iteratorT bbox<T,D>::endT () const { template<class T,int D> void bbox<T,D>::input (istream& is) { skipws (is); - assert (is.peek() == '('); - is.get(); + consume (is, '('); is >> _lower; skipws (is); - assert (is.peek() == ':'); - is.get(); + consume (is, ':'); is >> _upper; skipws (is); - assert (is.peek() == ':'); - is.get(); + consume (is, ':'); is >> _stride; skipws (is); - assert (is.peek() == ')'); - is.get(); + consume (is, ')'); } diff --git a/Carpet/CarpetLib/src/data.cc b/Carpet/CarpetLib/src/data.cc index f6158bae8..6f65158b8 100644 --- a/Carpet/CarpetLib/src/data.cc +++ b/Carpet/CarpetLib/src/data.cc @@ -1,10 +1,15 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/data.cc,v 1.38 2004/01/21 16:32:04 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/data.cc,v 1.39 2004/01/25 14:57:29 schnetter Exp $ #include <assert.h> #include <limits.h> +#include <stdlib.h> +#include <math.h> +#include <algorithm> #include <iostream> +#include <sstream> #include <string> +#include <vector> #include <mpi.h> @@ -34,16 +39,17 @@ static int nexttag () // Constructors template<class T, int D> -data<T,D>::data (const int varindex_) - : gdata<D>(varindex_), +data<T,D>::data (const int varindex_, const operator_type transport_operator_) + : gdata<D>(varindex_, transport_operator_), _storage(0), comm_active(false), tag(nexttag()) { } template<class T, int D> -data<T,D>::data (const int varindex_, const ibbox& extent_, const int proc_) - : gdata<D>(varindex_), +data<T,D>::data (const int varindex_, const operator_type transport_operator_, + const ibbox& extent_, const int proc_) + : gdata<D>(varindex_, transport_operator_), _storage(0), comm_active(false), tag(nexttag()) @@ -59,8 +65,11 @@ data<T,D>::~data () { // Pseudo constructors template<class T, int D> -data<T,D>* data<T,D>::make_typed (const int varindex_) const { - return new data(varindex_); +data<T,D>* data<T,D>::make_typed (const int varindex_, + const operator_type transport_operator_) + const +{ + return new data(varindex_, transport_operator_); } @@ -107,7 +116,7 @@ void data<T,D>::transfer_from (gdata<D>* gsrc) { data* src = (data*)gsrc; assert (!_storage); *this = *src; - *src = data(this->varindex); + *src = data(this->varindex, this->transport_operator); } @@ -159,9 +168,12 @@ void data<T,D>::change_processor_recv (const int newproc, void* const mem) _storage = (T*)mem; } + const double wtime1 = MPI_Wtime(); T dummy; MPI_Irecv (_storage, this->_size, dist::datatype(dummy), this->_proc, this->tag, dist::comm, &request); + const double wtime2 = MPI_Wtime(); + this->wtime_irecv += wtime2 - wtime1; } else if (rank == this->_proc) { // copy to other processor @@ -197,9 +209,12 @@ void data<T,D>::change_processor_send (const int newproc, void* const mem) assert (!mem); assert (_storage); + const double wtime1 = MPI_Wtime(); T dummy; MPI_Isend (_storage, this->_size, dist::datatype(dummy), newproc, this->tag, dist::comm, &request); + const double wtime2 = MPI_Wtime(); + this->wtime_isend += wtime2 - wtime1; } else { assert (!mem); @@ -227,8 +242,11 @@ void data<T,D>::change_processor_wait (const int newproc, void* const mem) if (rank == newproc) { // copy from other processor + const double wtime1 = MPI_Wtime(); MPI_Status status; MPI_Wait (&request, &status); + const double wtime2 = MPI_Wtime(); + this->wtime_irecvwait += wtime2 - wtime1; } else if (rank == this->_proc) { // copy to other processor @@ -236,8 +254,11 @@ void data<T,D>::change_processor_wait (const int newproc, void* const mem) assert (!mem); assert (_storage); + const double wtime1 = MPI_Wtime(); MPI_Status status; MPI_Wait (&request, &status); + const double wtime2 = MPI_Wtime(); + this->wtime_isendwait += wtime2 - wtime1; if (this->_owns_storage) { delete [] _storage; @@ -553,6 +574,15 @@ extern "C" { const int srcbbox[3][3], const int dstbbox[3][3], const int regbbox[3][3]); + void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_rf2) + (const CCTK_REAL8* src1, const CCTK_REAL8& t1, + const CCTK_REAL8* src2, const CCTK_REAL8& t2, + const int& srciext, const int& srcjext, const int& srckext, + CCTK_REAL8* dst, const CCTK_REAL8& t, + const int& dstiext, const int& dstjext, const int& dstkext, + const int srcbbox[3][3], + const int dstbbox[3][3], + const int regbbox[3][3]); void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_o3) (const CCTK_REAL8* src1, const CCTK_REAL8& t1, const CCTK_REAL8* src2, const CCTK_REAL8& t2, @@ -562,6 +592,15 @@ extern "C" { const int srcbbox[3][3], const int dstbbox[3][3], const int regbbox[3][3]); + void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_o3_rf2) + (const CCTK_REAL8* src1, const CCTK_REAL8& t1, + const CCTK_REAL8* src2, const CCTK_REAL8& t2, + const int& srciext, const int& srcjext, const int& srckext, + CCTK_REAL8* dst, const CCTK_REAL8& t, + const int& dstiext, const int& dstjext, const int& dstkext, + const int srcbbox[3][3], + const int dstbbox[3][3], + const int regbbox[3][3]); void CCTK_FCALL CCTK_FNAME(prolongate_3d_real8_2tl_minmod) (const CCTK_REAL8* src1, const CCTK_REAL8& t1, const CCTK_REAL8* src2, const CCTK_REAL8& t2, @@ -651,6 +690,8 @@ void data<CCTK_REAL8,3> const int order_space, const int order_time) { + const CCTK_REAL eps = 1.0e-10; + assert (has_storage()); assert (all(box.lower()>=extent().lower())); assert (all(box.upper()<=extent().upper())); @@ -694,8 +735,46 @@ void data<CCTK_REAL8,3> regbbox[2][d] = box.stride()[d]; } + // Check that the times are consistent + assert (times.size() > 0); + CCTK_REAL min_time = times[0]; + CCTK_REAL max_time = times[0]; + for (size_t tl=1; tl<times.size(); ++tl) { + min_time = min(min_time, times[tl]); + max_time = max(max_time, times[tl]); + } + if (time < min_time - eps || time > max_time + eps) { + ostringstream buf; + buf << "Internal error: extrapolation in time." + << " time=" << time + << " times=" << times; + CCTK_WARN (0, buf.str().c_str()); + } + + // Is it necessary to interpolate in time? + if (times.size() > 1) { + for (size_t tl=0; tl<times.size(); ++tl) { + assert (abs(1.5) > 1.4); + if (abs(times[tl] - time) < eps) { + // It is not. + vector<const gdata<3>*> my_gsrcs(1); + vector<CCTK_REAL> my_times(1); + my_gsrcs[0] = gsrcs[tl]; + my_times[0] = times[tl]; + const int my_order_time = 0; + this->interpolate_from_innerloop + (my_gsrcs, my_times, box, time, order_space, my_order_time); + return; + } + } + } + assert (all(dext.stride() == box.stride())); if (all(sext.stride() < dext.stride())) { + // Restrict + + assert (times.size() == 1); + assert (abs(times[0] - time) < eps); switch (transport_operator) { @@ -722,9 +801,10 @@ void data<CCTK_REAL8,3> default: assert (0); - } // switch (prolong_method) + } // switch (transport_operator) } else if (all(sext.stride() > dext.stride())) { + // Prolongate switch (transport_operator) { @@ -732,6 +812,8 @@ void data<CCTK_REAL8,3> switch (order_time) { case 0: + assert (times.size() == 1); + assert (abs(times[0] - time) < eps); assert (srcs.size()>=1); switch (order_space) { case 0: @@ -789,23 +871,43 @@ void data<CCTK_REAL8,3> switch (order_space) { case 0: case 1: - CCTK_FNAME(prolongate_3d_real8_2tl) - ((const CCTK_REAL8*)srcs[0]->storage(), times[0], - (const CCTK_REAL8*)srcs[1]->storage(), times[1], - srcshp[0], srcshp[1], srcshp[2], - (CCTK_REAL8*)storage(), time, - dstshp[0], dstshp[1], dstshp[2], - srcbbox, dstbbox, regbbox); + if (all (sext.stride() == dext.stride() * 2)) { + CCTK_FNAME(prolongate_3d_real8_2tl_rf2) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + } else { + CCTK_FNAME(prolongate_3d_real8_2tl) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + } break; case 2: case 3: - CCTK_FNAME(prolongate_3d_real8_2tl_o3) - ((const CCTK_REAL8*)srcs[0]->storage(), times[0], - (const CCTK_REAL8*)srcs[1]->storage(), times[1], - srcshp[0], srcshp[1], srcshp[2], - (CCTK_REAL8*)storage(), time, - dstshp[0], dstshp[1], dstshp[2], - srcbbox, dstbbox, regbbox); + if (all (sext.stride() == dext.stride() * 2)) { + CCTK_FNAME(prolongate_3d_real8_2tl_o3_rf2) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + } else { + CCTK_FNAME(prolongate_3d_real8_2tl_o3) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + } break; case 4: case 5: @@ -893,40 +995,75 @@ void data<CCTK_REAL8,3> case op_TVD: switch (order_time) { case 0: - CCTK_FNAME(prolongate_3d_real8_minmod) - ((const CCTK_REAL8*)srcs[0]->storage(), - srcshp[0], srcshp[1], srcshp[2], - (CCTK_REAL8*)storage(), - dstshp[0], dstshp[1], dstshp[2], - srcbbox, dstbbox, regbbox); + assert (times.size() == 1); + assert (abs(times[0] - time) < eps); + switch (order_space) { + case 0: + case 1: + CCTK_WARN (0, "There is no stencil for op=""TVD"" with order_space=1"); + break; + case 2: + case 3: + CCTK_FNAME(prolongate_3d_real8_minmod) + ((const CCTK_REAL8*)srcs[0]->storage(), + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + break; + default: + assert (0); + } break; case 1: - CCTK_FNAME(prolongate_3d_real8_2tl_minmod) - ((const CCTK_REAL8*)srcs[0]->storage(), times[0], - (const CCTK_REAL8*)srcs[1]->storage(), times[1], - srcshp[0], srcshp[1], srcshp[2], - (CCTK_REAL8*)storage(), time, - dstshp[0], dstshp[1], dstshp[2], - srcbbox, dstbbox, regbbox); + switch (order_space) { + case 0: + case 1: + CCTK_WARN (0, "There is no stencil for op=""TVD"" with order_space=1"); + break; + case 2: + case 3: + CCTK_FNAME(prolongate_3d_real8_2tl_minmod) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + break; + default: + assert (0); + } break; case 2: - CCTK_FNAME(prolongate_3d_real8_3tl_minmod) - ((const CCTK_REAL8*)srcs[0]->storage(), times[0], - (const CCTK_REAL8*)srcs[1]->storage(), times[1], - (const CCTK_REAL8*)srcs[2]->storage(), times[2], - srcshp[0], srcshp[1], srcshp[2], - (CCTK_REAL8*)storage(), time, - dstshp[0], dstshp[1], dstshp[2], - srcbbox, dstbbox, regbbox); + switch (order_space) { + case 0: + case 1: + CCTK_WARN (0, "There is no stencil for op=""TVD"" with order_space=1"); + break; + case 2: + case 3: + CCTK_FNAME(prolongate_3d_real8_3tl_minmod) + ((const CCTK_REAL8*)srcs[0]->storage(), times[0], + (const CCTK_REAL8*)srcs[1]->storage(), times[1], + (const CCTK_REAL8*)srcs[2]->storage(), times[2], + srcshp[0], srcshp[1], srcshp[2], + (CCTK_REAL8*)storage(), time, + dstshp[0], dstshp[1], dstshp[2], + srcbbox, dstbbox, regbbox); + break; + default: + assert (0); + } break; default: assert (0); - } + } // switch (order_time) break; default: assert(0); - } // switch (prolong_method) + } // switch (transport_operator) } else { assert (0); diff --git a/Carpet/CarpetLib/src/data.hh b/Carpet/CarpetLib/src/data.hh index bb8e5c497..016dfcd1e 100644 --- a/Carpet/CarpetLib/src/data.hh +++ b/Carpet/CarpetLib/src/data.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/data.hh,v 1.15 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/data.hh,v 1.16 2004/01/25 14:57:29 schnetter Exp $ #ifndef DATA_HH #define DATA_HH @@ -39,14 +39,17 @@ class data: public gdata<D> { public: // Constructors - data (const int varindex); - data (const int varindex, const ibbox& extent, const int proc); + data (const int varindex = -1, + const operator_type transport_operator = op_error); + data (const int varindex, const operator_type transport_operator, + const ibbox& extent, const int proc); // Destructors virtual ~data (); // Pseudo constructors - virtual data* make_typed (const int varindex) const; + virtual data* make_typed (const int varindex, + const operator_type transport_operator) const; // Storage management virtual void allocate (const ibbox& extent, const int proc, diff --git a/Carpet/CarpetLib/src/defs.cc b/Carpet/CarpetLib/src/defs.cc index b525578ef..28aaf0539 100644 --- a/Carpet/CarpetLib/src/defs.cc +++ b/Carpet/CarpetLib/src/defs.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.cc,v 1.17 2003/11/13 16:03:58 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.cc,v 1.18 2004/01/25 14:57:29 schnetter Exp $ #include <assert.h> #include <ctype.h> @@ -9,12 +9,31 @@ #include <stack> #include <vector> +#include "cctk.h" + #include "defs.hh" using namespace std; +template<class T> +T ipow (T x, int y) { + if (y<0) { + y = -y; + x = T(1)/x; + } + T res = T(1); + while (y>0) { + if (y%2) res *= x; + x *= x; + y /= 2; + } + return res; +} + + + void skipws (istream& is) { while (is.good() && isspace(is.peek())) { is.get(); @@ -23,26 +42,53 @@ void skipws (istream& is) { +void expect (istream& is, const char c) { + if (is.peek() == c) return; + cout << "While reading characters from a stream:" << endl + << " Character '" << c << "' expected, but not found." << endl + << " The next up to 100 available characters are \""; + for (int i=0; i<100; ++i) { + const int uc = is.get(); + if (uc<0) break; + cout << (unsigned char)uc; + } + cout << "\"." << endl; + throw input_error(); +} + + + +void consume (istream& is, const char c) { + expect (is, c); + is.get(); +} + + + // Vector input template<class T> istream& input (istream& is, vector<T>& v) { v.clear(); - skipws (is); - assert (is.peek() == '['); - is.get(); - skipws (is); - while (is.good() && is.peek() != ']') { - T elem; - is >> elem; - v.push_back (elem); + try { skipws (is); - if (is.peek() != ',') break; - is.get(); + consume (is, '['); + skipws (is); + while (is.good() && is.peek() != ']') { + T elem; + is >> elem; + v.push_back (elem); + skipws (is); + if (is.peek() != ',') break; + is.get(); + skipws (is); + } skipws (is); + consume (is, ']'); + } catch (input_error &err) { + cout << "Input error while reading a vector<>" << endl + << " The following elements have been read so far: " << v << endl; + throw err; } - skipws (is); - assert (is.peek() == ']'); - is.get(); return is; } @@ -102,10 +148,12 @@ ostream& output (ostream& os, const vector<T>& v) { #include "bbox.hh" #include "bboxset.hh" +template int ipow (int x, int y); + template istream& input (istream& os, vector<bbox<int,3> >& v); -template istream& input (istream& os, vector<bbox<double,3> >& v); +template istream& input (istream& os, vector<bbox<CCTK_REAL,3> >& v); template istream& input (istream& os, vector<vector<bbox<int,3> > >& v); -template istream& input (istream& os, vector<vector<bbox<double,3> > >& v); +template istream& input (istream& os, vector<vector<bbox<CCTK_REAL,3> > >& v); template istream& input (istream& os, vector<vect<vect<bool,2>,3> >& v); template istream& input (istream& os, vector<vector<vect<vect<bool,2>,3> > >& v); @@ -114,12 +162,14 @@ template ostream& output (ostream& os, const set<bbox<int,3> >& s); template ostream& output (ostream& os, const set<bboxset<int,3> >& s); template ostream& output (ostream& os, const stack<bbox<int,3> >& s); template ostream& output (ostream& os, const vector<int>& v); +template ostream& output (ostream& os, const vector<CCTK_REAL>& v); template ostream& output (ostream& os, const vector<bbox<int,3> >& v); -template ostream& output (ostream& os, const vector<bbox<double,3> >& v); +template ostream& output (ostream& os, const vector<bbox<CCTK_REAL,3> >& v); template ostream& output (ostream& os, const vector<list<bbox<int,3> > >& v); template ostream& output (ostream& os, const vector<vector<int> >& v); +template ostream& output (ostream& os, const vector<vector<CCTK_REAL> >& v); template ostream& output (ostream& os, const vector<vector<bbox<int,3> > >& v); -template ostream& output (ostream& os, const vector<vector<bbox<double,3> > >& v); +template ostream& output (ostream& os, const vector<vector<bbox<CCTK_REAL,3> > >& v); template ostream& output (ostream& os, const vector<vector<vect<vect<bool,2>,3> > >& v); template ostream& output (ostream& os, const vector<vect<vect<bool,2>,3> >& v); template ostream& output (ostream& os, const vector<vector<vector<bbox<int,3> > > >& v); diff --git a/Carpet/CarpetLib/src/defs.hh b/Carpet/CarpetLib/src/defs.hh index b948d7a9a..2d324032a 100644 --- a/Carpet/CarpetLib/src/defs.hh +++ b/Carpet/CarpetLib/src/defs.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.hh,v 1.11 2003/03/18 17:30:25 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/defs.hh,v 1.12 2004/01/25 14:57:29 schnetter Exp $ #ifndef DEFS_HH #define DEFS_HH @@ -34,24 +34,15 @@ inline T square (const T& x) { return x*x; } // Another useful helper template<class T> -inline T ipow (T x, int y) { - if (y<0) { - y = -y; - x = T(1)/x; - } - T res = T(1); - while (y>0) { - if (y%2) res *= x; - x *= x; - y /= 2; - } - return res; -} +T ipow (T x, int y); -// Skip whitespace +// Input streams +struct input_error { }; void skipws (istream& is); +void expect (istream& is, const char c); +void consume (istream& is, const char c); diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc index 3cb9ac1c0..e23268791 100644 --- a/Carpet/CarpetLib/src/dh.cc +++ b/Carpet/CarpetLib/src/dh.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.47 2004/01/21 14:25:35 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/dh.cc,v 1.48 2004/01/25 14:57:29 schnetter Exp $ #include <assert.h> @@ -127,7 +127,8 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { for (int cc=0; cc<h.components(rl); ++cc) { assert (ml<h.mglevels(rl,cc)); // intersect boundaries with interior of that component - const ibset ovlp = bnds & boxes.at(rl).at(cc).at(ml).interior; + ibset ovlp = bnds & boxes.at(rl).at(cc).at(ml).interior; + ovlp.normalize(); for (typename ibset::const_iterator b=ovlp.begin(); b!=ovlp.end(); ++b) { boxes.at(rl).at(c ).at(ml).recv_sync.at(cc).push_back(*b); @@ -179,7 +180,6 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { } // for c } // for rl - // TODO: prefer boxes from the same processor for (int rl=0; rl<h.reflevels(); ++rl) { for (int c=0; c<h.components(rl); ++c) { for (int ml=0; ml<h.mglevels(rl,c); ++ml) { @@ -191,6 +191,7 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { for (int cc=0; cc<h.components(rl+1); ++cc) { const ibbox intrf = boxes.at(rl+1).at(cc).at(ml).interior; // Prolongation (interior) + // TODO: prefer boxes from the same processor { // (the prolongation may use the exterior of the coarse // grid, and must fill all of the interior of the fine @@ -214,8 +215,10 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { const ibbox send = recv.expanded_for(extr); assert (! send.empty()); assert (send.is_contained_in(extr)); - boxes.at(rl+1).at(cc).at(ml).recv_ref_coarse.at(c ).push_back(recv); - boxes.at(rl ).at(c ).at(ml).send_ref_fine .at(cc).push_back(send); + boxes.at(rl+1).at(cc).at(ml).recv_ref_coarse.at(c ) + .push_back(recv); + boxes.at(rl ).at(c ).at(ml).send_ref_fine .at(cc) + .push_back(send); } } @@ -239,6 +242,7 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { const ibbox& extrf = boxes.at(rl+1).at(cc).at(ml).exterior; const ibset& bndsf = boxes.at(rl+1).at(cc).at(ml).boundaries; // Prolongation (boundaries) + // TODO: prefer boxes from the same processor { // (the prolongation may use the exterior of the coarse // grid, and must fill all of the boundary of the fine @@ -264,14 +268,14 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { { for (typename ibset::const_iterator pbi=pbndsf.begin(); pbi!=pbndsf.end(); ++pbi) { - buffers |= (*pbi).expand(buffer_width, buffer_width) & intrf; + buffers |= (*pbi).expand(buffer_width, buffer_width) & extrf; } buffers.normalize(); } // Add boundaries const ibbox maxrecvs = extr.expand(-pss,-pss).contracted_for(extrf); - ibset recvs = (bndsf | buffers) & maxrecvs; + ibset recvs = buffers & maxrecvs; recvs.normalize(); { // Do not prolongate what is already prolongated @@ -342,8 +346,10 @@ void dh<D>::recompose (const int initialise_from, const bool do_prolongate) { if (! recv.empty()) { const ibbox & send = recv.expanded_for(intrf); assert (! send.empty()); - boxes.at(rl+1).at(cc).at(ml).send_ref_coarse.at(c ).push_back(send); - boxes.at(rl ).at(c ).at(ml).recv_ref_fine .at(cc).push_back(recv); + boxes.at(rl+1).at(cc).at(ml).send_ref_coarse.at(c ) + .push_back(send); + boxes.at(rl ).at(c ).at(ml).recv_ref_fine .at(cc) + .push_back(recv); } } } diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc index 86537cea6..948047e5d 100644 --- a/Carpet/CarpetLib/src/gdata.cc +++ b/Carpet/CarpetLib/src/gdata.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.25 2003/11/21 13:55:46 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.26 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> #include <stdlib.h> @@ -54,10 +54,11 @@ comm_state<D>::~comm_state () // Constructors template<int D> -gdata<D>::gdata (const int varindex_) - : varindex(varindex_), - _has_storage(false), - transport_operator (find_transport_operator(varindex_)) +gdata<D>::gdata (const int varindex_, const operator_type transport_operator_) + : varindex(varindex_), transport_operator(transport_operator_), + wtime_isend(0.0), wtime_isendwait(0.0), + wtime_irecv(0.0), wtime_irecvwait(0.0), + _has_storage(false) { } // Destructors @@ -66,43 +67,6 @@ gdata<D>::~gdata () { } -// Transport operator types -template<int D> -typename gdata<D>::operator_type -gdata<D>::find_transport_operator (const int varindex) -{ - const operator_type default_operator = op_Lagrange; - if (varindex == -1) return op_error; - assert (varindex >= 0); - const int groupindex = CCTK_GroupIndexFromVarI (varindex); - assert (groupindex >= 0); - const int group_tags_table = CCTK_GroupTagsTableI(groupindex); - assert (group_tags_table >= 0); - char prolong_string[100]; - const int ierr = Util_TableGetString - (group_tags_table, sizeof prolong_string, prolong_string, "Prolongation"); - if (ierr == UTIL_ERROR_TABLE_NO_SUCH_KEY) { - char* groupname = CCTK_GroupName(groupindex); - CCTK_VWarn (4, __LINE__, __FILE__, CCTK_THORNSTRING, - "Tags table for group \"%s\" does not contain \"Prolongation\" tag", groupname); - ::free (groupname); - return default_operator; - } - assert (ierr >= 0); - if (CCTK_Equals(prolong_string, "None")) { - return op_none; - } else if (CCTK_Equals(prolong_string, "Lagrange")) { - return op_Lagrange; - } else if (CCTK_Equals(prolong_string, "TVD")) { - return op_TVD; - } else { - assert (0); - } - return op_error; -} - - - // Data manipulators template<int D> void gdata<D>::copy_from (comm_state<D>& state, @@ -174,7 +138,7 @@ void gdata<D>::copy_from_recv (comm_state<D>& state, } else { // copy to different processor - gdata<D>* const tmp = make_typed(varindex); + gdata<D>* const tmp = make_typed(varindex, transport_operator); // TODO: is this efficient? state.tmps.push_back (tmp); ++state.current; @@ -350,7 +314,7 @@ void gdata<D> } else { // interpolate from other processor - gdata<D>* const tmp = make_typed(varindex); + gdata<D>* const tmp = make_typed(varindex, transport_operator); // TODO: is this efficient? state.tmps.push_back (tmp); ++state.current; diff --git a/Carpet/CarpetLib/src/gdata.hh b/Carpet/CarpetLib/src/gdata.hh index 37b544d7d..7484d75da 100644 --- a/Carpet/CarpetLib/src/gdata.hh +++ b/Carpet/CarpetLib/src/gdata.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.hh,v 1.21 2003/11/21 13:55:46 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.hh,v 1.22 2004/01/25 14:57:30 schnetter Exp $ #ifndef GDATA_HH #define GDATA_HH @@ -15,6 +15,7 @@ #include "defs.hh" #include "dist.hh" #include "bbox.hh" +#include "operators.hh" #include "vect.hh" using namespace std; @@ -56,6 +57,10 @@ protected: // should be readonly // Fields int varindex; // Cactus variable index, or -1 + operator_type transport_operator; + + double wtime_isend, wtime_isendwait; + double wtime_irecv, wtime_irecvwait; bool _has_storage; // has storage associated (on some processor) bool _owns_storage; // owns the storage @@ -72,13 +77,16 @@ protected: // should be readonly public: // Constructors - gdata (const int varindex); + gdata (const int varindex, + const operator_type transport_operator = op_error); // Destructors virtual ~gdata (); // Pseudo constructors - virtual gdata<D>* make_typed (const int varindex) const = 0; + virtual gdata<D>* + make_typed (const int varindex, + const operator_type transport_operator = op_error) const = 0; // Processor management virtual void change_processor (comm_state<D>& state, @@ -144,14 +152,6 @@ public: return dot(ind, stride()); } - // Transport operator types - protected: - enum operator_type { op_error, op_none, op_Lagrange, op_TVD }; - // readonly - operator_type transport_operator; - private: - static operator_type find_transport_operator (const int varindex); - // Data manipulators public: void copy_from (comm_state<D>& state, const gdata* src, const ibbox& box); diff --git a/Carpet/CarpetLib/src/gf.cc b/Carpet/CarpetLib/src/gf.cc index 4a913d199..2fcec0d2f 100644 --- a/Carpet/CarpetLib/src/gf.cc +++ b/Carpet/CarpetLib/src/gf.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gf.cc,v 1.14 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gf.cc,v 1.15 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> @@ -15,9 +15,11 @@ using namespace std; // Constructors // VGF template<class T,int D> -gf<T,D>::gf (const int varindex, th<D>& t, dh<D>& d, +gf<T,D>::gf (const int varindex, const operator_type transport_operator, + th<D>& t, dh<D>& d, const int tmin, const int tmax, const int prolongation_order_time) - : ggf<D>(varindex, t, d, tmin, tmax, prolongation_order_time) + : ggf<D>(varindex, transport_operator, + t, d, tmin, tmax, prolongation_order_time) { // VGF this->recompose (0, true); diff --git a/Carpet/CarpetLib/src/gf.hh b/Carpet/CarpetLib/src/gf.hh index ee4ce684c..3f2027485 100644 --- a/Carpet/CarpetLib/src/gf.hh +++ b/Carpet/CarpetLib/src/gf.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gf.hh,v 1.10 2003/11/21 13:55:46 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gf.hh,v 1.11 2004/01/25 14:57:30 schnetter Exp $ #ifndef GF_HH #define GF_HH @@ -43,7 +43,8 @@ public: // Constructors // VGF - gf (const int varindex, th<D>& t, dh<D>& d, + gf (const int varindex, const operator_type transport_operator, + th<D>& t, dh<D>& d, const int tmin, const int tmax, const int prolongation_order_time); // Destructors @@ -55,7 +56,8 @@ public: protected: - virtual gdata<D>* typed_data() { return new data<T,D>(this->varindex); } + virtual gdata<D>* typed_data() + { return new data<T,D>(this->varindex, this->transport_operator); } diff --git a/Carpet/CarpetLib/src/ggf.cc b/Carpet/CarpetLib/src/ggf.cc index 6786e5eb7..88247a6bf 100644 --- a/Carpet/CarpetLib/src/ggf.cc +++ b/Carpet/CarpetLib/src/ggf.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/ggf.cc,v 1.29 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/ggf.cc,v 1.30 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> #include <stdlib.h> @@ -20,16 +20,17 @@ using namespace std; // Constructors template<int D> -ggf<D>::ggf (const int varindex, th<D>& t, dh<D>& d, +ggf<D>::ggf (const int varindex, const operator_type transport_operator, + th<D>& t, dh<D>& d, const int tmin, const int tmax, const int prolongation_order_time) - : varindex(varindex), t(t), + : varindex(varindex), transport_operator(transport_operator), t(t), tmin(tmin), tmax(tmax), prolongation_order_time(prolongation_order_time), h(d.h), d(d), storage(tmax-tmin+1) { - assert (t.h == &d.h); + assert (&t.h == &d.h); d.add(this); } @@ -140,13 +141,16 @@ void ggf<D>::recompose (const int initialise_from, const bool do_prolongate) { for (comm_state<D> state; !state.done(); state.step()) { sync (state,tl,rl,c,ml); } - // TODO: assert that reflevel 0 boundaries are copied - if (rl>0) { - for (comm_state<D> state; !state.done(); state.step()) { - const CCTK_REAL time = t.time(tl,rl,ml); - ref_bnd_prolongate (state,tl,rl,c,ml,time); - } - } // if rl + + if (do_prolongate) { + // TODO: assert that reflevel 0 boundaries are copied + if (rl>0) { + for (comm_state<D> state; !state.done(); state.step()) { + const CCTK_REAL time = t.time(tl,rl,ml); + ref_bnd_prolongate (state,tl,rl,c,ml,time); + } + } // if rl + } } // for ml } // for c @@ -318,7 +322,6 @@ void ggf<D>::intercat (comm_state<D>& state, gsrcs[i] = storage[tl2s[i]-tmin][rl2][c2][ml2]; times[i] = t.time(tl2s[i],rl2,ml2); } -// const CCTK_REAL time = t.time(tl1,rl1,ml1); const ibbox recv = d.boxes[rl1][c1][ml1].*recv_list; const ibbox send = d.boxes[rl2][c2][ml2].*send_list; @@ -359,7 +362,6 @@ void ggf<D>::intercat (comm_state<D>& state, gsrcs[i] = storage[tl2s[i]-tmin][rl2][c2][ml2]; times[i] = t.time(tl2s[i],rl2,ml2); } -// const CCTK_REAL time = t.time(tl1,rl1,ml1); const iblist recv = d.boxes[rl1][c1][ml1].*recv_list; const iblist send = d.boxes[rl2][c2][ml2].*send_list; @@ -405,7 +407,6 @@ void ggf<D>::intercat (comm_state<D>& state, gsrcs[i] = storage[tl2s[i]-tmin][rl2][c2][ml2]; times[i] = t.time(tl2s[i],rl2,ml2); } -// const CCTK_REAL time = t.time(tl1,rl1,ml1); const iblist recv = (d.boxes[rl1][c1][ml1].*recv_listvect)[c2]; const iblist send = (d.boxes[rl2][c2][ml2].*send_listvect)[c1]; diff --git a/Carpet/CarpetLib/src/ggf.hh b/Carpet/CarpetLib/src/ggf.hh index b99c7aa10..59f9bfd71 100644 --- a/Carpet/CarpetLib/src/ggf.hh +++ b/Carpet/CarpetLib/src/ggf.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/ggf.hh,v 1.17 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/ggf.hh,v 1.18 2004/01/25 14:57:30 schnetter Exp $ #ifndef GGF_HH #define GGF_HH @@ -52,6 +52,7 @@ public: // should be readonly // Fields int varindex; // Cactus variable index + operator_type transport_operator; th<D> &t; // time hierarchy int tmin, tmax; // timelevels @@ -66,7 +67,8 @@ protected: public: // Constructors - ggf (const int varindex, th<D>& t, dh<D>& d, + ggf (const int varindex, const operator_type transport_operator, + th<D>& t, dh<D>& d, const int tmin, const int tmax, const int prolongation_order_time); diff --git a/Carpet/CarpetLib/src/gh.cc b/Carpet/CarpetLib/src/gh.cc index b37fb3f15..fdaa3d054 100644 --- a/Carpet/CarpetLib/src/gh.cc +++ b/Carpet/CarpetLib/src/gh.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.cc,v 1.23 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.cc,v 1.24 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> #include <stdlib.h> @@ -21,7 +21,7 @@ using namespace std; template<int D> gh<D>::gh (const int reffact, const centering refcent, const int mgfact, const centering mgcent, - const ibbox& baseextent) + const ibbox baseextent) : reffact(reffact), refcent(refcent), mgfact(mgfact), mgcent(mgcent), baseextent(baseextent) @@ -169,69 +169,6 @@ void gh<D>::recompose (const rexts& exts, } } -// Helpers -template<int D> -typename gh<D>::cexts -gh<D>::make_reflevel_multigrid_boxes (const vector<ibbox>& exts, - const int mglevels) - const -{ - assert (mglevels>0); - - cexts mexts (exts.size()); - for (int c=0; c<(int)exts.size(); ++c) { - - mexts[c].resize(mglevels); - - ibbox ext = exts[c]; - for (int ml=0; ml<mglevels; ++ml) { - - mexts[c][ml] = ext; - - if (ml == mglevels-1) break; - - // This level's characteristics - ivect str = ext.stride(); - ivect lo = ext.lower(); - ivect up = ext.upper(); - - // Transform to next (coarser) level - switch (mgcent) { - case vertex_centered: - break; - case cell_centered: - for (int d=0; d<D; ++d) assert (str[d]%2 == 0); - lo += str/2; - break; - default: - assert (0); - } - str *= mgfact; - up = up - (up - lo) % str; - - ext = ibbox(lo,up,str); - } // for ml - } // for c - - return mexts; -} - -template<int D> -typename gh<D>::rexts -gh<D>::make_multigrid_boxes (const vector<vector<ibbox> >& exts, - const int mglevels) - const -{ - assert (mglevels>0); - - rexts mexts (exts.size()); - for (int rl=0; rl<(int)exts.size(); ++rl) { - mexts[rl] = make_reflevel_multigrid_boxes (exts[rl], mglevels); - } - - return mexts; -} - // Accessors @@ -277,7 +214,6 @@ ostream& gh<D>::output (ostream& os) const { os << "gh<" << D << ">:" << "reffactor=" << reffact << ",refcentering=" << refcent << "," << "mgfactor=" << mgfact << ",mgcentering=" << mgcent << "," - << "baseextent=" << baseextent << "," << "extents=" << extents << "," << "outer_boundaries=" << outer_boundaries << "," << "processors=" << processors << "," diff --git a/Carpet/CarpetLib/src/gh.hh b/Carpet/CarpetLib/src/gh.hh index db046185a..7db23b780 100644 --- a/Carpet/CarpetLib/src/gh.hh +++ b/Carpet/CarpetLib/src/gh.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.hh,v 1.14 2003/11/05 16:18:39 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gh.hh,v 1.15 2004/01/25 14:57:30 schnetter Exp $ #ifndef GH_HH #define GH_HH @@ -63,7 +63,7 @@ public: // should be readonly list<th<D>*> ths; // list of all time hierarchies - ibbox baseextent; // bounds (inclusive) of base level + ibbox baseextent; vector<vector<ibbox> > bases; // [rl][ml] // TODO: invent structure for this @@ -78,7 +78,7 @@ public: // Constructors gh (const int reffact, const centering refcent, const int mgfact, const centering mgcent, - const ibbox& baseextent); + const ibbox baseextent); // Destructors virtual ~gh (); @@ -89,14 +89,6 @@ public: const int initialise_from, const bool do_prolongate); - // Helpers - cexts make_reflevel_multigrid_boxes (const vector<ibbox>& exts, - const int mglevels) - const; - rexts make_multigrid_boxes (const vector<vector<ibbox> >& exts, - const int mglevels) - const; - // Accessors int reflevels () const { return (int)extents.size(); diff --git a/Carpet/CarpetLib/src/make.code.defn b/Carpet/CarpetLib/src/make.code.defn index 0b718747a..7cbecca46 100644 --- a/Carpet/CarpetLib/src/make.code.defn +++ b/Carpet/CarpetLib/src/make.code.defn @@ -1,5 +1,5 @@ # Main make.code.defn file for thorn CarpetLib -*-Makefile-*- -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/make.code.defn,v 1.10 2003/11/05 16:18:39 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/make.code.defn,v 1.11 2004/01/25 14:57:30 schnetter Exp $ # Source files in this directory SRCS = bbox.cc \ @@ -23,7 +23,9 @@ SRCS = bbox.cc \ prolongate_3d_real8_o3_rf2.F77 \ prolongate_3d_real8_o5.F77 \ prolongate_3d_real8_2tl.F77 \ + prolongate_3d_real8_2tl_rf2.F77 \ prolongate_3d_real8_2tl_o3.F77 \ + prolongate_3d_real8_2tl_o3_rf2.F77 \ prolongate_3d_real8_2tl_o5.F77 \ prolongate_3d_real8_3tl.F77 \ prolongate_3d_real8_3tl_rf2.F77 \ diff --git a/Carpet/CarpetLib/src/operators.hh b/Carpet/CarpetLib/src/operators.hh index b65ff23cb..d5ea168f3 100644 --- a/Carpet/CarpetLib/src/operators.hh +++ b/Carpet/CarpetLib/src/operators.hh @@ -1,8 +1,8 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/operators.hh,v 1.2 2004/03/03 15:30:40 hawke Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/operators.hh,v 1.1 2004/01/25 14:57:30 schnetter Exp $ #ifndef OPERATORS_HH #define OPERATORS_HH -enum operator_type { op_error, op_none, op_Lagrange, op_TVD, op_ENO }; +enum operator_type { op_error, op_none, op_Lagrange, op_TVD }; #endif // OPERATORS_HH diff --git a/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77 b/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77 index d9d62741b..0945909bf 100644 --- a/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77 +++ b/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77 @@ -1,5 +1,5 @@ c -*-Fortran-*- -c $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77,v 1.2 2004/03/11 12:03:09 schnetter Exp $ +c $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_o3_rf2.F77,v 1.1 2004/01/25 14:57:30 schnetter Exp $ #include "cctk.h" #include "cctk_Parameters.h" @@ -85,11 +85,6 @@ c bbox(:,3) is stride c This could be handled, but is likely to point to an error elsewhere call CCTK_WARN (0, "Internal error: region extent is empty") end if - if (mod(srcbbox(d,2) - srcbbox(d,1), srcbbox(d,3)).ne.0 - $ .or. mod(dstbbox(d,2) - dstbbox(d,1), dstbbox(d,3)).ne.0 - $ .or. mod(regbbox(d,2) - regbbox(d,1), regbbox(d,3)).ne.0) then - call CCTK_WARN (0, "Internal error: array extents are not integer multiples of the strides") - end if regkext = (regbbox(d,2) - regbbox(d,1)) / regbbox(d,3) + 1 srckoff = (regbbox(d,1) - srcbbox(d,1)) / dstbbox(d,3) offsetlo = regbbox(d,3) diff --git a/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77 b/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77 index 727c2581f..0708979f6 100644 --- a/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77 +++ b/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77 @@ -1,5 +1,5 @@ c -*-Fortran-*- -c $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77,v 1.2 2004/03/11 12:03:09 schnetter Exp $ +c $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/prolongate_3d_real8_2tl_rf2.F77,v 1.1 2004/01/25 14:57:30 schnetter Exp $ #include "cctk.h" #include "cctk_Parameters.h" @@ -75,11 +75,6 @@ c bbox(:,3) is stride c This could be handled, but is likely to point to an error elsewhere call CCTK_WARN (0, "Internal error: region extent is empty") end if - if (mod(srcbbox(d,2) - srcbbox(d,1), srcbbox(d,3)).ne.0 - $ .or. mod(dstbbox(d,2) - dstbbox(d,1), dstbbox(d,3)).ne.0 - $ .or. mod(regbbox(d,2) - regbbox(d,1), regbbox(d,3)).ne.0) then - call CCTK_WARN (0, "Internal error: array extents are not integer multiples of the strides") - end if if (regbbox(d,1).lt.srcbbox(d,1) $ .or. regbbox(d,1).lt.dstbbox(d,1) $ .or. regbbox(d,2).gt.srcbbox(d,2) diff --git a/Carpet/CarpetLib/src/th.cc b/Carpet/CarpetLib/src/th.cc index d849e3c47..5fd8eabee 100644 --- a/Carpet/CarpetLib/src/th.cc +++ b/Carpet/CarpetLib/src/th.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/th.cc,v 1.12 2003/08/10 21:58:45 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/th.cc,v 1.13 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> #include <math.h> @@ -18,23 +18,23 @@ using namespace std; // Constructors template<int D> -th<D>::th (gh<D>* h, const CCTK_REAL basedelta) +th<D>::th (gh<D>& h, const CCTK_REAL basedelta) : h(h), delta(basedelta) { - h->add(this); + h.add(this); } // Destructors template<int D> th<D>::~th () { - h->remove(this); + h.remove(this); } // Modifiers template<int D> void th<D>::recompose () { - times.resize(h->reflevels()); - deltas.resize(h->reflevels()); - for (int rl=0; rl<h->reflevels(); ++rl) { + times.resize(h.reflevels()); + deltas.resize(h.reflevels()); + for (int rl=0; rl<h.reflevels(); ++rl) { const int old_mglevels = times[rl].size(); CCTK_REAL mgtime; // Select default time @@ -45,15 +45,15 @@ void th<D>::recompose () { } else { mgtime = times[rl][old_mglevels-1]; } - times[rl].resize(h->mglevels(rl,0), mgtime); - deltas[rl].resize(h->mglevels(rl,0)); - for (int ml=0; ml<h->mglevels(rl,0); ++ml) { + times[rl].resize(h.mglevels(rl,0), mgtime); + deltas[rl].resize(h.mglevels(rl,0)); + for (int ml=0; ml<h.mglevels(rl,0); ++ml) { if (rl==0 && ml==0) { deltas[rl][ml] = delta; } else if (ml==0) { - deltas[rl][ml] = deltas[rl-1][ml] / h->reffact; + deltas[rl][ml] = deltas[rl-1][ml] / h.reffact; } else { - deltas[rl][ml] = deltas[rl][ml-1] * h->mgfact; + deltas[rl][ml] = deltas[rl][ml-1] * h.mgfact; } } } @@ -66,8 +66,8 @@ template<int D> void th<D>::output (ostream& os) const { os << "th<" << D << ">:" << "times={"; - for (int rl=0; rl<h->reflevels(); ++rl) { - for (int ml=0; ml<h->mglevels(rl,0); ++ml) { + for (int rl=0; rl<h.reflevels(); ++rl) { + for (int ml=0; ml<h.mglevels(rl,0); ++ml) { if (!(rl==0 && ml==0)) os << ","; os << rl << ":" << ml << ":" << times[rl][ml] << "(" << deltas[rl][ml] << ")"; diff --git a/Carpet/CarpetLib/src/th.hh b/Carpet/CarpetLib/src/th.hh index 85d028ac3..26deb154a 100644 --- a/Carpet/CarpetLib/src/th.hh +++ b/Carpet/CarpetLib/src/th.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/th.hh,v 1.9 2003/01/03 15:49:36 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/th.hh,v 1.10 2004/01/25 14:57:30 schnetter Exp $ #ifndef TH_HH #define TH_HH @@ -33,7 +33,7 @@ class th { public: // should be readonly // Fields - gh<D> *h; // hierarchy + gh<D>& h; // hierarchy private: @@ -44,7 +44,7 @@ private: public: // Constructors - th (gh<D>* h, const CCTK_REAL basedelta); + th (gh<D>& h, const CCTK_REAL basedelta); // Destructors ~th (); @@ -54,14 +54,14 @@ public: // Time management CCTK_REAL get_time (const int rl, const int ml) const { - assert (rl>=0 && rl<h->reflevels()); - assert (ml>=0 && ml<h->mglevels(rl,0)); + assert (rl>=0 && rl<h.reflevels()); + assert (ml>=0 && ml<h.mglevels(rl,0)); return times[rl][ml]; } void set_time (const int rl, const int ml, const CCTK_REAL t) { - assert (rl>=0 && rl<h->reflevels()); - assert (ml>=0 && ml<h->mglevels(rl,0)); + assert (rl>=0 && rl<h.reflevels()); + assert (ml>=0 && ml<h.mglevels(rl,0)); times[rl][ml] = t; } @@ -70,20 +70,20 @@ public: } CCTK_REAL get_delta (const int rl, const int ml) const { - assert (rl>=0 && rl<h->reflevels()); - assert (ml>=0 && ml<h->mglevels(rl,0)); + assert (rl>=0 && rl<h.reflevels()); + assert (ml>=0 && ml<h.mglevels(rl,0)); return deltas[rl][ml]; } void set_delta (const int rl, const int ml, const CCTK_REAL dt) { - assert (rl>=0 && rl<h->reflevels()); - assert (ml>=0 && ml<h->mglevels(rl,0)); + assert (rl>=0 && rl<h.reflevels()); + assert (ml>=0 && ml<h.mglevels(rl,0)); deltas[rl][ml] = dt; } CCTK_REAL time (const int tl, const int rl, const int ml) const { - assert (rl>=0 && rl<h->reflevels()); - assert (ml>=0 && ml<h->mglevels(rl,0)); + assert (rl>=0 && rl<h.reflevels()); + assert (ml>=0 && ml<h.mglevels(rl,0)); return get_time(rl, ml) + tl * get_delta(rl, ml); } diff --git a/Carpet/CarpetLib/src/vect.cc b/Carpet/CarpetLib/src/vect.cc index 7fe311fdb..b5773eebf 100644 --- a/Carpet/CarpetLib/src/vect.cc +++ b/Carpet/CarpetLib/src/vect.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/vect.cc,v 1.12 2003/11/13 16:03:58 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/vect.cc,v 1.13 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> @@ -16,19 +16,16 @@ using namespace std; template<class T,int D> void vect<T,D>::input (istream& is) { skipws (is); - assert (is.peek() == '['); - is.get(); + consume (is, '['); for (int d=0; d<D; ++d) { is >> (*this)[d]; if (d<D-1) { skipws (is); - assert (is.peek() == ','); - is.get(); + consume (is, ','); } } skipws (is); - assert (is.peek() == ']'); - is.get(); + consume (is, ']'); } diff --git a/Carpet/CarpetLib/src/vect.hh b/Carpet/CarpetLib/src/vect.hh index 5271626d2..1df8f874c 100644 --- a/Carpet/CarpetLib/src/vect.hh +++ b/Carpet/CarpetLib/src/vect.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/vect.hh,v 1.21 2003/11/13 16:03:58 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/vect.hh,v 1.22 2004/01/25 14:57:30 schnetter Exp $ #ifndef VECT_HH #define VECT_HH @@ -82,14 +82,17 @@ public: elt[0]=x; elt[1]=y; elt[2]=z; elt[3]=t; } +#if 0 + // This creates confusion /** Constructor from a pointer, i.e.\ a C array. */ - vect (const T* const x) { + explicit vect (const T* const x) { for (int d=0; d<D; ++d) elt[d]=x[d]; } +#endif /** Constructor from a vector with a different type. */ template<class S> - explicit vect (const vect<S,D>& a) { + /*explicit*/ vect (const vect<S,D>& a) { for (int d=0; d<D; ++d) elt[d]=(T)a[d]; } @@ -160,8 +163,9 @@ public: // Accessors /** Return a non-writable element of a vector. */ - // Don't return a reference; *this might be a temporary - T operator[] (const int d) const { + // (Don't return a reference; *this might be a temporary) + // Do return a reference, so that a vector can be accessed as array + const T& operator[] (const int d) const { assert(d>=0 && d<D); return elt[d]; } @@ -172,6 +176,14 @@ public: return elt[d]; } +#if 0 + // This creates confusion + /** Return a pointer to a vector. */ + operator const T* () const { + return this; + } +#endif + /** Return a combination of the vector elements e[a[i]]. The element combination is selected by another vector. */ template<class TT, int DD> @@ -492,6 +504,25 @@ public: // Operators +/** This corresponds to the ?: operator. Return a vector with the + elements set to either b[i] or c[i], depending on whether a[i] is + true or not. */ +template<class T,int D> +inline vect<T,D> either (const vect<bool,D>& a, + const vect<T,D>& b, const vect<T,D>& c) { + vect<T,D> r; + for (int d=0; d<D; ++d) r[d]=a[d]?b[d]:c[d]; + return r; +} + +/** Transpose a vector of a vector */ +template<class T, int D, int DD> +inline vect<vect<T,D>,DD> xpose (vect<vect<T,DD>,D> const & a) { + vect<vect<T,D>,DD> r; + for (int dd=0; dd<DD; ++dd) for (int d=0; d<D; ++d) r[dd][d] = a[d][dd]; + return r; +} + /** Return the element-wise absolute value. */ template<class T,int D> inline vect<T,D> abs (const vect<T,D>& a) { diff --git a/Carpet/CarpetReduce/README b/Carpet/CarpetReduce/README index e5c64331a..c53bd647d 100644 --- a/Carpet/CarpetReduce/README +++ b/Carpet/CarpetReduce/README @@ -1,7 +1,8 @@ Cactus Code Thorn CarpetReduce Authors : Erik Schnetter <schnetter@uni-tuebingen.de> -CVS info : $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/README,v 1.1 2001/12/11 13:08:56 schnetter Exp $ +CVS info : $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/README,v 1.2 2004/01/25 14:57:30 schnetter Exp $ -------------------------------------------------------------------------- Purpose of the thorn: +This thorn provides parallel reduction operators for Carpet. diff --git a/Carpet/CarpetReduce/src/reduce.cc b/Carpet/CarpetReduce/src/reduce.cc index 4a37ddbeb..bf3c69b7f 100644 --- a/Carpet/CarpetReduce/src/reduce.cc +++ b/Carpet/CarpetReduce/src/reduce.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.30 2003/11/12 17:29:30 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.31 2004/01/25 14:57:30 schnetter Exp $ #include <assert.h> #include <float.h> @@ -23,7 +23,7 @@ #include "reduce.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.30 2003/11/12 17:29:30 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetReduce/src/reduce.cc,v 1.31 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetReduce_reduce_cc); } @@ -50,25 +50,26 @@ namespace CarpetReduce { template<> inline complex<long double> mymax (const complex<long double> x, const complex<long double> y) { return complex<long double> (max(x.real(), y.real()), max(x.imag(), y.imag())); } #endif - template<class T> inline T mymin () { return mymin(numeric_limits<T>::min(),-numeric_limits<T>::max()); } - template<> inline complex<float> mymin () { return complex<float> (mymin<float>(),mymin<float>()); } - template<> inline complex<double> mymin () { return complex<double> (mymin<double>(),mymin<double>()); } + template<class T> inline T mymin (const T& dummy) { return mymin(numeric_limits<T>::min(),-numeric_limits<T>::max()); } + template<> inline complex<float> mymin (const complex<float>& dummy) { return complex<float> (mymin<float>(dummy.real()),mymin<float>(dummy.imag())); } + template<> inline complex<double> mymin (const complex<double>& dummy) { return complex<double> (mymin<double>(dummy.real()),mymin<double>(dummy.imag())); } #ifdef LDBL_MAX - template<> inline complex<long double> mymin () { return complex<long double> (mymin<long double>(),mymin<long double>()); } + template<> inline complex<long double> mymin (const complex<long double>& dummy) { return complex<long double> (mymin<long double>(dummy.real()),mymin<long double>(dummy.imag())); } #endif - template<class T> inline T mymax () { return mymax(numeric_limits<T>::max(),-numeric_limits<T>::min()); } - template<> inline complex<float> mymax () { return complex<float> (mymax<float>(),mymax<float>()); } - template<> inline complex<double> mymax () { return complex<double> (mymax<double>(),mymax<double>()); } + template<class T> inline T mymax (const T& dummy) { return mymax(numeric_limits<T>::max(),-numeric_limits<T>::min()); } + template<> inline complex<float> mymax (const complex<float>& dummy) { return complex<float> (mymax<float>(dummy.real()),mymax<float>(dummy.imag())); } + template<> inline complex<double> mymax (const complex<double>& dummy) { return complex<double> (mymax<double>(dummy.real()),mymax<double>(dummy.imag())); } #ifdef LDBL_MAX - template<> inline complex<long double> mymax () { return complex<long double> (mymax<long double>(),mymax<long double>()); } + template<> inline complex<long double> mymax (const complex<long double>& dummy) { return complex<long double> (mymax<long double>(dummy.real()),mymax<long double>(dummy.imag())); } #endif // Poor man's RTTI - enum ared { do_count, do_minimum, do_maximum, do_product, do_sum, do_sum_abs, - do_sum_squared, do_average, do_norm1, do_norm2, do_norm_inf }; + enum ared { do_count, do_origin, do_minimum, do_maximum, do_product, do_sum, + do_sum_abs, do_sum_squared, do_average, do_norm1, do_norm2, + do_norm_inf }; @@ -94,13 +95,26 @@ namespace CarpetReduce { MPI_Op mpi_op () const { return MPI_SUM; } }; + struct origin : reduction { + origin () { } + ared thered () const { return do_origin; } + bool uses_cnt () const { return false; } + template<class T> + struct op { + static inline void initialise (T& accum) { accum = 0; } + static inline void reduce (T& accum, const T& val) { assert(0); } + static inline void finalise (T& accum, const T& cnt) { } + }; + MPI_Op mpi_op () const { return MPI_SUM; } + }; + struct minimum : reduction { minimum () { } ared thered () const { return do_minimum; } bool uses_cnt () const { return false; } template<class T> struct op { - static inline void initialise (T& accum) { accum = mymax<T>(); } + static inline void initialise (T& accum) { accum = mymax(accum); } static inline void reduce (T& accum, const T& val) { accum = mymin(accum, val); } static inline void finalise (T& accum, const T& cnt) { } }; @@ -113,7 +127,7 @@ namespace CarpetReduce { bool uses_cnt () const { return false; } template<class T> struct op { - static inline void initialise (T& accum) { accum = mymin<T>(); } + static inline void initialise (T& accum) { accum = mymin(accum); } static inline void reduce (T& accum, const T& val) { accum = mymax(accum, val); } static inline void finalise (T& accum, const T& cnt) { } }; @@ -307,6 +321,7 @@ namespace CarpetReduce { case N: { \ switch (red->thered()) { \ INITIALISE(count,T); \ + INITIALISE(origin,T); \ INITIALISE(minimum,T); \ INITIALISE(maximum,T); \ INITIALISE(product,T); \ @@ -435,6 +450,7 @@ namespace CarpetReduce { case N: { \ switch (red->thered()) { \ REDUCE(count,T); \ + REDUCE(origin,T); \ REDUCE(minimum,T); \ REDUCE(maximum,T); \ REDUCE(product,T); \ @@ -524,6 +540,7 @@ namespace CarpetReduce { case N: { \ switch (red->thered()) { \ FINALISE(count,T); \ + FINALISE(origin,T); \ FINALISE(minimum,T); \ FINALISE(maximum,T); \ FINALISE(product,T); \ @@ -678,7 +695,10 @@ namespace CarpetReduce { } // global mode - if (! reduce_arrays && reflevel == -1) { + if (! reduce_arrays && is_meta_mode()) { + CCTK_WARN (0, "Grid function reductions are not possible in meta mode"); + } + if (! reduce_arrays && is_global_mode()) { CCTK_WARN (0, "Grid function reduction operators in global mode are not yet implemented"); } @@ -690,53 +710,68 @@ namespace CarpetReduce { Initialise (cgh, proc, num_invars * num_outvals, &myoutvals[0], outtype, &mycounts[0], red); - BEGIN_LOCAL_COMPONENT_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) { - - assert (grpdim<=dim); - int lsh[dim], bbox[2*dim], nghostzones[dim]; - ierr = CCTK_GrouplshVI(cgh, grpdim, lsh, vi); - assert (!ierr); - ierr = CCTK_GroupbboxVI(cgh, 2*grpdim, bbox, vi); - assert (!ierr); - ierr = CCTK_GroupnghostzonesVI(cgh, grpdim, nghostzones, vi); - assert (!ierr); - for (int d=0; d<grpdim; ++d) { - assert (lsh[d]>=0); - assert (nghostzones[d]>=0 && 2*nghostzones[d]<=lsh[d]); - } - - vector<const void*> inarrays (num_invars); - for (int n=0; n<num_invars; ++n) { - inarrays[n] = CCTK_VarDataPtrI(cgh, 0, invars[n]); - assert (inarrays[n]); - } - - const int intype = CCTK_VarTypeI(vi); - for (int n=0; n<num_invars; ++n) { - assert (CCTK_VarTypeI(invars[n]) == intype); - } - - vect<int,dim> mylsh, mynghostzones; - vect<vect<int,2>,dim> mybbox; - for (int d=0; d<grpdim; ++d) { - mylsh[d] = lsh[d]; - mybbox[d][0] = bbox[2*d ]; - mybbox[d][1] = bbox[2*d+1]; - mynghostzones[d] = nghostzones[d]; - } - for (int d=grpdim; d<dim; ++d) { - mylsh[d] = 1; - mybbox[d][0] = 0; - mybbox[d][1] = 0; - mynghostzones[d] = 0; - } - - Reduce (cgh, proc, &mylsh[0], &mybbox[0][0], &mynghostzones[0], - num_invars, &inarrays[0], intype, - num_invars * num_outvals, &myoutvals[0], outtype, - &mycounts[0], red); - - } END_LOCAL_COMPONENT_LOOP; + int const saved_reflevel = reflevel; + int const saved_map = Carpet::map; + int const saved_component = component; + + BEGIN_GLOBAL_MODE(cgh) { + BEGIN_REFLEVEL_LOOP(cgh) { + BEGIN_MAP_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) { + BEGIN_LOCAL_COMPONENT_LOOP(cgh, reduce_arrays ? CCTK_ARRAY : CCTK_GF) { + if (reduce_arrays + || ((saved_reflevel==-1 || reflevel==saved_reflevel) + && (saved_map==-1 || Carpet::map==saved_map) + && (saved_component==-1 || component==saved_component))) { + + assert (grpdim<=dim); + int lsh[dim], bbox[2*dim], nghostzones[dim]; + ierr = CCTK_GrouplshVI(cgh, grpdim, lsh, vi); + assert (!ierr); + ierr = CCTK_GroupbboxVI(cgh, 2*grpdim, bbox, vi); + assert (!ierr); + ierr = CCTK_GroupnghostzonesVI(cgh, grpdim, nghostzones, vi); + assert (!ierr); + for (int d=0; d<grpdim; ++d) { + assert (lsh[d]>=0); + assert (nghostzones[d]>=0 && 2*nghostzones[d]<=lsh[d]); + } + + vector<const void*> inarrays (num_invars); + for (int n=0; n<num_invars; ++n) { + inarrays[n] = CCTK_VarDataPtrI(cgh, 0, invars[n]); + assert (inarrays[n]); + } + + const int intype = CCTK_VarTypeI(vi); + for (int n=0; n<num_invars; ++n) { + assert (CCTK_VarTypeI(invars[n]) == intype); + } + + vect<int,dim> mylsh, mynghostzones; + vect<vect<int,2>,dim> mybbox; + for (int d=0; d<grpdim; ++d) { + mylsh[d] = lsh[d]; + mybbox[d][0] = bbox[2*d ]; + mybbox[d][1] = bbox[2*d+1]; + mynghostzones[d] = nghostzones[d]; + } + for (int d=grpdim; d<dim; ++d) { + mylsh[d] = 1; + mybbox[d][0] = 0; + mybbox[d][1] = 0; + mynghostzones[d] = 0; + } + + Reduce (cgh, proc, &mylsh[0], &mybbox[0][0], &mynghostzones[0], + num_invars, &inarrays[0], intype, + num_invars * num_outvals, &myoutvals[0], outtype, + &mycounts[0], red); + + } + } END_LOCAL_COMPONENT_LOOP; + } END_MAP_LOOP; + } END_REFLEVEL_LOOP; + } END_GLOBAL_MODE; Finalise (cgh, proc, num_invars * num_outvals, outvals, outtype, &myoutvals[0], &mycounts[0], red); @@ -773,6 +808,7 @@ namespace CarpetReduce { } REDUCTION(count); + REDUCTION(origin); REDUCTION(minimum); REDUCTION(maximum); REDUCTION(product); @@ -791,6 +827,7 @@ namespace CarpetReduce { void CarpetReduceStartup () { CCTK_RegisterReductionOperator (count_GVs, "count"); + CCTK_RegisterReductionOperator (origin_GVs, "origin"); CCTK_RegisterReductionOperator (minimum_GVs, "minimum"); CCTK_RegisterReductionOperator (maximum_GVs, "maximum"); CCTK_RegisterReductionOperator (product_GVs, "product"); @@ -803,6 +840,7 @@ namespace CarpetReduce { CCTK_RegisterReductionOperator (norm_inf_GVs, "norm_inf"); CCTK_RegisterReductionArrayOperator (count_arrays, "count"); + CCTK_RegisterReductionArrayOperator (origin_arrays, "origin"); CCTK_RegisterReductionArrayOperator (minimum_arrays, "minimum"); CCTK_RegisterReductionArrayOperator (maximum_arrays, "maximum"); CCTK_RegisterReductionArrayOperator (product_arrays, "product"); diff --git a/Carpet/CarpetRegrid/interface.ccl b/Carpet/CarpetRegrid/interface.ccl index 3c6352fda..efc701a73 100644 --- a/Carpet/CarpetRegrid/interface.ccl +++ b/Carpet/CarpetRegrid/interface.ccl @@ -1,5 +1,5 @@ # Interface definition for thorn CarpetRegrid -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/interface.ccl,v 1.5 2004/01/15 09:45:58 cott Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/interface.ccl,v 1.6 2004/01/25 14:57:30 schnetter Exp $ implements: CarpetRegrid @@ -14,8 +14,31 @@ uses include header: vect.hh uses include header: gf.hh uses include header: gh.hh -CCTK_INT FUNCTION RegridLevel (CCTK_POINTER_TO_CONST IN cctkGH, \ - CCTK_INT IN current_reflevel, \ + + +# The true prototype of the routine below: +# int Carpet_Regrid (const cGH * cctkGH, +# gh<dim>::rexts * bbsss, +# gh<dim>::rbnds * obss, +# gh<dim>::rprocs * pss); +CCTK_INT FUNCTION Carpet_Regrid (CCTK_POINTER_TO_CONST IN cctkGH, \ + CCTK_INT IN reflevel, \ + CCTK_INT IN map, \ + CCTK_INT IN size, \ + CCTK_INT IN ARRAY nboundaryzones, \ + CCTK_INT IN ARRAY is_internal, \ + CCTK_INT IN ARRAY is_staggered, \ + CCTK_INT IN ARRAY shiftout, \ + CCTK_POINTER IN bsss, \ + CCTK_POINTER IN obss, \ + CCTK_POINTER IN pss) +PROVIDES FUNCTION Carpet_Regrid WITH CarpetRegrid_Regrid LANGUAGE C + + + + +CCTK_INT FUNCTION RegridLevel (CCTK_POINTER_TO_CONST IN cctkGH, \ + CCTK_INT IN current_reflevel, \ CCTK_INT IN current_max_reflevel, \ CCTK_INT IN max_reflevels) USES FUNCTION RegridLevel diff --git a/Carpet/CarpetRegrid/param.ccl b/Carpet/CarpetRegrid/param.ccl index 9f2dc9bf9..1de876c73 100644 --- a/Carpet/CarpetRegrid/param.ccl +++ b/Carpet/CarpetRegrid/param.ccl @@ -1,5 +1,5 @@ # Parameter definitions for thorn CarpetRegrid -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/param.ccl,v 1.13 2004/01/13 13:50:05 hawke Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/param.ccl,v 1.14 2004/01/25 14:57:30 schnetter Exp $ @@ -17,16 +17,19 @@ CCTK_INT regrid_every "Regrid every n time steps" STEERABLE=always 1:* :: "regrid every n time steps" } 0 -CCTK_INT activate_newlevels_on_regrid "When regridding, activate this many new levels (if possible)" -# Note that this feature will STEER the parameter refinement_levels!!! -# It does this by adding activate_newlevels_on_regrid to refinement levels + + +CCTK_INT activate_newlevels_on_regrid "When regridding, activate this many new levels (if possible). Note that this will steer the parameter refinement_levels." STEERABLE=always { - : :: "Number of new levels to activate. Negative numbers mean to de-activate" + : :: "Number of new levels to activate (negative numbers deactivate)" } 0 +CCTK_INT activate_next "The next iteration at which new levels should be activated" STEERABLE=always +{ + 0: :: "Note that this parameter is steered when new levels are activated" +} 1 - -BOOLEAN outside_boundary_points[3] "On finer grids, where the upper grid boundary is adjacent to the outer boundary, put points outside the outer boundary (needed e.g. for periodicity)" +BOOLEAN regrid_from_function "Regridding criteria from the aliased function?" { } "no" @@ -45,6 +48,20 @@ KEYWORD refined_regions "Regions where the grid is refined" STEERABLE=always +# Region specifications for centre refinement + +BOOLEAN symmetry_x "Refine the lower half in x-direction" +{ +} "no" +BOOLEAN symmetry_y "Refine the lower half in y-direction" +{ +} "no" +BOOLEAN symmetry_z "Refine the lower half in z-direction" +{ +} "no" + + + # Region specifications for manual gridpoint refinement CCTK_INT l1ixmin "Lower boundary of level 1 box in x-direction" STEERABLE=always @@ -268,10 +285,3 @@ CCTK_STRING errorvar "Name of grid function that contains the error" STEERABLE=a { ".*" :: "must be the name of a grid function" } "" - - -# Should we set the regridding criteria from an aliased function? - -BOOLEAN regrid_from_function "Regridding criteria from the aliased function?" -{ -} "no" diff --git a/Carpet/CarpetRegrid/src/automatic.cc b/Carpet/CarpetRegrid/src/automatic.cc index a3b4da055..6add8c770 100644 --- a/Carpet/CarpetRegrid/src/automatic.cc +++ b/Carpet/CarpetRegrid/src/automatic.cc @@ -16,7 +16,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/automatic.cc,v 1.5 2004/08/04 16:25:58 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/automatic.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_automatic_cc); } @@ -31,12 +31,22 @@ namespace CarpetRegrid { int Automatic (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + assert (refinement_levels >= 1); assert (bbsss.size() >= 1); @@ -54,19 +64,15 @@ namespace CarpetRegrid { assert (CCTK_VarTypeI(vi) == CCTK_VARIABLE_REAL); assert (CCTK_GroupDimI(gi) == dim); - assert (arrdata.at(gi).at(Carpet::map).data.at(vi-v1)); - const gf<CCTK_REAL,dim>& errorgf + assert (arrdata.at(gi).at(map).data.at(vi-v1)); + const gf<CCTK_REAL,dim>& errorvar = (*dynamic_cast<const gf<CCTK_REAL,dim>*> - (arrdata.at(gi).at(Carpet::map).data.at(vi-v1))); - - assert (! smart_outer_boundaries); + (arrdata.at(gi).at(map).data.at(vi-v1))); vector<ibbox> bbs; gh<dim>::cbnds obs; Automatic_OneLevel - (cctkGH, hh, - reflevel, min(reflevels+1, maxreflevels), - minwidth, minfraction, maxerror, errorgf, + (cctkGH, hh, reflevel, minwidth, minfraction, maxerror, errorvar, bbs, obs); // make multiprocessor aware @@ -75,7 +81,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); @@ -106,32 +115,31 @@ namespace CarpetRegrid { void Automatic_OneLevel (const cGH * const cctkGH, const gh<dim> & hh, - const int rl, - const int numrl, + const int reflevel, const int minwidth, const CCTK_REAL minfraction, const CCTK_REAL maxerror, - const gf<CCTK_REAL,dim> & errorgf, + const gf<CCTK_REAL,dim> & errorvar, vector<ibbox> & bbs, vector<bbvect> & obs) { - if (rl+1 >= numrl) return; + if (reflevel+1 >= maxreflevels) return; // Arbitrary const int tl = 0; const int ml = 0; -// cout << endl << "MRA: Choosing regions to refine in " << hh.components(rl) << " components" << endl; +// cout << endl << "MRA: Choosing regions to refine in " << hh.components(reflevel) << " components" << endl; list<ibbox> bbl; - for (int c=0; c<hh.components(rl); ++c) { - const ibbox region = hh.extents.at(rl).at(c).at(ml); + for (int c=0; c<hh.components(reflevel); ++c) { + const ibbox region = hh.extents.at(reflevel).at(c).at(ml); assert (! region.empty()); - const data<CCTK_REAL,dim>& errordata = *errorgf(tl,rl,c,ml); + const data<CCTK_REAL,dim>& errdata = *errorvar(tl,reflevel,c,ml); Automatic_Recursive (cctkGH, hh, minwidth, minfraction, maxerror, - errordata, bbl, region); + errdata, bbl, region); } // int numpoints = 0; @@ -177,7 +185,7 @@ namespace CarpetRegrid { const int minwidth, const CCTK_REAL minfraction, const CCTK_REAL maxerror, - const data<CCTK_REAL,dim> & errordata, + const data<CCTK_REAL,dim> & errorvar, list<ibbox> & bbl, const ibbox & region) { @@ -188,12 +196,8 @@ namespace CarpetRegrid { // (this doesn't work yet on multiple processors) assert (CCTK_nProcs(cctkGH)==1); int cnt = 0; - { - ibbox::iterator it=region.begin(); - do { - if (errordata[*it] > maxerror) ++cnt; - ++it; - } while (it!=region.end()); + for (ibbox::iterator it=region.begin(); it!=region.end(); ++it) { + if (errorvar[*it] > maxerror) ++cnt; } const CCTK_REAL fraction = (CCTK_REAL)cnt / region.size(); const int width = maxval(region.shape() / region.stride()); @@ -227,9 +231,9 @@ namespace CarpetRegrid { assert (region1 + region2 == region); list<ibbox> bbl1, bbl2; Automatic_Recursive (cctkGH, hh, minwidth, minfraction, maxerror, - errordata, bbl1, region1); + errorvar, bbl1, region1); Automatic_Recursive (cctkGH, hh, minwidth, minfraction, maxerror, - errordata, bbl2, region2); + errorvar, bbl2, region2); // Combine regions if possible up2 += str-str/reffact; up2[dir] = lo2[dir]; diff --git a/Carpet/CarpetRegrid/src/baselevel.cc b/Carpet/CarpetRegrid/src/baselevel.cc index c380c6844..18cc8a95c 100644 --- a/Carpet/CarpetRegrid/src/baselevel.cc +++ b/Carpet/CarpetRegrid/src/baselevel.cc @@ -9,7 +9,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/baselevel.cc,v 1.2 2004/04/18 13:29:43 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/baselevel.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_baselevel_cc); } @@ -24,12 +24,22 @@ namespace CarpetRegrid { int BaseLevel (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + assert (refinement_levels == 1); assert (bbsss.size() == 1); diff --git a/Carpet/CarpetRegrid/src/centre.cc b/Carpet/CarpetRegrid/src/centre.cc index 7599324ea..46a23f794 100644 --- a/Carpet/CarpetRegrid/src/centre.cc +++ b/Carpet/CarpetRegrid/src/centre.cc @@ -9,7 +9,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/centre.cc,v 1.4 2004/04/28 15:45:25 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/centre.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_centre_cc); } @@ -24,16 +24,26 @@ namespace CarpetRegrid { int Centre (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + assert (refinement_levels >= 1); // do nothing if the levels already exist - if (reflevel == refinement_levels) return 0; + if (bbsss.size() == refinement_levels) return 0; assert (bbsss.size() >= 1); @@ -48,8 +58,6 @@ namespace CarpetRegrid { ivect rlb = hh.baseextent.lower(); ivect rub = hh.baseextent.upper(); - assert (! smart_outer_boundaries); - for (size_t rl=1; rl<bbsss.size(); ++rl) { // save old values @@ -81,7 +89,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); bbsss.at(rl) = bbss; obss.at(rl) = obs; diff --git a/Carpet/CarpetRegrid/src/make.code.defn b/Carpet/CarpetRegrid/src/make.code.defn index ec45ed348..0b692a316 100644 --- a/Carpet/CarpetRegrid/src/make.code.defn +++ b/Carpet/CarpetRegrid/src/make.code.defn @@ -1,8 +1,16 @@ # Main make.code.defn file for thorn CarpetRegrid -# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/make.code.defn,v 1.2 2002/05/16 23:25:54 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/make.code.defn,v 1.3 2004/01/25 14:57:30 schnetter Exp $ # Source files in this directory -SRCS = regrid.cc paramcheck.cc +SRCS = automatic.cc \ + baselevel.cc \ + centre.cc \ + manualcoordinatelist.cc \ + manualcoordinates.cc \ + manualgridpointlist.cc \ + manualgridpoints.cc \ + regrid.cc \ + paramcheck.cc # Subdirectories containing source files SUBDIRS = diff --git a/Carpet/CarpetRegrid/src/manualcoordinatelist.cc b/Carpet/CarpetRegrid/src/manualcoordinatelist.cc index 10e40ecbd..92ab6900a 100644 --- a/Carpet/CarpetRegrid/src/manualcoordinatelist.cc +++ b/Carpet/CarpetRegrid/src/manualcoordinatelist.cc @@ -1,7 +1,6 @@ -#include <cassert> -#include <cmath> -#include <cstring> -#include <sstream> +#include <assert.h> +#include <string.h> + #include <vector> #include "cctk.h" @@ -13,7 +12,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualcoordinatelist.cc,v 1.12 2004/08/14 07:42:00 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualcoordinatelist.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_manualcoordinatelist_cc); } @@ -28,35 +27,29 @@ namespace CarpetRegrid { int ManualCoordinateList (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; - int ierr; + + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); assert (refinement_levels >= 1); // do nothing if the levels already exist - if (reflevel == refinement_levels) return 0; + if (bbsss.size() == refinement_levels) return 0; assert (bbsss.size() >= 1); - jjvect nboundaryzones, is_internal, is_staggered, shiftout; - ierr = GetBoundarySpecification - (2*dim, &nboundaryzones[0][0], &is_internal[0][0], - &is_staggered[0][0], &shiftout[0][0]); - assert (!ierr); - rvect physical_min, physical_max; - rvect interior_min, interior_max; - rvect exterior_min, exterior_max; - rvect base_spacing; - ierr = GetDomainSpecification - (dim, &physical_min[0], &physical_max[0], - &interior_min[0], &interior_max[0], - &exterior_min[0], &exterior_max[0], &base_spacing[0]); - assert (!ierr); - bbsss.resize (refinement_levels); obss.resize (refinement_levels); pss.resize (refinement_levels); @@ -70,79 +63,39 @@ namespace CarpetRegrid { CCTK_WARN (0, "Could not parse parameter \"coordinates\""); } } - + vector<vector<bbvect> > newobss; - if (smart_outer_boundaries) { - // TODO: - // assert (domain_from_coordbase); - + if (strcmp(outerbounds, "") !=0 ) { + istringstream ob_str (outerbounds); + try { + ob_str >> newobss; + } catch (input_error) { + CCTK_WARN (0, "Could not parse parameter \"outerbounds\""); + } + bool good = newobss.size() == newbbss.size(); + if (good) { + for (size_t rl=0; rl<newobss.size(); ++rl) { + good = good && newobss.at(rl).size() == newbbss.at(rl).size(); + } + } + if (! good) { + cout << "coordinates: " << newbbss << endl; + cout << "outerbounds: " << newobss << endl; + CCTK_WARN (0, "The parameters \"outerbounds\" and \"coordinates\" must have the same structure"); + } + } else { newobss.resize(newbbss.size()); for (size_t rl=0; rl<newobss.size(); ++rl) { newobss.at(rl).resize(newbbss.at(rl).size()); for (size_t c=0; c<newobss.at(rl).size(); ++c) { - for (int d=0; d<dim; ++d) { - assert (mglevel==0); - rvect const spacing = base_spacing * pow(CCTK_REAL(mgfact), basemglevel) / ipow(reffact, rl+1); - ierr = ConvertFromPhysicalBoundary - (dim, &physical_min[0], &physical_max[0], - &interior_min[0], &interior_max[0], - &exterior_min[0], &exterior_max[0], &spacing[0]); - assert (!ierr); - newobss.at(rl).at(c)[d][0] = abs(newbbss.at(rl).at(c).lower()[d] - physical_min[d]) < 1.0e-6 * spacing[d]; - if (newobss.at(rl).at(c)[d][0]) { - rvect lo = newbbss.at(rl).at(c).lower(); - rvect up = newbbss.at(rl).at(c).upper(); - rvect str = newbbss.at(rl).at(c).stride(); - lo[d] = exterior_min[d]; - newbbss.at(rl).at(c) = rbbox(lo, up, str); - } - newobss.at(rl).at(c)[d][1] = abs(newbbss.at(rl).at(c).upper()[d] - physical_max[d]) < 1.0e-6 * base_spacing[d] / ipow(reffact, rl); - if (newobss.at(rl).at(c)[d][1]) { - rvect lo = newbbss.at(rl).at(c).lower(); - rvect up = newbbss.at(rl).at(c).upper(); - rvect str = newbbss.at(rl).at(c).stride(); - up[d] = exterior_max[d]; - newbbss.at(rl).at(c) = rbbox(lo, up, str); - } - } + newobss.at(rl).at(c) = bbvect(false); } } - - } else { // if ! smart_outer_boundaries - - if (strcmp(outerbounds, "") !=0 ) { - istringstream ob_str (outerbounds); - try { - ob_str >> newobss; - } catch (input_error) { - CCTK_WARN (0, "Could not parse parameter \"outerbounds\""); - } - bool good = newobss.size() == newbbss.size(); - if (good) { - for (size_t rl=0; rl<newobss.size(); ++rl) { - good = good && newobss.at(rl).size() == newbbss.at(rl).size(); - } - } - if (! good) { - cout << "coordinates: " << newbbss << endl; - cout << "outerbounds: " << newobss << endl; - CCTK_WARN (0, "The parameters \"outerbounds\" and \"coordinates\" must have the same structure"); - } - } else { - newobss.resize(newbbss.size()); - for (size_t rl=0; rl<newobss.size(); ++rl) { - newobss.at(rl).resize(newbbss.at(rl).size()); - for (size_t c=0; c<newobss.at(rl).size(); ++c) { - newobss.at(rl).at(c) = bbvect(false); - } - } - } - - } // if ! smart_outer_boundaries + } if (newbbss.size() < refinement_levels-1) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "The parameter \"coordinates\" must contain at least \"refinement_levels-1\" (here: %d) levels", int(refinement_levels-1)); + "The parameter \"coordinates\" must contain at least \"refinement_levels-1\" (here: %d) levels", (int)refinement_levels-1); } for (size_t rl=1; rl<refinement_levels; ++rl) { @@ -156,10 +109,6 @@ namespace CarpetRegrid { for (size_t c=0; c<newbbss.at(rl-1).size(); ++c) { rbbox const & ext = newbbss.at(rl-1).at(c); bbvect const & ob = newobss.at(rl-1).at(c); - // TODO: - // assert (domain_from_coordbase); - rvect const spacing = base_spacing * pow(CCTK_REAL(mgfact), basemglevel) / ipow(reffact, rl); - assert (all(abs(ext.stride() - spacing) < spacing * 1.0e-10)); ManualCoordinates_OneLevel (cctkGH, hh, rl, refinement_levels, ext.lower(), ext.upper(), ob, bbs, obs); @@ -171,7 +120,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); bbsss.at(rl) = bbss; obss.at(rl) = obs; diff --git a/Carpet/CarpetRegrid/src/manualcoordinates.cc b/Carpet/CarpetRegrid/src/manualcoordinates.cc index 91018cf95..1e3719f18 100644 --- a/Carpet/CarpetRegrid/src/manualcoordinates.cc +++ b/Carpet/CarpetRegrid/src/manualcoordinates.cc @@ -11,7 +11,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualcoordinates.cc,v 1.5 2004/04/28 15:45:25 schnetter Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualcoordinates.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_manualcoordinates_cc); } @@ -26,19 +26,29 @@ namespace CarpetRegrid { int ManualCoordinates (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + if (refinement_levels > 4) { CCTK_WARN (0, "Cannot currently specify manual refinement regions for more than 4 refinement levels"); } assert (refinement_levels >= 1 && refinement_levels <= 4); // do nothing if the levels already exist - if (reflevel == refinement_levels) return 0; + if (bbsss.size() == refinement_levels) return 0; assert (bbsss.size() >= 1); @@ -54,8 +64,6 @@ namespace CarpetRegrid { lower.at(2) = rvect (l3xmin, l3ymin, l3zmin); upper.at(2) = rvect (l3xmax, l3ymax, l3zmax); - assert (! smart_outer_boundaries); - for (size_t rl=1; rl<bbsss.size(); ++rl) { bbvect const ob (false); @@ -73,7 +81,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); bbsss.at(rl) = bbss; obss.at(rl) = obs; @@ -88,27 +99,27 @@ namespace CarpetRegrid { void ManualCoordinates_OneLevel (const cGH * const cctkGH, const gh<dim> & hh, - const int rl, - const int numrl, + const int reflevel, + const int reflevels, const rvect lower, const rvect upper, const bbvect obound, vector<ibbox> & bbs, vector<bbvect> & obs) { - if (rl >= numrl) return; + if (reflevel >= reflevels) return; - jvect const ilower = pos2int (cctkGH, hh, lower, rl); - jvect const iupper = pos2int (cctkGH, hh, upper, rl); + jvect const ilower = pos2int (cctkGH, hh, lower, reflevel); + jvect const iupper = pos2int (cctkGH, hh, upper, reflevel); ManualGridpoints_OneLevel - (cctkGH, hh, rl, numrl, ilower, iupper, obound, bbs, obs); + (cctkGH, hh, reflevel, reflevels, ilower, iupper, obound, bbs, obs); } ivect delta2int (const cGH * const cctkGH, const gh<dim>& hh, - const rvect & rpos, const int rl) + const rvect & rpos, const int reflevel) { rvect global_lower, global_upper; for (int d=0; d<dim; ++d) { @@ -121,13 +132,15 @@ namespace CarpetRegrid { } const ivect global_extent (hh.baseextent.upper() - hh.baseextent.lower()); + CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; + const rvect scale = rvect(global_extent) / (global_upper - global_lower); - const int levfac = ipow(hh.reffact, rl); + const int levfac = ipow(hh.reffact, reflevel); assert (all (hh.baseextent.stride() % levfac == 0)); const ivect istride = hh.baseextent.stride() / levfac; const ivect ipos - = ivect(floor(rpos * scale / rvect(istride) + 0.5)) * istride; + = ivect(::map(rfloor, rpos * scale / rvect(istride) + 0.5)) * istride; const rvect apos = rpos * scale; assert (all(abs(apos - rvect(ipos)) < rvect(istride)*0.01)); @@ -138,7 +151,7 @@ namespace CarpetRegrid { ivect pos2int (const cGH* const cctkGH, const gh<dim>& hh, - const rvect & rpos, const int rl) + const rvect & rpos, const int reflevel) { rvect global_lower, global_upper; for (int d=0; d<dim; ++d) { @@ -151,14 +164,16 @@ namespace CarpetRegrid { } const ivect global_extent (hh.baseextent.upper() - hh.baseextent.lower()); + CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; + const rvect scale = rvect(global_extent) / (global_upper - global_lower); - const int levfac = ipow(hh.reffact, rl); + const int levfac = ipow(hh.reffact, reflevel); assert (all (hh.baseextent.stride() % levfac == 0)); const ivect istride = hh.baseextent.stride() / levfac; const ivect ipos - = (ivect(floor((rpos - global_lower) * scale / rvect(istride) + 0.5)) - * istride); + = ivect(::map(rfloor, (rpos - global_lower) * scale / rvect(istride) + + 0.5)) * istride; return ipos; } diff --git a/Carpet/CarpetRegrid/src/manualgridpointlist.cc b/Carpet/CarpetRegrid/src/manualgridpointlist.cc index e53866e6c..6976b982b 100644 --- a/Carpet/CarpetRegrid/src/manualgridpointlist.cc +++ b/Carpet/CarpetRegrid/src/manualgridpointlist.cc @@ -1,7 +1,6 @@ #include <assert.h> #include <string.h> -#include <sstream> #include <vector> #include "cctk.h" @@ -13,7 +12,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualgridpointlist.cc,v 1.4 2004/07/02 10:14:51 tradke Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualgridpointlist.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_manualgridpointlist_cc); } @@ -28,16 +27,26 @@ namespace CarpetRegrid { int ManualGridpointList (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + assert (refinement_levels >= 1); // do nothing if the levels already exist - if (reflevel == refinement_levels) return 0; + if (bbsss.size() == refinement_levels) return 0; assert (bbsss.size() >= 1); @@ -111,7 +120,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); bbsss.at(rl) = bbss; obss.at(rl) = obs; diff --git a/Carpet/CarpetRegrid/src/manualgridpoints.cc b/Carpet/CarpetRegrid/src/manualgridpoints.cc index 2a8bedb5e..e8cd6c17a 100644 --- a/Carpet/CarpetRegrid/src/manualgridpoints.cc +++ b/Carpet/CarpetRegrid/src/manualgridpoints.cc @@ -1,6 +1,5 @@ #include <assert.h> -#include <sstream> #include <vector> #include "cctk.h" @@ -12,7 +11,7 @@ #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualgridpoints.cc,v 1.5 2004/07/02 10:14:51 tradke Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/manualgridpoints.cc,v 1.1 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_manualgridpoints_cc); } @@ -27,19 +26,29 @@ namespace CarpetRegrid { int ManualGridpoints (cGH const * const cctkGH, gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, gh<dim>::rexts & bbsss, gh<dim>::rbnds & obss, gh<dim>::rprocs & pss) { DECLARE_CCTK_PARAMETERS; + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + if (refinement_levels > 4) { CCTK_WARN (0, "Cannot currently specify manual refinement regions for more than 4 refinement levels"); } assert (refinement_levels >= 1 && refinement_levels <= 4); // do nothing if the levels already exist - if (reflevel == refinement_levels) return 0; + if (bbsss.size() == refinement_levels) return 0; assert (bbsss.size() >= 1); @@ -55,8 +64,6 @@ namespace CarpetRegrid { ilower.at(2) = ivect (l3ixmin, l3iymin, l3izmin); iupper.at(2) = ivect (l3ixmax, l3iymax, l3izmax); - assert (! smart_outer_boundaries); - for (size_t rl=1; rl<bbsss.size(); ++rl) { bbvect const ob (false); @@ -65,7 +72,7 @@ namespace CarpetRegrid { gh<dim>::cbnds obs; ManualGridpoints_OneLevel - (cctkGH, hh, rl,refinement_levels, + (cctkGH, hh, rl, refinement_levels, ilower.at(rl-1), iupper.at(rl-1), ob, bbs, obs); // make multiprocessor aware @@ -74,7 +81,10 @@ namespace CarpetRegrid { // make multigrid aware vector<vector<ibbox> > bbss; - MakeMultigridBoxes (cctkGH, bbs, obs, bbss); + MakeMultigridBoxes + (cctkGH, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbs, obs, bbss); bbsss.at(rl) = bbss; obss.at(rl) = obs; @@ -89,36 +99,38 @@ namespace CarpetRegrid { void ManualGridpoints_OneLevel (const cGH * const cctkGH, const gh<dim> & hh, - const int rl, - const int numrl, + const int reflevel, + const int reflevels, const ivect ilower, const ivect iupper, const bbvect obound, vector<ibbox> & bbs, vector<bbvect> & obs) { + if (reflevel >= reflevels) return; + const ivect rstr = hh.baseextent.stride(); const ivect rlb = hh.baseextent.lower(); const ivect rub = hh.baseextent.upper(); - const int levfac = ipow(hh.reffact, rl); + const int levfac = ipow(hh.reffact, reflevel); assert (all (rstr % levfac == 0)); const ivect str (rstr / levfac); const ivect lb (ilower); const ivect ub (iupper); if (! all(lb>=rlb && ub<=rub)) { ostringstream buf; - buf << "The refinement region boundaries for refinement level #" << rl << " are not within the main grid. Allowed are the grid point boundaries " << rlb << " - " << rub << "; specified were " << lb << " - " << ub << ends; + buf << "The refinement region boundaries for refinement level #" << reflevel << " are not within the main grid. Allowed are the grid point boundaries " << rlb << " - " << rub << "; specified were " << lb << " - " << ub << ends; CCTK_WARN (0, buf.str().c_str()); } if (! all(lb<=ub)) { ostringstream buf; - buf << "The refinement region boundaries for refinement level #" << rl << " have the upper boundary (" << ub << ") less than the lower boundary (" << lb << ")" << ends; + buf << "The refinement region boundaries for refinement level #" << reflevel << " have the upper boundary (" << ub << ") less than the lower boundary (" << lb << ")" << ends; CCTK_WARN (0, buf.str().c_str()); } if (! all(lb%str==0 && ub%str==0)) { CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "The refinement region boundaries for refinement level #%d are not a multiple of the stride for that level", rl); + "The refinement region boundaries for refinement level #%d are not a multiple of the stride for that level", reflevel); } assert (all(lb>=rlb && ub<=rub)); assert (all(lb<=ub)); diff --git a/Carpet/CarpetRegrid/src/regrid.cc b/Carpet/CarpetRegrid/src/regrid.cc index 82f6e0954..3c6b8b0a7 100644 --- a/Carpet/CarpetRegrid/src/regrid.cc +++ b/Carpet/CarpetRegrid/src/regrid.cc @@ -1,81 +1,70 @@ #include <assert.h> -#include <math.h> -#include <stdarg.h> -#include <stdlib.h> -#include <string.h> -#include <algorithm> -#include <list> #include <sstream> #include <string> -#include <vector> #include "cctk.h" #include "cctk_Parameters.h" -#include "bbox.hh" -#include "bboxset.hh" -#include "defs.hh" #include "gh.hh" -#include "gf.hh" #include "vect.hh" #include "carpet.hh" #include "regrid.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/regrid.cc,v 1.33 2004/01/15 09:45:58 cott Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/regrid.cc,v 1.34 2004/01/25 14:57:30 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetRegrid_regrid_cc); } -#undef AUTOMATIC_BOUNDARIES - - - namespace CarpetRegrid { using namespace std; using namespace Carpet; - int initial_refinement_levels = 0; // Store the initial number of - // levels to refine for correct - // progressive MR. - // Note that this does not need - // to be checkpointed as it is - // reset at the beginning of every - // iteration. - int last_regridded_iteration = -1; // The last iteration where any - // regridding was done. - - int CarpetRegridStartup () - { - RegisterRegridRoutine (CarpetRegridRegrid); - return 0; - } - - - - int CarpetRegridRegrid (const cGH * const cctkGH, - gh<dim>::rexts& bbsss, - gh<dim>::rbnds& obss, - gh<dim>::rprocs& pss) + CCTK_INT CarpetRegrid_Regrid (CCTK_POINTER_TO_CONST const cctkGH_, + CCTK_INT const reflevel, + CCTK_INT const map, + CCTK_INT const size, + CCTK_INT const * const nboundaryzones_, + CCTK_INT const * const is_internal_, + CCTK_INT const * const is_staggered_, + CCTK_INT const * const shiftout_, + CCTK_POINTER const bbsss_, + CCTK_POINTER const obss_, + CCTK_POINTER const pss_) { DECLARE_CCTK_PARAMETERS; + const cGH * const cctkGH = (const cGH *) cctkGH_; + + assert (reflevel>=0 && reflevel<maxreflevels); + assert (map>=0 && map<maps); + + assert (size == 2*dim); + jjvect const nboundaryzones (* (jjvect const *) nboundaryzones_); + jjvect const is_internal (* (jjvect const *) is_internal_); + jjvect const is_staggered (* (jjvect const *) is_staggered_); + jjvect const shiftout (* (jjvect const *) shiftout_); + + gh<dim>::rexts & bbsss = * (gh<dim>::rexts *) bbsss_; + gh<dim>::rbnds & obss = * (gh<dim>::rbnds *) obss_; + gh<dim>::rprocs & pss = * (gh<dim>::rprocs *) pss_; + + gh<dim> const & hh = *vhh.at(map); + + assert (is_meta_mode()); + + + assert (regrid_every == -1 || regrid_every == 0 || regrid_every % maxmglevelfact == 0); // Return if no regridding is desired if (regrid_every == -1) return 0; -#if 0 - // Return if this is not the main hierarchy - if (mglevel != 0) return 0; -#endif - assert (mglevel == -1); - // Return if we want to regrid during initial data only, and this // is not the time for initial data if (regrid_every == 0 && cctkGH->cctk_iteration != 0) return 0; @@ -85,833 +74,105 @@ namespace CarpetRegrid { && cctkGH->cctk_iteration % regrid_every != 0) { return 0; } - - // Find the number of levels that should be active. - int newnumlevels; - - // Is this the first time that we have regridded this iteration? - if (cctkGH->cctk_iteration > last_regridded_iteration) { - last_regridded_iteration = cctkGH->cctk_iteration; - initial_refinement_levels = refinement_levels; + + + + // Steer parameters + if (activate_newlevels_on_regrid != 0) { + if (cctkGH->cctk_iteration >= activate_next) { + const int newnumlevels + = min(refinement_levels + activate_newlevels_on_regrid, + maxreflevels); + assert (newnumlevels>0 && newnumlevels<=maxreflevels); + + *const_cast<CCTK_INT*>(&activate_next) = cctkGH->cctk_iteration + 1; + ostringstream next; + next << activate_next; + CCTK_ParameterSet + ("activate_next", "CarpetRegrid", next.str().c_str()); + + *const_cast<CCTK_INT*>(&refinement_levels) = newnumlevels; + ostringstream param; + param << refinement_levels; + CCTK_ParameterSet + ("refinement_levels", "CarpetRegrid", param.str().c_str()); + + } } - if (cctkGH->cctk_iteration == 0) { - newnumlevels = refinement_levels; - } else { - if (regrid_from_function) { - if (CCTK_IsFunctionAliased("RegridLevel")) { - int tempnewnumlevels = 0; - newnumlevels = 0; - BEGIN_MGLEVEL_LOOP(cctkGH) { - tempnewnumlevels = - RegridLevel(cctkGH, reflevel, refinement_levels,maxreflevels); - newnumlevels = max(tempnewnumlevels, newnumlevels); - } END_MGLEVEL_LOOP; - } else { - CCTK_WARN(1, "No thorn has provided the function " - "\"RegridLevel\". Regridding will not be done."); - } - } else { - // The standard progressive MR number of levels. - newnumlevels = initial_refinement_levels + - activate_newlevels_on_regrid; + if (regrid_from_function) { + if (! CCTK_IsFunctionAliased("RegridLevel")) { + CCTK_WARN (0, "No thorn has provided the function \"RegridLevel\""); } + const int newnumlevels + = RegridLevel (cctkGH, reflevel, refinement_levels, maxreflevels); + assert (newnumlevels>0 && newnumlevels<=maxreflevels); + + *const_cast<CCTK_INT*>(&refinement_levels) = newnumlevels; + ostringstream param; + param << refinement_levels; + CCTK_ParameterSet + ("refinement_levels", "CarpetRegrid", param.str().c_str()); } - - newnumlevels = min(newnumlevels, maxreflevels); - - if ( ( newnumlevels >= 1) && (newnumlevels <= maxreflevels )) { - char numlevelstring[10]; - sprintf(numlevelstring,"%d",newnumlevels); - CCTK_ParameterSet("refinement_levels","carpetregrid",numlevelstring); - } - list<ibbox> bbl; - list<bvect> obl; + + + int do_recompose; if (CCTK_EQUALS(refined_regions, "none")) { - MakeRegions_BaseLevel (cctkGH, bbl, obl); - + do_recompose = BaseLevel + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "centre")) { - MakeRegions_RefineCentre (cctkGH, newnumlevels, bbl, obl); - + do_recompose = Centre + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "manual-gridpoints")) { - if (refinement_levels > 4) { - CCTK_WARN (0, "Cannot currently specify manual refinement regions for more than 4 refinement levels"); - } - assert (refinement_levels<=4); - vector<ivect> lower(3), upper(3); - lower[0] = ivect (l1ixmin, l1iymin, l1izmin); - upper[0] = ivect (l1ixmax, l1iymax, l1izmax); - lower[1] = ivect (l2ixmin, l2iymin, l2izmin); - upper[1] = ivect (l2ixmax, l2iymax, l2izmax); - lower[2] = ivect (l3ixmin, l3iymin, l3izmin); - upper[2] = ivect (l3ixmax, l3iymax, l3izmax); - MakeRegions_AsSpecified (cctkGH, newnumlevels, lower, upper, - bbl, obl); - + do_recompose = ManualGridpoints + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "manual-coordinates")) { - if (refinement_levels > 4) { - CCTK_WARN (0, "Cannot currently specify manual refinement regions for more than 4 refinement levels"); - } - assert (refinement_levels<=4); - vector<rvect> lower(3), upper(3); - lower[0] = rvect (l1xmin, l1ymin, l1zmin); - upper[0] = rvect (l1xmax, l1ymax, l1zmax); - lower[1] = rvect (l2xmin, l2ymin, l2zmin); - upper[1] = rvect (l2xmax, l2ymax, l2zmax); - lower[2] = rvect (l3xmin, l3ymin, l3zmin); - upper[2] = rvect (l3xmax, l3ymax, l3zmax); - MakeRegions_AsSpecified (cctkGH, newnumlevels, lower, upper, - bbl, obl); - + do_recompose = ManualCoordinates + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "manual-gridpoint-list")) { - vector<vector<ibbox> > bbss; - if (strcmp(gridpoints, "") != 0) { - istringstream gp_str(gridpoints); - gp_str >> bbss; - } - - vector<vector<vect<vect<bool,2>,dim> > > obss; - if (strcmp(outerbounds, "") !=0 ) { - istringstream ob_str (outerbounds); - ob_str >> obss; - bool good = obss.size() == bbss.size(); - if (good) { - for (int rl=0; rl<(int)obss.size(); ++rl) { - good = good && obss[rl].size() == bbss[rl].size(); - } - } - if (! good) { - cout << "gridpoints: " << bbss << endl; - cout << "outerbounds: " << obss << endl; - CCTK_WARN (0, "The parameters \"outerbounds\" and \"gridpoints\" must have the same structure"); - } - } else { - obss.resize(bbss.size()); - for (int rl=0; rl<(int)obss.size(); ++rl) { - obss[rl].resize(bbss[rl].size()); - for (int c=0; c<(int)obss[rl].size(); ++c) { - obss[rl][c] = vect<vect<bool,2>,dim>(vect<bool,2>(false)); - } - } - } - - MakeRegions_AsSpecified (cctkGH, newnumlevels, bbss, obss, - bbl, obl); - + do_recompose = ManualGridpointList + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "manual-coordinate-list")) { - vector<vector<rbbox> > bbss; - if (strcmp(coordinates, "") != 0) { - istringstream co_str(coordinates); - co_str >> bbss; - } - - vector<vector<vect<vect<bool,2>,dim> > > obss; - if (strcmp(outerbounds, "") !=0 ) { - istringstream ob_str (outerbounds); - ob_str >> obss; - bool good = obss.size() == bbss.size(); - if (good) { - for (int rl=0; rl<(int)obss.size(); ++rl) { - good = good && obss[rl].size() == bbss[rl].size(); - } - } - if (! good) { - cout << "coordinates: " << bbss << endl; - cout << "outerbounds: " << obss << endl; - CCTK_WARN (0, "The parameters \"outerbounds\" and \"coordinates\" must have the same structure"); - } - } else { - obss.resize(bbss.size()); - for (int rl=0; rl<(int)obss.size(); ++rl) { - obss[rl].resize(bbss[rl].size()); - for (int c=0; c<(int)obss[rl].size(); ++c) { - obss[rl][c] = vect<vect<bool,2>,dim>(vect<bool,2>(false)); - } - } - } - - MakeRegions_AsSpecified (cctkGH, newnumlevels, bbss, obss, - bbl, obl); - + do_recompose = ManualCoordinateList + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else if (CCTK_EQUALS(refined_regions, "automatic")) { - const int vi = CCTK_VarIndex (errorvar); - assert (vi>=0 && vi<CCTK_NumVars()); - const int gi = CCTK_GroupIndexFromVarI (vi); - assert (gi>=0 && gi<CCTK_NumGroups()); - const int v1 = CCTK_FirstVarIndexI(gi); - assert (v1>=0 && v1<=vi && v1<CCTK_NumVars()); - - assert (CCTK_GroupTypeI(gi) == CCTK_GF); - assert (CCTK_VarTypeI(vi) == CCTK_VARIABLE_REAL); - assert (CCTK_GroupDimI(gi) == dim); - - assert (gi < (int)arrdata.size()); - assert (vi-v1 < (int)arrdata[gi].data.size()); - assert (arrdata[gi].data[vi-v1]); - const gf<CCTK_REAL,dim>& error - = *dynamic_cast<const gf<CCTK_REAL,dim>*>(arrdata[gi].data[vi-v1]); - - MakeRegions_Adaptively (cctkGH, minwidth, minfraction, maxerror, error, - bbl, obl); - + do_recompose = Automatic + (cctkGH, hh, reflevel, map, + size, nboundaryzones, is_internal, is_staggered, shiftout, + bbsss, obss, pss); + } else { assert (0); } -#ifdef AUTOMATIC_BOUNDARIES - assert (obl.size() == 0); -#else - assert (bbl.size() == obl.size()); -#endif - - // transform bbox list into bbox vector - vector<ibbox> bbs; - bbs.reserve (bbl.size()); - for (list<ibbox>::const_iterator it = bbl.begin(); - it != bbl.end(); - ++it) { - bbs.push_back (*it); - } -#ifdef AUTOMATIC_BOUNDARIES - vector<bvect> obs; - obs.resize (bbs.size()); - for (int c=0; c<(int)bbs.size(); ++c) { - ivect extlo = hh->baseextent.lower(); - ivect extup = hh->baseextent.upper(); - ivect str = bbs[c].stride(); - for (int d=0; d<dim; ++d) { - // Decide what is an outer boundary. - // Allow it to be one grid point to the interior. - assert (bbs[c].lower()[d] >= extlo[d]); - obs[c][d][0] = bbs[c].lower()[d] <= extlo[d] + str[d]; - assert (bbs[c].upper()[d] <= extup[d]); - obs[c][d][1] = bbs[c].upper()[d] >= extup[d] - str[d]; - } - } -#else - vector<bvect> obs; - obs.reserve (obl.size()); - for (list<bvect>::const_iterator it = obl.begin(); - it != obl.end(); - ++it) { - obs.push_back (*it); - } -#endif - - - - // TODO: ensure nesting properties - - // make multiprocessor aware - SplitRegions (cctkGH, bbs, obs); - - // make multigrid aware - gh<dim>::cexts bbss; - bbss = hh->make_reflevel_multigrid_boxes(bbs, mglevels); - - // insert into grid hierarchy - bbsss = hh->extents; - obss = hh->outer_boundaries; - if (bbss.size() == 0) { - // remove all finer levels - // TODO: this might not work - bbsss.resize(reflevel+1); - obss.resize(reflevel+1); - } else { - assert (reflevel < (int)bbsss.size()); - if (reflevel+1 == (int)bbsss.size()) { - // add a finer level - bbsss.push_back (bbss); - obss.push_back (obs); - } else { - // change a finer level - bbsss[reflevel+1] = bbss; - obss[reflevel+1] = obs; - } - } - - MakeProcessors (cctkGH, bbsss, pss); - - return 1; // do recompose - } - - - - void MakeRegions_BaseLevel (const cGH* cctkGH, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (bbl.empty()); - assert (obl.empty()); - } - - - - // This is a helpful helper routine. The user can use it to define - // how the hierarchy should be refined. But the result of this - // routine is rather arbitrary. - void MakeRegions_RefineCentre (const cGH* cctkGH, const int reflevels, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (bbl.empty()); - assert (obl.empty()); - - if (reflevel+1 >= reflevels) return; - - ivect rstr = hh->baseextent.stride(); - ivect rlb = hh->baseextent.lower(); - ivect rub = hh->baseextent.upper(); - - for (int rl=0; rl<reflevel+1; ++rl) { - // save old values - const ivect oldrlb = rlb; - const ivect oldrub = rub; - // refined boxes have smaller stride - assert (all(rstr%hh->reffact == 0)); - rstr /= hh->reffact; - // calculate new extent - const ivect quarter = (rub - rlb) / 4 / rstr * rstr; - rlb = oldrlb + quarter; - rub = oldrub - quarter; - assert (all(rlb >= oldrlb && rub <= oldrub)); - } - - bbl.push_back (ibbox(rlb, rub, rstr)); - obl.push_back (bvect(vect<bool,2>(false))); - } - - - - static void - MakeRegions_AsSpecified_OneLevel (const cGH* cctkGH, const int reflevels, - const ivect ilower, - const ivect iupper, - const bvect obound, - list<ibbox>& bbl, list<bvect>& obl) - { - if (reflevel+1 >= reflevels) return; - - const ivect rstr = hh->baseextent.stride(); - const ivect rlb = hh->baseextent.lower(); - const ivect rub = hh->baseextent.upper(); - - const int rl = reflevel+1; - - const int levfac = ipow(hh->reffact, rl); - assert (all (rstr % levfac == 0)); - const ivect str (rstr / levfac); - const ivect lb (ilower); - const ivect ub (iupper); - if (! all(lb>=rlb && ub<=rub)) { - ostringstream buf; - buf << "The refinement region boundaries for refinement level #" << rl << " are not within the main grid. Allowed are the grid point boundaries " << rlb << " - " << rub << "; specified were " << lb << " - " << ub << ends; - CCTK_WARN (0, buf.str().c_str()); - } - if (! all(lb<=ub)) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "The refinement region boundaries for refinement level #%d have the upper boundary less than the lower boundary", rl); - } - if (! all(lb%str==0 && ub%str==0)) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "The refinement region boundaries for refinement level #%d are not a multiple of the stride for that level", rl); - } - assert (all(lb>=rlb && ub<=rub)); - assert (all(lb<=ub)); - assert (all(lb%str==0 && ub%str==0)); - - bbl.push_back (ibbox(lb, ub, str)); - obl.push_back (obound); - } - - - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<ivect> lower, - const vector<ivect> upper, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (lower.size() == upper.size()); - - if (reflevel+1 >= reflevels) return; - - const int rl = reflevel+1; - - const ivect ilower = lower[rl-1]; - const ivect iupper = upper[rl-1]; - const bvect obound = bvect(vect<bool,2>(false)); - - MakeRegions_AsSpecified_OneLevel (cctkGH, reflevels, - ilower, iupper, obound, - bbl, obl); - } - - - - static ivect delta2int (const cGH* cctkGH, const rvect & rpos, const int rl) - { - rvect global_lower, global_upper; - for (int d=0; d<dim; ++d) { - const int ierr = CCTK_CoordRange - (cctkGH, &global_lower[d], &global_upper[d], d+1, 0, "cart3d"); - if (ierr<0) { - global_lower[d] = 0; - global_upper[d] = 1; - } - } - const ivect global_extent (hh->baseextent.upper() - hh->baseextent.lower()); - - CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; - - const rvect scale = rvect(global_extent) / (global_upper - global_lower); - const int levfac = ipow(hh->reffact, rl); - assert (all (hh->baseextent.stride() % levfac == 0)); - const ivect istride = hh->baseextent.stride() / levfac; - - const ivect ipos = ivect(map(rfloor, rpos * scale / rvect(istride) + 0.5)) * istride; - - const rvect apos = rpos * scale; - assert (all(abs(apos - rvect(ipos)) < rvect(istride)*0.01)); - - return ipos; - } - - - - static ivect pos2int (const cGH* cctkGH, const rvect & rpos, const int rl) - { - rvect global_lower, global_upper; - for (int d=0; d<dim; ++d) { - const int ierr = CCTK_CoordRange - (cctkGH, &global_lower[d], &global_upper[d], d+1, 0, "cart3d"); - if (ierr<0) { - global_lower[d] = 0; - global_upper[d] = 1; - } - } - const ivect global_extent (hh->baseextent.upper() - hh->baseextent.lower()); - - CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; - - const rvect scale = rvect(global_extent) / (global_upper - global_lower); - const int levfac = ipow(hh->reffact, rl); - assert (all (hh->baseextent.stride() % levfac == 0)); - const ivect istride = hh->baseextent.stride() / levfac; - - const ivect ipos = ivect(map(rfloor, (rpos - global_lower) * scale / rvect(istride) + 0.5)) * istride; - - return ipos; - } - - - -#if 0 - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<rvect> lower, - const vector<rvect> upper, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (lower.size() == upper.size()); - - if (reflevel+1 >= reflevels) return; - - rvect global_lower, global_upper; - for (int d=0; d<dim; ++d) { - const int ierr = CCTK_CoordRange - (cctkGH, &global_lower[d], &global_upper[d], d+1, 0, "cart3d"); - if (ierr<0) { - global_lower[d] = 0; - global_upper[d] = 1; - } - } - const ivect global_extent (hh->baseextent.upper() - hh->baseextent.lower()); - - const int rl = reflevel+1; - - const rvect scale = rvect(global_extent) / (global_upper - global_lower); - const rvect rlower = (lower[rl-1] - global_lower) * scale; - const rvect rupper = (upper[rl-1] - global_lower) * scale; -// const ivect ilower = ivect(map((CCTK_REAL(*)(CCTK_REAL))floor, rlower + 0.5)); -// const ivect iupper = ivect(map((CCTK_REAL(*)(CCTK_REAL))floor, rupper + 0.5)); - CCTK_REAL (* const rfloor) (CCTK_REAL const) = floor; - const int levfac = ipow(hh->reffact, rl); - assert (all (hh->baseextent.stride() % levfac == 0)); - const ivect istride = hh->baseextent.stride() / levfac; - const ivect ilower = ivect(map(rfloor, rlower / rvect(istride) + 0.5)) * istride; - const ivect iupper = ivect(map(rfloor, rupper / rvect(istride) + 0.5)) * istride; - const bvect obound = bvect(vect<bool,2>(false)); - - MakeRegions_AsSpecified_OneLevel (cctkGH, reflevels, - ilower, iupper, obound, - bbl, obl); - } -#endif - - - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<rvect> lower, - const vector<rvect> upper, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (lower.size() == upper.size()); - vector<ivect> ilower, iupper; - ilower.resize (lower.size()); - iupper.resize (upper.size()); - for (int rl=1; rl<lower.size()+1; ++rl) { - ilower[rl-1] = pos2int (cctkGH, lower[rl-1], rl); - iupper[rl-1] = pos2int (cctkGH, upper[rl-1], rl); - } - MakeRegions_AsSpecified (cctkGH, reflevels, ilower, iupper, bbl, obl); - } - - - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<vector<ibbox> > bbss, - const vector<vector<bvect> > obss, - list<ibbox>& bbl, list<bvect>& obl) - { - if (reflevel+1 >= reflevels) return; - - const int rl = reflevel+1; - assert (rl-1 < (int)bbss.size()); - assert (obss.size() == bbss.size()); - - for (int c=0; c<(int)bbss[rl-1].size(); ++c) { - assert (obss[rl-1].size() == bbss[rl-1].size()); - - const ivect ilower = bbss[rl-1][c].lower(); - const ivect iupper = bbss[rl-1][c].upper(); - const bvect obound = obss[rl-1][c]; - - MakeRegions_AsSpecified_OneLevel (cctkGH, reflevels, - ilower, iupper, obound, - bbl, obl); - - } - } - - - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<vector<rbbox> > bbss, - const vector<vector<bvect> > obss, - list<ibbox>& bbl, list<bvect>& obl) - { - vector<vector<ibbox> > ibbss; - ibbss.resize (bbss.size()); - for (int rl=1; rl<bbss.size()+1; ++rl) { - ibbss[rl-1].resize (bbss[rl-1].size()); - for (int c=0; c<bbss[rl-1].size(); ++c) { - const ivect ilower = pos2int (cctkGH, bbss[rl-1][c].lower(), rl); - const ivect iupper = pos2int (cctkGH, bbss[rl-1][c].upper(), rl); - const ivect istride = delta2int (cctkGH, bbss[rl-1][c].stride(), rl); - ibbss[rl-1][c] = ibbox (ilower, iupper, istride); - } - } - MakeRegions_AsSpecified (cctkGH, reflevels, ibbss, obss, bbl, obl); - } - - - - static void - MakeRegions_Adaptively_Recombine (list<ibbox>& bbl1, - list<ibbox>& bbl2, - list<ibbox>& bbl, - const ibbox& iface, - const int dir) - { - assert (!iface.empty()); - assert (iface.lower()[dir] == iface.upper()[dir]); - - const int oldnumboxes = bbl.size() + bbl1.size() + bbl2.size(); - int numcombinedboxes = 0; - - int oldnumpoints = 0; - for (list<ibbox>::const_iterator ibb = bbl.begin(); ibb != bbl.end(); ++ibb) { - oldnumpoints += ibb->size(); - } - for (list<ibbox>::const_iterator ibb1 = bbl1.begin(); ibb1 != bbl1.end(); ++ibb1) { - oldnumpoints += ibb1->size(); - } - for (list<ibbox>::const_iterator ibb2 = bbl2.begin(); ibb2 != bbl2.end(); ++ibb2) { - oldnumpoints += ibb2->size(); - } - -#if 0 - // remember old bounding boxes - bboxset<int,dim> oldboxes; - for (list<ibbox>::const_iterator ibb1 = bbl1.begin(); ibb1 != bbl1.end(); ++ibb1) { - oldboxes += *ibb1; - } - for (list<ibbox>::const_iterator ibb2 = bbl2.begin(); ibb2 != bbl2.end(); ++ibb2) { - oldboxes += *ibb2; - } -#endif -#if 0 - cout << endl; - cout << "MakeRegions_Adaptively_Recombine: initial list:" << endl; - cout << bbl << endl; - cout << "MakeRegions_Adaptively_Recombine: initial list 1:" << endl; - cout << bbl1 << endl; - cout << "MakeRegions_Adaptively_Recombine: initial list 2:" << endl; - cout << bbl2 << endl; -#endif - - const ivect lo = iface.lower(); - const ivect up = iface.upper(); - const ivect str = iface.stride(); - - { - // prune boxes on the left - list<ibbox>::iterator ibb1 = bbl1.begin(); - while (ibb1 != bbl1.end()) { - // is this bbox just to the left of the interface? -// const ivect lo1 = ibb1->lower(); - const ivect up1 = ibb1->upper(); - const ivect str1 = ibb1->stride(); - assert (up1[dir]+str1[dir] <= lo[dir]); - assert (all(str1 == str)); - if (up1[dir]+str1[dir] < lo[dir]) { - // no: forget it - bbl.push_back (*ibb1); - ibb1 = bbl1.erase(ibb1); - continue; - } - ++ibb1; - } // while - } - - { - // prune boxes on the right - list<ibbox>::iterator ibb2 = bbl2.begin(); - while (ibb2 != bbl2.end()) { - // is this bbox just to the right of the interface? - const ivect lo2 = ibb2->lower(); -// const ivect up2 = ibb2->upper(); - const ivect str2 = ibb2->stride(); - assert (up[dir] <= lo2[dir]); - assert (all(str2 == str)); - if (up[dir] < lo2[dir]) { - // no: forget it - bbl.push_back (*ibb2); - ibb2 = bbl2.erase(ibb2); - continue; - } - ++ibb2; - } // while - } - - { - // walk all boxes on the left - list<ibbox>::iterator ibb1 = bbl1.begin(); - while (ibb1 != bbl1.end()) { - ivect lo1 = ibb1->lower(); - ivect up1 = ibb1->upper(); - ivect str1 = ibb1->stride(); - assert (up1[dir]+str1[dir] == lo[dir]); - lo1[dir] = lo[dir]; - up1[dir] = up[dir]; - const ibbox iface1 (lo1,up1,str1); - - { - // walk all boxes on the right - list<ibbox>::iterator ibb2 = bbl2.begin(); - while (ibb2 != bbl2.end()) { - ivect lo2 = ibb2->lower(); - ivect up2 = ibb2->upper(); - ivect str2 = ibb2->stride(); - assert (lo2[dir] == up[dir]); - lo2[dir] = lo[dir]; - up2[dir] = up[dir]; - const ibbox iface2 (lo2,up2,str2); - - // check for a match - if (iface1 == iface2) { - const ibbox combined (ibb1->lower(), ibb2->upper(), str); - bbl.push_back (combined); - ibb1 = bbl1.erase(ibb1); - ibb2 = bbl2.erase(ibb2); - ++numcombinedboxes; -// cout << "MRA: Combining along " << "xyz"[dir] << " to " << bbl.back() << " size " << bbl.back().size() << endl; - goto continue_search; - } - - ++ibb2; - } // while - } - - ++ibb1; - continue_search:; - } // while - } - - bbl.splice (bbl.end(), bbl1); - bbl.splice (bbl.end(), bbl2); - - assert (bbl1.empty() && bbl2.empty()); - - const int newnumboxes = bbl.size(); - assert (newnumboxes + numcombinedboxes == oldnumboxes); - - int newnumpoints = 0; - for (list<ibbox>::const_iterator ibb = bbl.begin(); ibb != bbl.end(); ++ibb) { - newnumpoints += ibb->size(); - } - assert (newnumpoints == oldnumpoints); - -#if 0 - // find new bounding boxes - bboxset<int,dim> newboxes; - for (list<ibbox>::const_iterator ibb = bbl.begin(); ibb != bbl.end(); ++ibb) { - newboxes += *ibb; - } - - // Check that they are equal - assert (newboxes.size() <= oldboxes.size()); - assert ((newboxes.size()==0) == (oldboxes.size()==0)); - assert (oldboxes == newboxes); -#endif -#if 0 - cout << "MakeRegions_Adaptively_Recombine: final list:" << endl; - cout << bbl << endl; - cout << endl; -#endif - } - - static void - MakeRegions_Adaptively (const cGH* const cctkGH, - const int minwidth, - const CCTK_REAL minfraction, - const CCTK_REAL maxerror, - const data<CCTK_REAL,dim>& error, - list<ibbox>& bbl, - const ibbox& region) - { - // Just to be sure - assert (! region.empty()); - - // Count grid points that need to be refined - // (this doesn't work yet on multiple processors) - assert (CCTK_nProcs(cctkGH)==1); - int cnt = 0; - for (ibbox::iterator it=region.begin(); it!=region.end(); ++it) { - if (error[*it] > maxerror) ++cnt; - } - const CCTK_REAL fraction = (CCTK_REAL)cnt / region.size(); - const int width = maxval(region.shape() / region.stride()); - - if (cnt == 0) { - // Don't refine - } else if (width < 2*minwidth || fraction >= minfraction) { - // Refine the whole region - const ivect lo(region.lower()); - const ivect up(region.upper()); - const ivect str(region.stride()); - bbl.push_back (ibbox(lo,up+str-str/reffact,str/reffact)); -// cout << "MRA: Refining to " << bbl.back() << " size " << bbl.back().size() << " fraction " << fraction << endl; - } else { - // Split the region and check recursively - const int dir = maxloc(region.shape()); - const ivect lo(region.lower()); - const ivect up(region.upper()); - const ivect str(region.stride()); - ivect lo1(lo), lo2(lo); - ivect up1(up), up2(up); - const int mgstr = ipow(hh->mgfact, mglevels); // honour multigrid factors - const int step = str[dir]*mgstr; - lo2[dir] = ((up[dir]+lo[dir])/2/step)*step; - up1[dir] = lo2[dir]-str[dir]; - const ibbox region1(lo1,up1,str); - const ibbox region2(lo2,up2,str); - assert (region1.is_contained_in(region)); - assert (region2.is_contained_in(region)); - assert ((region1 & region2).empty()); - assert (region1 + region2 == region); - list<ibbox> bbl1, bbl2; - MakeRegions_Adaptively (cctkGH, minwidth, minfraction, maxerror, - error, bbl1, region1); - MakeRegions_Adaptively (cctkGH, minwidth, minfraction, maxerror, - error, bbl2, region2); - // Combine regions if possible - up2 += str-str/reffact; - up2[dir] = lo2[dir]; - const ibbox iface(lo2,up2,str/reffact); - MakeRegions_Adaptively_Recombine (bbl1, bbl2, bbl, iface, dir); - } - - } - - void - MakeRegions_Adaptively (const cGH* const cctkGH, - const int minwidth, - const CCTK_REAL minfraction, - const CCTK_REAL maxerror, - const gf<CCTK_REAL,dim>& error, - list<ibbox>& bbl, list<bvect>& obl) - { - assert (bbl.empty()); - - if (reflevel+1 >= maxreflevels) return; - - assert (component == -1); - - const int rl = reflevel; - - // Arbitrary - const int tl = 0; - const int ml = 0; - -// cout << endl << "MRA: Choosing regions to refine in " << hh->components(rl) << " components" << endl; - - for (int c=0; c<hh->components(rl); ++c) { - const ibbox region = hh->extents[rl][c][ml]; - assert (! region.empty()); - - const data<CCTK_REAL,dim>& errdata = *error(tl,rl,c,ml); - - MakeRegions_Adaptively (cctkGH, minwidth, minfraction, maxerror, - errdata, bbl, region); - } - -// int numpoints = 0; -// for (list<ibbox>::const_iterator ibb = bbl.begin(); ibb != bbl.end(); ++ibb) { -// numpoints += ibb->size(); -// } -// cout << "MRA: Chose " << bbl.size() << " regions with a total size of " << numpoints << " to refine." << endl << endl; - - // Remove grid points outside the outer boundary - vect<bool,dim> obp; - { - DECLARE_CCTK_PARAMETERS; - assert (sizeof outside_boundary_points == dim * sizeof (CCTK_INT)); - obp = outside_boundary_points; - } - for (list<ibbox>::iterator it = bbl.begin(); - it != bbl.end(); - ++it) { - const ivect ub = obp.ifthen (it->upper(), - min (it->upper(), hh->baseextent.upper())); - *it = ibbox(it->lower(), ub, it->stride()); - } - - // Create obl from bbl - for (list<ibbox>::const_iterator it = bbl.begin(); - it != bbl.end(); - ++it) { - obl.push_back (zip ((vect<bool,2> (*) (bool, bool)) &vect<bool,2>::make, - it->lower() == hh->baseextent.lower(), - it->upper() == hh->baseextent.upper())); - } - + return do_recompose; } } // namespace CarpetRegrid diff --git a/Carpet/CarpetRegrid/src/regrid.hh b/Carpet/CarpetRegrid/src/regrid.hh index 45a390a97..27edc99c9 100644 --- a/Carpet/CarpetRegrid/src/regrid.hh +++ b/Carpet/CarpetRegrid/src/regrid.hh @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/regrid.hh,v 1.10 2003/11/13 16:04:37 schnetter Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetRegrid/src/regrid.hh,v 1.11 2004/01/25 14:57:30 schnetter Exp $ #ifndef CARPETREGRID_HH #define CARPETREGRID_HH @@ -6,14 +6,15 @@ #include <list> #include "cctk.h" +#include "cctk_Arguments.h" +#include "bbox.hh" #include "gf.hh" #include "gh.hh" +#include "vect.hh" #include "carpet.hh" -#include "regrid.h" - namespace CarpetRegrid { @@ -23,53 +24,171 @@ namespace CarpetRegrid { - typedef vect<int,dim> ivect; - typedef bbox<int,dim> ibbox; - - typedef vect<vect<bool,2>,dim> bvect; - - typedef vect<CCTK_REAL,dim> rvect; - typedef bbox<CCTK_REAL,dim> rbbox; - - - - int CarpetRegridRegrid (const cGH * const cctkGH, - gh<dim>::rexts& bbsss, - gh<dim>::rbnds& obss, - gh<dim>::rprocs& pss); - - - - void MakeRegions_BaseLevel (const cGH* cctkGH, - list<ibbox>& bbl, list<bvect>& obl); - - void MakeRegions_RefineCentre (const cGH* cctkGH, const int reflevels, - list<ibbox>& bbl, list<bvect>& obl); - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<ivect> lower, - const vector<ivect> upper, - list<ibbox>& bbl, list<bvect>& obl); - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<rvect> lower, - const vector<rvect> upper, - list<ibbox>& bbl, list<bvect>& obl); - - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<vector<ibbox> > bbss, - const vector<vector<bvect> > obss, - list<ibbox>& bbl, list<bvect>& obl); - void MakeRegions_AsSpecified (const cGH* cctkGH, const int reflevels, - const vector<vector<rbbox> > bbss, - const vector<vector<bvect> > obss, - list<ibbox>& bbl, list<bvect>& obl); - - void MakeRegions_Adaptively (const cGH* cctkGH, - const int minwidth, - const CCTK_REAL minfraction, - const CCTK_REAL maxerror, - const gf<CCTK_REAL,dim>& error, - list<ibbox>& bbl, list<bvect>& obl); + extern "C" { + + /* Scheduled functions */ + int CarpetRegridParamcheck (CCTK_ARGUMENTS); + + /* Aliased functions */ +// CCTK_INT CarpetRegrid_Regrid (const cGH * const cctkGH, +// gh<dim>::rexts * bbsss, +// gh<dim>::rbnds * obss, +// gh<dim>::rprocs * pss); + CCTK_INT CarpetRegrid_Regrid (CCTK_POINTER_TO_CONST const cctkGH_, + CCTK_INT const reflevel, + CCTK_INT const map, + CCTK_INT const size, + CCTK_INT const * const nboundaryzones, + CCTK_INT const * const is_internal, + CCTK_INT const * const is_staggered, + CCTK_INT const * const shiftout, + CCTK_POINTER const bbsss_, + CCTK_POINTER const obss_, + CCTK_POINTER const pss_); + } + + + + int BaseLevel (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + int Centre (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + int ManualGridpoints (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + void ManualGridpoints_OneLevel (const cGH * const cctkGH, + const gh<dim> & hh, + const int reflevel, + const int reflevels, + const ivect ilower, + const ivect iupper, + const bbvect obound, + vector<ibbox> & bbs, + vector<bbvect> & obs); + + int ManualCoordinates (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + void ManualCoordinates_OneLevel (const cGH * const cctkGH, + const gh<dim> & hh, + const int reflevel, + const int reflevels, + const rvect ilower, + const rvect iupper, + const bbvect obound, + vector<ibbox> & bbs, + vector<bbvect> & obs); + + ivect delta2int (const cGH * const cctkGH, const gh<dim>& hh, + const rvect & rpos, const int reflevel); + ivect pos2int (const cGH* const cctkGH, const gh<dim>& hh, + const rvect & rpos, const int reflevel); + + int ManualGridpointList (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + int ManualCoordinateList (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + int Automatic (cGH const * const cctkGH, + gh<dim> const & hh, + int const reflevel, + int const map, + int const size, + jjvect const & nboundaryzones, + jjvect const & is_internal, + jjvect const & is_staggered, + jjvect const & shiftout, + gh<dim>::rexts & bbsss, + gh<dim>::rbnds & obss, + gh<dim>::rprocs & pss); + + void Automatic_OneLevel (const cGH * const cctkGH, + const gh<dim> & hh, + const int reflevel, + const int minwidth, + const CCTK_REAL minfraction, + const CCTK_REAL maxerror, + const gf<CCTK_REAL,dim> & errorvar, + vector<ibbox> & bbs, + vector<bbvect> & obs); + + void Automatic_Recursive (const cGH * const cctkGH, + const gh<dim> & hh, + const int minwidth, + const CCTK_REAL minfraction, + const CCTK_REAL maxerror, + const data<CCTK_REAL,dim> & errorvar, + list<ibbox> & bbl, + const ibbox & region); + + void Automatic_Recombine (list<ibbox> & bbl1, + list<ibbox> & bbl2, + list<ibbox> & bbl, + const ibbox & iface, + const int dir); } // namespace CarpetRegrid diff --git a/Carpet/CarpetSlab/src/slab.cc b/Carpet/CarpetSlab/src/slab.cc index d15f9d48d..deff045d7 100644 --- a/Carpet/CarpetSlab/src/slab.cc +++ b/Carpet/CarpetSlab/src/slab.cc @@ -1,4 +1,4 @@ -// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetSlab/src/slab.cc,v 1.14 2004/01/22 13:31:07 tradke Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetSlab/src/slab.cc,v 1.15 2004/01/25 14:57:31 schnetter Exp $ #include <assert.h> #include <stdlib.h> @@ -23,7 +23,7 @@ #include "slab.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetSlab/src/slab.cc,v 1.14 2004/01/22 13:31:07 tradke Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetSlab/src/slab.cc,v 1.15 2004/01/25 14:57:31 schnetter Exp $"; CCTK_FILEVERSION(Carpet_CarpetSlab_slab_cc); } @@ -110,10 +110,15 @@ namespace CarpetSlab { assert (typesize>0); if (gp.grouptype==CCTK_GF && reflevel==-1) { - CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in global mode"); + CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in global mode (use singlemap mode instead)"); } const int rl = gp.grouptype==CCTK_GF ? reflevel : 0; + if (gp.grouptype==CCTK_GF && Carpet::map==-1) { + CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in level mode (use singlemap mode instead)"); + } + const int m = gp.grouptype==CCTK_GF ? Carpet::map : 0; + // Check dimension assert (hdim>=0 && hdim<=gp.dim); @@ -152,12 +157,12 @@ namespace CarpetSlab { const dh<dim>* mydd; const ggf<dim>* myff; assert (group < (int)arrdata.size()); - myhh = arrdata[group].hh; + myhh = arrdata[group][m].hh; assert (myhh); - mydd = arrdata[group].dd; + mydd = arrdata[group][m].dd; assert (mydd); - assert (var < (int)arrdata[group].data.size()); - myff = arrdata[group].data[var]; + assert (var < (int)arrdata[group][m].data.size()); + myff = arrdata[group][m].data[var]; assert (myff); // Detemine collecting processor @@ -313,10 +318,15 @@ namespace CarpetSlab { assert (typesize>0); if (gp.grouptype==CCTK_GF && reflevel==-1) { - CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in global mode"); + CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in global mode (use singlemap mode instead)"); } const int rl = gp.grouptype==CCTK_GF ? reflevel : 0; + if (gp.grouptype==CCTK_GF && Carpet::map==-1) { + CCTK_WARN (0, "It is not possible to use hyperslabbing for a grid function in level mode (use singlemap mode instead)"); + } + const int m = gp.grouptype==CCTK_GF ? Carpet::map : 0; + // Check dimension assert (hdim>=0 && hdim<=gp.dim); @@ -355,12 +365,12 @@ namespace CarpetSlab { const dh<dim>* mydd; const ggf<dim>* myff; assert (group < (int)arrdata.size()); - myhh = arrdata[group].hh; + myhh = arrdata[group][m].hh; assert (myhh); - mydd = arrdata[group].dd; + mydd = arrdata[group][m].dd; assert (mydd); - assert (var < (int)arrdata[group].data.size()); - myff = arrdata[group].data[var]; + assert (var < (int)arrdata[group][m].data.size()); + myff = arrdata[group][m].data[var]; assert (myff); // Detemine collecting processor @@ -486,7 +496,7 @@ namespace CarpetSlab { - CCTK_INT CarpetSlab_Get (cGH const * const cctkGH, + CCTK_INT CarpetSlab_Get (CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const mapping_handle, CCTK_INT const proc, CCTK_INT const vindex, @@ -494,6 +504,8 @@ namespace CarpetSlab { CCTK_INT const hdatatype, void * const hdata) { + cGH const * const cctkGH = (cGH const *) cctkGH_; + // Check arguments assert (cctkGH); assert (mapping_handle>=0); @@ -526,7 +538,7 @@ namespace CarpetSlab { - CCTK_INT CarpetSlab_GetList (cGH const * const cctkGH, + CCTK_INT CarpetSlab_GetList (CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const mapping_handle, CCTK_INT const num_arrays, CCTK_INT const * const procs, @@ -536,6 +548,8 @@ namespace CarpetSlab { void * const * const hdata, CCTK_INT * const retvals) { + cGH const * const cctkGH = (cGH const *) cctkGH_; + // Check arguments assert (cctkGH); assert (mapping_handle>=0); @@ -564,7 +578,7 @@ namespace CarpetSlab { - CCTK_INT CarpetSlab_LocalMappingByIndex (cGH const * const cctkGH, + CCTK_INT CarpetSlab_LocalMappingByIndex (CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const vindex, CCTK_INT const hdim, CCTK_INT const * const direction, @@ -589,7 +603,7 @@ namespace CarpetSlab { - CCTK_INT CarpetSlab_GlobalMappingByIndex (cGH const * const cctkGH, + CCTK_INT CarpetSlab_GlobalMappingByIndex (CCTK_POINTER_TO_CONST const cctkGH_, CCTK_INT const vindex, CCTK_INT const hdim, CCTK_INT const * const direction, @@ -606,6 +620,8 @@ namespace CarpetSlab { void * const to), CCTK_INT * const hsize) { + cGH const * const cctkGH = (cGH const *) cctkGH_; + // Check arguments assert (cctkGH); assert (vindex>=0 && vindex<CCTK_NumVars()); @@ -751,7 +767,7 @@ namespace CarpetSlab { - int Hyperslab_GetHyperslab (cGH* const GH, + int Hyperslab_GetHyperslab (const cGH* const GH, const int target_proc, const int vindex, const int vtimelvl, diff --git a/CarpetAttic/CarpetIOFlexIO/param.ccl b/CarpetAttic/CarpetIOFlexIO/param.ccl index f377271cb..1061d142e 100644 --- a/CarpetAttic/CarpetIOFlexIO/param.ccl +++ b/CarpetAttic/CarpetIOFlexIO/param.ccl @@ -1,5 +1,5 @@ # Parameter definitions for thorn CarpetIOFlexIO -# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/param.ccl,v 1.7 2003/09/02 12:37:51 tradke Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/param.ccl,v 1.8 2004/01/25 14:57:29 schnetter Exp $ @@ -50,19 +50,19 @@ CCTK_INT out3D_every "How often to do 3D FlexIO output, overrides out_every" STE -CCTK_INT out3D_max_num_lower_ghosts "Maximum number of lower ghost zones that are output" +CCTK_INT out3D_max_num_lower_ghosts "Maximum number of lower ghost zones that are output" STEERABLE = ALWAYS { -1 :: "output all ghost zones" 0:* :: "output that many ghost zones" } -1 -CCTK_INT out3D_max_num_upper_ghosts "Maximum number of upper ghost zones that are output" +CCTK_INT out3D_max_num_upper_ghosts "Maximum number of upper ghost zones that are output" STEERABLE = ALWAYS { -1 :: "output all ghost zones" 0:* :: "output that many ghost zones" } -1 -BOOLEAN out3D_output_outer_boundary "Output all of the outer boundary ghost zones even if not all ghost zones are output" +BOOLEAN out3D_output_outer_boundary "Output all of the outer boundary ghost zones even if not all ghost zones are output" STEERABLE = ALWAYS { } "yes" diff --git a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc index cf9c2462f..cb40406b1 100644 --- a/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc +++ b/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc @@ -11,11 +11,14 @@ #include <sstream> #include <vector> - - #include "cctk.h" #include "cctk_Parameters.h" +extern "C" { + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc,v 1.39 2004/01/25 14:57:29 schnetter Exp $"; + CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_ioflexio_cc); +} + #include "AMRwriter.hh" #include "AmrGridReader.hh" #ifdef HDF4 @@ -43,11 +46,6 @@ #include "ioflexio.hh" -extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIO/src/ioflexio.cc,v 1.37 2003/12/10 18:13:16 schnetter Exp $"; - CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_ioflexio_cc); -} - namespace CarpetIOFlexIO { @@ -164,7 +162,7 @@ namespace CarpetIOFlexIO { assert (iogh); // Create the output directory - const char* myoutdir = GetStringParameter("out3D_dir", out_dir); + const char* const myoutdir = GetStringParameter("out3D_dir", out_dir); if (CCTK_MyProc(cgh)==0) { CCTK_CreateDirectory (0755, myoutdir); } @@ -265,9 +263,9 @@ namespace CarpetIOFlexIO { int interlevel_timerefinement; int interlevel_spacerefinement[dim]; int initial_gridplacementrefinement[dim]; - interlevel_timerefinement = hh->reffact; + interlevel_timerefinement = vhh[0]->reffact; for (int d=0; d<dim; ++d) { - interlevel_spacerefinement[d] = hh->reffact; + interlevel_spacerefinement[d] = vhh[0]->reffact; initial_gridplacementrefinement[d] = 1; } amrwriter->setRefinement @@ -282,75 +280,95 @@ namespace CarpetIOFlexIO { } // Traverse all components on this refinement and multigrid level - BEGIN_COMPONENT_LOOP(cgh, grouptype) { - - const ggf<dim>* ff = 0; - - assert (var < (int)arrdata[group].data.size()); - ff = (ggf<dim>*)arrdata[group].data[var]; - - const gdata<dim>* const data - = (*ff) (tl, rl, component, mglevel); - - // Make temporary copy on processor 0 - bbox<int,dim> ext = data->extent(); - vect<int,dim> lo = ext.lower(); - vect<int,dim> hi = ext.upper(); - vect<int,dim> str = ext.stride(); - - // Ignore ghost zones if desired - for (int d=0; d<dim; ++d) { - const int max_lower_ghosts = (cgh->cctk_bbox[2*d ] && !out3D_output_outer_boundary) ? -1 : out3D_max_num_lower_ghosts; - const int max_upper_ghosts = (cgh->cctk_bbox[2*d+1] && !out3D_output_outer_boundary) ? -1 : out3D_max_num_upper_ghosts; - - const int num_lower_ghosts = max_lower_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_lower_ghosts, cgh->cctk_nghostzones[d]); - const int num_upper_ghosts = max_upper_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_upper_ghosts, cgh->cctk_nghostzones[d]); - - lo[d] += (cgh->cctk_nghostzones[d] - num_lower_ghosts) * str[d]; - hi[d] -= (cgh->cctk_nghostzones[d] - num_upper_ghosts) * str[d]; - } - - ext = bbox<int,dim>(lo,hi,str); - - gdata<dim>* const tmp = data->make_typed (n); - tmp->allocate (ext, 0); - for (comm_state<dim> state; !state.done(); state.step()) { - tmp->copy_from (state, data, ext); - } - - // Write data - if (CCTK_MyProc(cgh)==0) { - int origin[dim], dims[dim]; - for (int d=0; d<dim; ++d) { - origin[d] = (ext.lower() / ext.stride())[d]; - dims[d] = (ext.shape() / ext.stride())[d]; - } - amrwriter->write (origin, dims, (void*)tmp->storage()); + BEGIN_MAP_LOOP(cgh, grouptype) { + BEGIN_COMPONENT_LOOP(cgh, grouptype) { - // Write some additional attributes - WriteAttribute (writer, "carpet_flexio_version", 1); - WriteAttribute (writer, "cctk_dim", cgh->cctk_dim); - WriteAttribute (writer, "cctk_iteration", cgh->cctk_iteration); - WriteAttribute (writer, "cctk_gsh", cgh->cctk_gsh, dim); - WriteAttribute (writer, "cctk_lsh", cgh->cctk_lsh, dim); - WriteAttribute (writer, "cctk_lbnd", cgh->cctk_lbnd, dim); - WriteAttribute (writer, "cctk_delta_time", cgh->cctk_delta_time); - WriteAttribute (writer, "cctk_delta_space", cgh->cctk_delta_space, dim); - WriteAttribute (writer, "cctk_origin_space", cgh->cctk_origin_space, dim); - WriteAttribute (writer, "cctk_bbox", cgh->cctk_bbox, 2*dim); - WriteAttribute (writer, "cctk_levfac", cgh->cctk_levfac, dim); - WriteAttribute (writer, "cctk_levoff", cgh->cctk_levoff, dim); - WriteAttribute (writer, "cctk_levoffdenom", cgh->cctk_levoffdenom, dim); - WriteAttribute (writer, "cctk_timefac", cgh->cctk_timefac); - WriteAttribute (writer, "cctk_convlevel", cgh->cctk_convlevel); - WriteAttribute (writer, "cctk_nghostzones", cgh->cctk_nghostzones, dim); - WriteAttribute (writer, "cctk_time", cgh->cctk_time); - } - - // Delete temporary copy - delete tmp; - - } END_COMPONENT_LOOP; + const ggf<dim>* ff = 0; + + assert (var < (int)arrdata[group][Carpet::map].data.size()); + ff = (ggf<dim>*)arrdata[group][Carpet::map].data[var]; + + const gdata<dim>* const data + = (*ff) (tl, rl, component, mglevel); + + // Make temporary copy on processor 0 + bbox<int,dim> ext = data->extent(); + vect<int,dim> lo = ext.lower(); + vect<int,dim> hi = ext.upper(); + vect<int,dim> str = ext.stride(); + + // Ignore ghost zones if desired + for (int d=0; d<dim; ++d) { + const int max_lower_ghosts = (cgh->cctk_bbox[2*d ] && out3D_output_outer_boundary) ? -1 : out3D_max_num_lower_ghosts; + const int max_upper_ghosts = (cgh->cctk_bbox[2*d+1] && out3D_output_outer_boundary) ? -1 : out3D_max_num_upper_ghosts; + + const int num_lower_ghosts = max_lower_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_lower_ghosts, cgh->cctk_nghostzones[d]); + const int num_upper_ghosts = max_upper_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_upper_ghosts, cgh->cctk_nghostzones[d]); + + lo[d] += (cgh->cctk_nghostzones[d] - num_lower_ghosts) * str[d]; + hi[d] -= (cgh->cctk_nghostzones[d] - num_upper_ghosts) * str[d]; + } + + ext = bbox<int,dim>(lo,hi,str); + + gdata<dim>* const tmp = data->make_typed (n); + tmp->allocate (ext, 0); + for (comm_state<dim> state; !state.done(); state.step()) { + tmp->copy_from (state, data, ext); + } + + // Write data + if (CCTK_MyProc(cgh)==0) { + int origin[dim], dims[dim]; + for (int d=0; d<dim; ++d) { + origin[d] = (ext.lower() / ext.stride())[d]; + dims[d] = (ext.shape() / ext.stride())[d]; + } + amrwriter->write (origin, dims, (void*)tmp->storage()); + + // Write some additional attributes + WriteAttribute (writer, "cctk_version", 1); + WriteAttribute (writer, "cctk_dim", cgh->cctk_dim); + WriteAttribute (writer, "cctk_iteration", cgh->cctk_iteration); +// TODO: disable temporarily +// WriteAttribute (writer, "cctk_nmaps", cgh->cctk_nmaps); +// WriteAttribute (writer, "cctk_map", cgh->cctk_map); + WriteAttribute (writer, "cctk_gsh", cgh->cctk_gsh, dim); + WriteAttribute (writer, "cctk_lsh", cgh->cctk_lsh, dim); + WriteAttribute (writer, "cctk_lbnd", cgh->cctk_lbnd, dim); + WriteAttribute (writer, "cctk_delta_time", cgh->cctk_delta_time); + WriteAttribute (writer, "cctk_delta_space", cgh->cctk_delta_space, dim); + WriteAttribute (writer, "cctk_origin_space", cgh->cctk_origin_space, dim); + WriteAttribute (writer, "cctk_bbox", cgh->cctk_bbox, 2*dim); + WriteAttribute (writer, "cctk_levfac", cgh->cctk_levfac, dim); + WriteAttribute (writer, "cctk_levoff", cgh->cctk_levoff, dim); + WriteAttribute (writer, "cctk_levoffdenom", cgh->cctk_levoffdenom, dim); + WriteAttribute (writer, "cctk_timefac", cgh->cctk_timefac); + WriteAttribute (writer, "cctk_convlevel", cgh->cctk_convlevel); +// TODO: disable temporarily +// WriteAttribute (writer, "cctk_convfac", cgh->cctk_convfac); + WriteAttribute (writer, "cctk_nghostzones", cgh->cctk_nghostzones, dim); + WriteAttribute (writer, "cctk_time", cgh->cctk_time); + WriteAttribute (writer, "carpet_version", 1); + WriteAttribute (writer, "carpet_dim", dim); + WriteAttribute (writer, "carpet_basemglevel", basemglevel); + WriteAttribute (writer, "carpet_mglevel", mglevel); + WriteAttribute (writer, "carpet_mglevels", mglevels); + WriteAttribute (writer, "carpet_mgface", mgfact); + WriteAttribute (writer, "carpet_reflevel", reflevel); + WriteAttribute (writer, "carpet_reflevels", reflevels); + WriteAttribute (writer, "carpet_reffact", reffact); + WriteAttribute (writer, "carpet_map", Carpet::map); + WriteAttribute (writer, "carpet_maps", maps); + WriteAttribute (writer, "carpet_component", component); + WriteAttribute (writer, "carpet_components", vhh[Carpet::map]->components(reflevel)); + } + + // Delete temporary copy + delete tmp; + + } END_COMPONENT_LOOP; + } END_MAP_LOOP; // Close the file if (CCTK_MyProc(cgh)==0) { @@ -606,38 +624,41 @@ namespace CarpetIOFlexIO { // Traverse all components on this refinement and multigrid // level - BEGIN_COMPONENT_LOOP(cgh, grouptype) { - - ggf<dim>* ff = 0; - - assert (var < (int)arrdata[group].data.size()); - ff = (ggf<dim>*)arrdata[group].data[var]; - - gdata<dim>* const data = (*ff) (tl, rl, component, mglevel); - - // Create temporary data storage on processor 0 - const vect<int,dim> str = vect<int,dim>(reflevelfact); - const vect<int,dim> lb = vect<int,dim>(amr_origin) * str; - const vect<int,dim> ub - = lb + (vect<int,dim>(amr_dims) - 1) * str; - const bbox<int,dim> ext(lb,ub,str); - gdata<dim>* const tmp = data->make_typed (n); - - if (CCTK_MyProc(cgh)==0) { - tmp->allocate (ext, 0, amrgrid->data); - } else { - tmp->allocate (ext, 0, 0); - } - - // Copy into grid function - for (comm_state<dim> state; !state.done(); state.step()) { - data->copy_from (state, tmp, ext); - } - - // Delete temporary copy - delete tmp; - - } END_COMPONENT_LOOP; + BEGIN_MAP_LOOP(cgh, grouptype) { + BEGIN_COMPONENT_LOOP(cgh, grouptype) { + + ggf<dim>* ff = 0; + + assert (var < (int)arrdata[group][Carpet::map].data.size()); + ff = (ggf<dim>*)arrdata[group][Carpet::map].data[var]; + + gdata<dim>* const data = (*ff) (tl, rl, component, mglevel); + + // Create temporary data storage on processor 0 + const vect<int,dim> str + = vect<int,dim>(maxreflevelfact/reflevelfact); + const vect<int,dim> lb = vect<int,dim>::ref(amr_origin) * str; + const vect<int,dim> ub + = lb + (vect<int,dim>::ref(amr_dims) - 1) * str; + const bbox<int,dim> ext(lb,ub,str); + gdata<dim>* const tmp = data->make_typed (n); + + if (CCTK_MyProc(cgh)==0) { + tmp->allocate (ext, 0, amrgrid->data); + } else { + tmp->allocate (ext, 0, 0); + } + + // Copy into grid function + for (comm_state<dim> state; !state.done(); state.step()) { + data->copy_from (state, tmp, ext); + } + + // Delete temporary copy + delete tmp; + + } END_COMPONENT_LOOP; + } END_MAP_LOOP; if (CCTK_MyProc(cgh)==0) { free (amrgrid->data); diff --git a/CarpetExtra/FOWaveToyF77/interface.ccl b/CarpetExtra/FOWaveToyF77/interface.ccl index 22f12f5db..b865b6b57 100644 --- a/CarpetExtra/FOWaveToyF77/interface.ccl +++ b/CarpetExtra/FOWaveToyF77/interface.ccl @@ -1,7 +1,7 @@ # Interface definition for thorn WaveToyF77 -# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/FOWaveToyF77/interface.ccl,v 1.8 2003/11/05 16:18:39 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/FOWaveToyF77/interface.ccl,v 1.9 2004/01/25 14:57:31 schnetter Exp $ -implements: fowavetoy +implements: FOwavetoy inherits: boundary grid idfoscalarwave USES INCLUDE: Symmetry.h diff --git a/CarpetExtra/WaveToyMoL/schedule.ccl b/CarpetExtra/WaveToyMoL/schedule.ccl index 2b405ae15..7d3f01265 100644 --- a/CarpetExtra/WaveToyMoL/schedule.ccl +++ b/CarpetExtra/WaveToyMoL/schedule.ccl @@ -1,5 +1,5 @@ # Schedule definitions for thorn WaveToyMoL -# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyMoL/schedule.ccl,v 1.3 2003/11/05 16:18:41 schnetter Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetExtra/WaveToyMoL/schedule.ccl,v 1.4 2004/01/25 14:57:31 schnetter Exp $ STORAGE: scalarevolve[3] STORAGE: scalarevolvedot @@ -39,15 +39,3 @@ SCHEDULE WaveToyMoL_Boundaries IN MoL_PostStep SCHEDULE GROUP ApplyBCs IN MoL_PostStep AFTER WaveToyMoL_Boundaries { } "Apply boundary conditions in MoL" - - - -SCHEDULE WaveToyMoL_Boundaries AT POSTRESTRICT -{ - LANG: Fortran - SYNC: scalarevolve -} "Select boundary conditions after restricting" - -SCHEDULE GROUP ApplyBCs AT POSTRESTRICT AFTER WaveToyMoL_Boundaries -{ -} "Apply boundary conditions after restricting" |