aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Carpet/Carpet/src/CallFunction.cc27
-rw-r--r--Carpet/Carpet/src/Comm.cc42
-rw-r--r--Carpet/Carpet/src/Restrict.cc40
-rw-r--r--Carpet/Carpet/src/carpet.hh4
-rw-r--r--Carpet/Carpet/src/variables.hh6
-rw-r--r--Carpet/CarpetIOASCII/src/ioascii.cc2
-rw-r--r--Carpet/CarpetIOHDF5/src/Output.cc4
-rw-r--r--Carpet/CarpetLib/src/commstate.cc145
-rw-r--r--Carpet/CarpetLib/src/commstate.hh82
-rw-r--r--Carpet/CarpetLib/src/data.hh9
-rw-r--r--Carpet/CarpetLib/src/dh.cc55
-rw-r--r--Carpet/CarpetLib/src/dist.hh83
-rw-r--r--Carpet/CarpetLib/src/gdata.cc163
-rw-r--r--Carpet/CarpetLib/src/gdata.hh7
-rw-r--r--Carpet/CarpetLib/src/ggf.cc10
-rw-r--r--Carpet/CarpetSlab/src/GetHyperslab.cc2
-rw-r--r--Carpet/CarpetSlab/src/slab.cc2
17 files changed, 355 insertions, 328 deletions
diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc
index 87b909bc1..9b0e416d3 100644
--- a/Carpet/Carpet/src/CallFunction.cc
+++ b/Carpet/Carpet/src/CallFunction.cc
@@ -252,33 +252,16 @@ namespace Carpet {
// check if there is anything to do
if (attribute->n_SyncGroups <= 0) return;
- vector<group_set> groups;
-
- // sort all grid variables into sets of the same vartype
+ // remove all empty groups from the list
+ vector<int> groups;
for (int g = 0; g < attribute->n_SyncGroups; g++) {
- // skip empty groups
const int group = attribute->SyncGroups[g];
- if (CCTK_NumVarsInGroupI (group) <= 0) continue;
-
- group_set newset;
- const int firstvar = CCTK_FirstVarIndexI (group);
- newset.vartype = CCTK_VarTypeI (firstvar);
- assert (newset.vartype >= 0);
- int c;
- for (c = 0; c < groups.size(); c++) {
- if (newset.vartype == groups[c].vartype) {
- break;
- }
+ if (CCTK_NumVarsInGroupI (group) > 0) {
+ groups.push_back (group);
}
- if (c == groups.size()) {
- groups.push_back (newset);
- }
- groups[c].members.push_back (group);
}
- for (int c = 0; c < groups.size(); c++) {
- SyncProlongateGroups (cgh, groups[c]);
- }
+ SyncProlongateGroups (cgh, groups);
}
} // namespace Carpet
diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index 0e0965ae4..3069e43ba 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -18,7 +18,7 @@ namespace Carpet {
static void ProlongateGroupBoundaries (const cGH* cctkGH,
CCTK_REAL initial_time,
- group_set& groups);
+ const vector<int>& groups);
// Carpet's overload function for CCTK_SyncGroup()
@@ -38,8 +38,7 @@ namespace Carpet {
const int firstvar = CCTK_FirstVarIndexI (group);
const int vartype = CCTK_VarTypeI (firstvar);
- const vector<int> members(1, group);
- group_set groups = {vartype, members};
+ const vector<int> groups(1, group);
retval = SyncProlongateGroups (cctkGH, groups);
} else {
retval = -2;
@@ -49,22 +48,21 @@ namespace Carpet {
}
- // synchronises ghostzones and prolongates boundaries
- // of a set of groups which all have the same vartype
+ // synchronises ghostzones and prolongates boundaries of a set of groups
//
// returns 0 for success and -1 if the set contains a group with no storage
- int SyncProlongateGroups (const cGH* cctkGH, group_set& groups)
+ int SyncProlongateGroups (const cGH* cctkGH, const vector<int>& groups)
{
int retval = 0;
DECLARE_CCTK_PARAMETERS;
- assert (groups.members.size() > 0);
+ assert (groups.size() > 0);
// check consistency of all groups:
// create a new set with empty and no-storage groups removed
- group_set goodgroups = {groups.vartype};
- for (size_t g = 0; g < groups.members.size(); g++) {
- const int group = groups.members[g];
+ vector<int> goodgroups;
+ for (size_t g = 0; g < groups.size(); g++) {
+ const int group = groups[g];
const int grouptype = CCTK_GroupTypeI (group);
char* groupname = CCTK_GroupName (group);
Checkpoint ("SyncGroup \"%s\" time=%g",
@@ -110,13 +108,13 @@ namespace Carpet {
retval = -1;
}
else if (CCTK_NumVarsInGroupI (group) > 0) {
- goodgroups.members.push_back(group);
+ goodgroups.push_back(group);
}
free (groupname);
}
- if (goodgroups.members.size() > 0) {
+ if (goodgroups.size() > 0) {
// prolongate boundaries
if (do_prolongate && reflevel > 0) {
ProlongateGroupBoundaries (cctkGH, cctk_initial_time, goodgroups);
@@ -132,7 +130,7 @@ namespace Carpet {
// Prolongate the boundaries of all CCTK_GF groups in the given set
static void ProlongateGroupBoundaries (const cGH* cctkGH,
CCTK_REAL initial_time,
- group_set& groups)
+ const vector<int>& groups)
{
DECLARE_CCTK_PARAMETERS;
const int tl = 0;
@@ -141,9 +139,9 @@ namespace Carpet {
const CCTK_REAL time
= (cctkGH->cctk_time - initial_time) / delta_time;
- for (comm_state state(groups.vartype); ! state.done(); state.step()) {
- for (int group = 0; group < groups.members.size(); ++group) {
- const int g = groups.members.at(group);
+ for (comm_state state; ! state.done(); state.step()) {
+ for (int group = 0; group < groups.size(); ++group) {
+ const int g = groups[group];
const int grouptype = CCTK_GroupTypeI (g);
if (grouptype != CCTK_GF) {
continue;
@@ -163,17 +161,17 @@ namespace Carpet {
}
- // synchronises a set of group which all have the same vartype
- void SyncGroups (const cGH* cctkGH, group_set& groups)
+ // synchronises a set of groups
+ void SyncGroups (const cGH* cctkGH, const vector<int>& groups)
{
DECLARE_CCTK_PARAMETERS;
const int tl = 0;
- assert (groups.members.size() > 0);
+ assert (groups.size() > 0);
- for (comm_state state(groups.vartype); ! state.done(); state.step()) {
- for (int group = 0; group < groups.members.size(); ++group) {
- const int g = groups.members.at(group);
+ for (comm_state state; ! state.done(); state.step()) {
+ for (int group = 0; group < groups.size(); ++group) {
+ const int g = groups[group];
const int grouptype = CCTK_GroupTypeI (g);
const int ml = grouptype == CCTK_GF ? mglevel : 0;
const int rl = grouptype == CCTK_GF ? reflevel : 0;
diff --git a/Carpet/Carpet/src/Restrict.cc b/Carpet/Carpet/src/Restrict.cc
index 42d4d8720..e26fb9416 100644
--- a/Carpet/Carpet/src/Restrict.cc
+++ b/Carpet/Carpet/src/Restrict.cc
@@ -16,8 +16,8 @@ namespace Carpet {
using namespace std;
- // restricts a set of groups which all have the same vartype
- static void RestrictGroups (const cGH* cgh, group_set& groups);
+ // restricts a set of groups
+ static void RestrictGroups (const cGH* cgh, const vector<int>& groups);
void Restrict (const cGH* cgh)
@@ -38,52 +38,34 @@ namespace Carpet {
return;
}
- // sort all grid functions into sets of the same vartype
- vector<group_set> groups;
+ // remove all groups with are non-GFs, empty, or have no storage assigned
+ vector<int> groups;
for (int group = 0; group < CCTK_NumGroups(); ++group) {
if (CCTK_GroupTypeI(group) == CCTK_GF
&& CCTK_NumVarsInGroupI(group) > 0
&& CCTK_QueryGroupStorageI(cgh, group)) {
-
- group_set newset;
- const int firstvar = CCTK_FirstVarIndexI (group);
- newset.vartype = CCTK_VarTypeI (firstvar);
- assert (newset.vartype >= 0);
- int c;
- for (c = 0; c < groups.size(); c++) {
- if (newset.vartype == groups[c].vartype) {
- break;
- }
- }
- if (c == groups.size()) {
- groups.push_back (newset);
- }
- groups[c].members.push_back (group);
+ groups.push_back (group);
}
}
// Restrict
- for (int c = 0; c < groups.size(); c++) {
- RestrictGroups (cgh, groups[c]);
- }
+ RestrictGroups (cgh, groups);
// Synchronise
- for (int c = 0; c < groups.size(); c++) {
- SyncGroups (cgh, groups[c]);
- }
+ SyncGroups (cgh, groups);
}
// restricts a set of groups which all have the same vartype
- static void RestrictGroups (const cGH* cgh, group_set& groups) {
+ static void RestrictGroups (const cGH* cgh, const vector<int>& groups) {
DECLARE_CCTK_PARAMETERS;
const int tl = 0;
- for (comm_state state(groups.vartype); ! state.done(); state.step()) {
- for (int c = 0; c < groups.members.size(); ++c) {
- const int group = groups.members[c];
+ for (comm_state state; ! state.done(); state.step()) {
+ for (int c = 0; c < groups.size(); ++c) {
+ const int group = groups[c];
for (int m=0; m<(int)arrdata.at(group).size(); ++m) {
// use background time here (which may not be modified
diff --git a/Carpet/Carpet/src/carpet.hh b/Carpet/Carpet/src/carpet.hh
index 0ac34613e..726c24ea1 100644
--- a/Carpet/Carpet/src/carpet.hh
+++ b/Carpet/Carpet/src/carpet.hh
@@ -36,8 +36,8 @@ namespace Carpet {
void CycleTimeLevels (const cGH* cgh);
void FlipTimeLevels (const cGH* cgh);
void Restrict (const cGH* cgh);
- void SyncGroups (const cGH* cgh, group_set& groups);
- int SyncProlongateGroups (const cGH* cgh, group_set& groups);
+ void SyncGroups (const cGH* cgh, const vector<int>& groups);
+ int SyncProlongateGroups (const cGH* cgh, const vector<int>& groups);
// Sanity checks
enum checktimes { currenttime,
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index aeed387ff..1e5330d80 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -130,12 +130,6 @@ namespace Carpet {
};
extern vector<groupdesc> groupdata; // [group]
- // structure to hold a set of groups which all have the same CCTK vartype
- struct group_set {
- int vartype; // eg. CCTK_VARIABLE_REAL, etc.
- vector<int> members; // members of this set
- }; // (given by their CCTK group indices)
-
// Data for everything
struct arrdesc {
// points to hh etc. for GF, and is unique for SCALAR and ARRAY
diff --git a/Carpet/CarpetIOASCII/src/ioascii.cc b/Carpet/CarpetIOASCII/src/ioascii.cc
index a0aac6a3b..193dddafd 100644
--- a/Carpet/CarpetIOASCII/src/ioascii.cc
+++ b/Carpet/CarpetIOASCII/src/ioascii.cc
@@ -1260,7 +1260,7 @@ namespace CarpetIOASCII {
for (size_t n=0; n<gfdatas.size(); ++n) {
gdata * const tmp = gfdatas.at(n)->make_typed(vi);
tmp->allocate(gfdatas.at(n)->extent(), 0);
- for (comm_state state(vartype); !state.done(); state.step()) {
+ for (comm_state state; !state.done(); state.step()) {
tmp->copy_from (state, gfdatas.at(n), gfdatas.at(n)->extent());
}
tmps.at(n) = tmp;
diff --git a/Carpet/CarpetIOHDF5/src/Output.cc b/Carpet/CarpetIOHDF5/src/Output.cc
index a42c9a727..272150c40 100644
--- a/Carpet/CarpetIOHDF5/src/Output.cc
+++ b/Carpet/CarpetIOHDF5/src/Output.cc
@@ -160,7 +160,7 @@ int WriteVarUnchunked (const cGH* const cctkGH,
gdata* const processor_component = data->make_typed (request->vindex);
processor_component->allocate (overlap, 0);
- for (comm_state state(group.vartype); not state.done(); state.step()) {
+ for (comm_state state; not state.done(); state.step()) {
processor_component->copy_from (state, data, overlap);
}
@@ -314,7 +314,7 @@ int WriteVarChunkedSequential (const cGH* const cctkGH,
gdata* const processor_component = data->make_typed (request->vindex);
processor_component->allocate (bbox, 0);
- for (comm_state state(group.vartype); not state.done(); state.step()) {
+ for (comm_state state; not state.done(); state.step()) {
processor_component->copy_from (state, data, bbox);
}
diff --git a/Carpet/CarpetLib/src/commstate.cc b/Carpet/CarpetLib/src/commstate.cc
index be3e485a2..1bd9a2a8c 100644
--- a/Carpet/CarpetLib/src/commstate.cc
+++ b/Carpet/CarpetLib/src/commstate.cc
@@ -10,54 +10,61 @@
// Communication state control
-comm_state::comm_state (int vartype_) : vartype(vartype_)
+comm_state::comm_state ()
{
- // If this comm state is created with a valid (ie. non-negative) CCTK vartype
- // then it is assumed to be used in collective communications for that type
- // later on, ie. it will step through
+ // If CarpetLib::use_collective_communication_buffers is set to true,
+ // this comm_state object will use collective communications,
+ // ie. it will step through
// state_get_buffer_sizes
// state_fill_send_buffers
// state_empty_recv_buffers
- // This path needs the size of the vartype, the corresponding MPI datatype
- // and the collective communication buffers initialized also.
//
- // If this comm state is created with a negative vartype then individual
- // communications on single components are used by stepping through
+ // If CarpetLib::use_collective_communication_buffers is false
+ // then individual communications on single components are used
+ // by stepping through
// state_post
// state_wait
DECLARE_CCTK_PARAMETERS;
- uses_collective_communication_buffers =
- use_collective_communication_buffers && vartype >= 0;
+ thestate = use_collective_communication_buffers ?
+ state_get_buffer_sizes : state_post;
- if (uses_collective_communication_buffers) {
- vartypesize = CCTK_VarTypeSize (vartype);
- assert (vartypesize > 0);
-
- thestate = state_get_buffer_sizes;
- switch (vartype) {
-#define TYPECASE(N,T) \
- case N: { T dummy; datatype = dist::datatype(dummy); } break;
-#include "carpet_typecase.hh"
-#undef TYPECASE
- default: assert (0 && "invalid datatype");
- }
- collbufs.resize (dist::size());
- rrequests.resize (dist::size(), MPI_REQUEST_NULL);
- srequests.resize (dist::size(), MPI_REQUEST_NULL);
- recvbuffers_ready.resize (dist::size());
+ typebufs.resize (dist::c_ndatatypes());
+ for (int type = 0; type < typebufs.size(); type++) {
+ typebufs[type].procbufs.resize(dist::size());
+ }
- } else {
- thestate = state_post;
+#define INSTANTIATE(T) \
+ { \
+ T dummy; \
+ int type = dist::c_datatype (dummy); \
+ typebufs.at(type).datatypesize = sizeof (dummy); \
+ typebufs.at(type).mpi_datatype = dist::datatype (dummy); \
}
+#include "instantiate"
+#undef INSTANTIATE
+
+ recvbuffers_ready.resize (dist::c_ndatatypes() * dist::size());
+ srequests.resize (dist::c_ndatatypes() * dist::size(), MPI_REQUEST_NULL);
+ rrequests.resize (dist::c_ndatatypes() * dist::size(), MPI_REQUEST_NULL);
}
+
void comm_state::step ()
{
assert (thestate != state_done);
- if (uses_collective_communication_buffers) {
- switch (thestate) {
+ switch (thestate) {
+ case state_post:
+ // After all sends/recvs have been posted in 'state_post'
+ // now wait for their completion in 'state_wait'.
+ thestate = state_wait;
+ break;
+
+ case state_wait:
+ thestate = state_done;
+ break;
+
case state_get_buffer_sizes:
// The sizes of the collective communication buffers are known
// so now allocate them.
@@ -65,21 +72,33 @@ void comm_state::step ()
// (a clever MPI layer may take advantage of such early posting).
num_posted_recvs = num_completed_recvs = 0;
- for (size_t i = 0; i < collbufs.size(); i++) {
- collbufs[i].sendbufbase = new char[collbufs[i].sendbufsize*vartypesize];
- collbufs[i].recvbufbase = new char[collbufs[i].recvbufsize*vartypesize];
- collbufs[i].sendbuf = collbufs[i].sendbufbase;
- collbufs[i].recvbuf = collbufs[i].recvbufbase;
+ for (int type = 0; type < typebufs.size(); type++) {
+
+ // skip unused datatype buffers
+ if (not typebufs[type].in_use) {
+ continue;
+ }
- if (collbufs[i].recvbufsize > 0) {
- MPI_Irecv (collbufs[i].recvbufbase, collbufs[i].recvbufsize,
- datatype, i, MPI_ANY_TAG, dist::comm, &rrequests[i]);
- num_posted_recvs++;
+ int& datatypesize = typebufs[type].datatypesize;
+ for (int proc = 0; proc < typebufs[type].procbufs.size(); proc++) {
+ procbufdesc& procbuf = typebufs[type].procbufs[proc];
+
+ procbuf.sendbufbase = new char[procbuf.sendbufsize*datatypesize];
+ procbuf.recvbufbase = new char[procbuf.recvbufsize*datatypesize];
+ procbuf.sendbuf = procbuf.sendbufbase;
+ procbuf.recvbuf = procbuf.recvbufbase;
+
+ if (procbuf.recvbufsize > 0) {
+ MPI_Irecv (procbuf.recvbufbase, procbuf.recvbufsize,
+ typebufs[type].mpi_datatype, proc, type,
+ dist::comm, &rrequests[dist::size()*type + proc]);
+ num_posted_recvs++;
+ }
}
}
// Now go and get the send buffers filled with data.
- // Once a buffer is full it will get posted right away
+ // Once a buffer is full it will be posted right away
// (see gdata::copy_into_sendbuffer() and
// gdata::interpolate_into_sendbuffer()).
thestate = state_fill_send_buffers;
@@ -92,14 +111,16 @@ void comm_state::step ()
case state_empty_recv_buffers:
// Finish (at least one of) the posted communications
- if (! AllPostedCommunicationsFinished ()) {
+ if (not AllPostedCommunicationsFinished ()) {
// No state change if there are still outstanding communications;
// do another comm_state loop iteration.
} else {
// Everything is done so release the collective communication buffers.
- for (size_t i = 0; i < collbufs.size(); i++) {
- delete[] collbufs[i].sendbufbase;
- delete[] collbufs[i].recvbufbase;
+ for (int type = 0; type < typebufs.size(); type++) {
+ for (int proc = 0; proc < typebufs[type].procbufs.size(); proc++) {
+ delete[] typebufs[type].procbufs[proc].sendbufbase;
+ delete[] typebufs[type].procbufs[proc].recvbufbase;
+ }
}
thestate = state_done;
}
@@ -107,33 +128,23 @@ void comm_state::step ()
default:
assert (0 && "invalid state");
- }
- } else {
- switch (thestate) {
- case state_post:
- // After all sends/recvs have been posted in 'state_post'
- // now wait for their completion in 'state_wait'.
- thestate = state_wait;
- break;
- case state_wait:
- thestate = state_done;
- break;
- default:
- assert (0 && "invalid state");
- }
}
}
+
bool comm_state::done ()
{
return thestate == state_done;
}
+
comm_state::~comm_state ()
{
- assert (thestate == state_done ||
- thestate == uses_collective_communication_buffers ?
- state_get_buffer_sizes : state_post);
+ DECLARE_CCTK_PARAMETERS;
+
+ assert (thestate == state_done or
+ thestate == (use_collective_communication_buffers ?
+ state_get_buffer_sizes : state_post));
assert (requests.empty());
}
@@ -141,14 +152,14 @@ comm_state::~comm_state ()
// wait for completion of posted collective buffer sends/receives
//
// Depending on the parameter CarpetLib::use_waitall, this function will wait
-// for all at once (true) or at least one (false) of the
-// posted receive operations to finish.
+// for all (true) or at least one (false) of the posted receive operations
+// to finish.
//
// It returns true if all posted communications have been completed.
bool comm_state::AllPostedCommunicationsFinished ()
{
DECLARE_CCTK_PARAMETERS;
-
+
// check if all outstanding receives have been completed already
if (num_posted_recvs == num_completed_recvs) {
// finalize the outstanding sends in one go
@@ -158,13 +169,11 @@ bool comm_state::AllPostedCommunicationsFinished ()
}
// reset completion flag for all receive buffers
- for (size_t i = 0; i < recvbuffers_ready.size(); i++) {
- recvbuffers_ready[i] = false;
- }
+ recvbuffers_ready.assign (recvbuffers_ready.size(), false);
if (use_waitall) {
// mark all posted recveive buffers as ready
- for (size_t i = 0; i < rrequests.size(); i++) {
+ for (size_t i = 0; i < recvbuffers_ready.size(); i++) {
recvbuffers_ready[i] = rrequests[i] != MPI_REQUEST_NULL;
}
diff --git a/Carpet/CarpetLib/src/commstate.hh b/Carpet/CarpetLib/src/commstate.hh
index e120fa5e7..e64fa8781 100644
--- a/Carpet/CarpetLib/src/commstate.hh
+++ b/Carpet/CarpetLib/src/commstate.hh
@@ -6,27 +6,22 @@
#include <mpi.h>
-
+#include "dist.hh"
using namespace std;
-
-class gdata;
-
-
-
// State information for communications
//
// Depending on how a comm state object was created,
// it will step through one of two state transitions (in the given order):
enum astate {
- // these are used for communications on individual components
- state_post, state_wait,
-
// these are used for collective communications
state_get_buffer_sizes, state_fill_send_buffers, state_empty_recv_buffers,
+ // these are used for communications on individual components
+ state_post, state_wait,
+
// all transition graphs must end with here
state_done
};
@@ -34,31 +29,25 @@ enum astate {
struct comm_state {
astate thestate;
- // If no vartype is given the comm state will be set up
- // for single-component communications.
- comm_state (int vartype = -1);
+ comm_state ();
void step ();
bool done ();
~comm_state ();
-
+
private:
// Forbid copying and passing by value
comm_state (comm_state const &);
comm_state& operator= (comm_state const &);
- // flag to indicate whether this comm state object is used for
- // collective (true) or single-component communications (false)
- bool uses_collective_communication_buffers;
-
public:
-
+
//////////////////////////////////////////////////////////////////////////
// the following members are used for single-component communications
//////////////////////////////////////////////////////////////////////////
// List of MPI requests for use_waitall
vector<MPI_Request> requests;
-
+
// Lists of communication buffers for use_lightweight_buffers
struct gcommbuf {
gcommbuf () {};
@@ -88,16 +77,13 @@ public:
//////////////////////////////////////////////////////////////////////////
public:
- // CCTK vartype used for this comm_state object
- int vartype;
-
- // size of CCTK vartype
- // (used as stride for advancing the char-based buffer pointers)
- int vartypesize;
+ // structure describing a per-processor buffer for collective communications
+ struct procbufdesc {
+ // the allocated communication buffers
+ char* sendbufbase;
+ char* recvbufbase;
- // buffers for collective communications
- struct collbufdesc {
- // the sizes of communication buffers (in elements of type <vartype>)
+ // the sizes of communication buffers (in elements of type <datatype>)
size_t sendbufsize;
size_t recvbufsize;
@@ -106,26 +92,44 @@ public:
char* sendbuf;
char* recvbuf;
- collbufdesc() : sendbufsize(0), recvbufsize(0),
+ // constructor for an instance of this structure
+ procbufdesc() : sendbufsize(0), recvbufsize(0),
sendbuf(NULL), recvbuf(NULL),
sendbufbase(NULL), recvbufbase(NULL) {}
+ };
- // the allocated communication buffers
- char* sendbufbase;
- char* recvbufbase;
+ // structure describing a collective communications buffer for a C datatype
+ struct typebufdesc {
+ // flag indicating whether this buffer is in use
+ bool in_use;
+
+ // the size of this datatype (in bytes)
+ int datatypesize;
+
+ // the corresponding MPI datatype
+ MPI_Datatype mpi_datatype;
+
+ // per-processor buffers
+ vector<procbufdesc> procbufs; // [dist::size()]
+
+ // constructor for an instance of this structure
+ typebufdesc() : in_use(false), datatypesize(0),
+ mpi_datatype(MPI_DATATYPE_NULL)
+ {
+ procbufs.resize (dist::size());
+ }
};
- vector<collbufdesc> collbufs; // [nprocs]
- // flags indicating which receive buffers are ready to be emptied
- vector<bool> recvbuffers_ready; // [nprocs]
+ // list of datatype buffers
+ vector<typebufdesc> typebufs; // [dist::c_ndatatypes()]
- // MPI datatype corresponding to CCTK vartype
- MPI_Datatype datatype;
+ // flags indicating which receive buffers are ready to be emptied
+ vector<bool> recvbuffers_ready; // [dist::size() * dist::c_ndatatypes()]
// lists of outstanding requests for posted send/recv communications
- vector<MPI_Request> srequests; // [nprocs]
+ vector<MPI_Request> srequests; // [dist::size() * dist::c_ndatatypes()]
private:
- vector<MPI_Request> rrequests; // [nprocs]
+ vector<MPI_Request> rrequests; // [dist::size() * dist::c_ndatatypes()]
// number of posted and already completed receive communications
int num_posted_recvs;
diff --git a/Carpet/CarpetLib/src/data.hh b/Carpet/CarpetLib/src/data.hh
index 663c58843..8993e4d43 100644
--- a/Carpet/CarpetLib/src/data.hh
+++ b/Carpet/CarpetLib/src/data.hh
@@ -125,6 +125,15 @@ protected:
const ibbox& box);
#endif
+ // Datatype accessors
+private:
+ // maps the C datatype of a data class object to a 0-based index
+ virtual unsigned int c_datatype () const
+ {
+ T dummy;
+ return dist::c_datatype (dummy);
+ }
+
// Data manipulators
private:
virtual comm_state::gcommbuf *
diff --git a/Carpet/CarpetLib/src/dh.cc b/Carpet/CarpetLib/src/dh.cc
index 6b15cacfa..8a67903b2 100644
--- a/Carpet/CarpetLib/src/dh.cc
+++ b/Carpet/CarpetLib/src/dh.cc
@@ -14,15 +14,6 @@ using namespace std;
-// structure to hold a set of grid functions
-// which all have the same CCTK vartype
-struct gf_set {
- int vartype; // e.g. CCTK_VARIABLE_REAL, etc.
- vector<ggf*> members; // members of this set
-};
-
-
-
// Constructors
dh::dh (gh& h_,
const ivect& lghosts_, const ivect& ughosts_,
@@ -573,24 +564,6 @@ void dh::save_time (bool do_prolongate)
{
DECLARE_CCTK_PARAMETERS;
- // sort all grid functions into sets of the same vartype
- vector<gf_set> ggfs;
- for (list<ggf*>::iterator f = gfs.begin(); f != gfs.end(); ++f) {
- gf_set newset;
- newset.vartype = CCTK_VarTypeI ((*f)->varindex);
- assert (newset.vartype >= 0);
- int c;
- for (c = 0; c < ggfs.size(); c++) {
- if (newset.vartype == ggfs[c].vartype) {
- break;
- }
- }
- if (c == ggfs.size()) {
- ggfs.push_back (newset);
- }
- ggfs[c].members.push_back (*f);
- }
-
for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
(*f)->recompose_crop ();
}
@@ -603,27 +576,17 @@ void dh::save_time (bool do_prolongate)
any_level_changed |= this_level_changed;
if (! fast_recomposing or any_level_changed) {
- for (int c = 0; c < ggfs.size(); c++) {
- for (int g = 0; g < ggfs[c].members.size(); g++) {
- ggfs[c].members[g]->recompose_allocate (rl);
- }
- for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
- for (int g = 0; g < ggfs[c].members.size(); g++) {
- ggfs[c].members[g]->recompose_fill (state, rl, do_prolongate);
- }
- }
- for (int g = 0; g < ggfs[c].members.size(); g++) {
- ggfs[c].members[g]->recompose_free (rl);
+ for (list<ggf*>::iterator f=gfs.begin(); f!=gfs.end(); ++f) {
+ (*f)->recompose_allocate (rl);
+ for (comm_state state; ! state.done(); state.step()) {
+ (*f)->recompose_fill (state, rl, do_prolongate);
}
- for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
- for (int g = 0; g < ggfs[c].members.size(); g++) {
- ggfs[c].members[g]->recompose_bnd_prolongate (state, rl, do_prolongate);
- }
+ (*f)->recompose_free (rl);
+ for (comm_state state; ! state.done(); state.step()) {
+ (*f)->recompose_bnd_prolongate (state, rl, do_prolongate);
}
- for (comm_state state(ggfs[c].vartype); ! state.done(); state.step()) {
- for (int g = 0; g < ggfs[c].members.size(); g++) {
- ggfs[c].members[g]->recompose_sync (state, rl, do_prolongate);
- }
+ for (comm_state state; ! state.done(); state.step()) {
+ (*f)->recompose_sync (state, rl, do_prolongate);
}
} // for all grid functions of same vartype
diff --git a/Carpet/CarpetLib/src/dist.hh b/Carpet/CarpetLib/src/dist.hh
index fc188e37e..4435c6fb2 100644
--- a/Carpet/CarpetLib/src/dist.hh
+++ b/Carpet/CarpetLib/src/dist.hh
@@ -63,7 +63,88 @@ namespace dist {
- // Datatype helpers
+ /////////////////////////////////////////////////////////////////////////
+ // C Datatype helpers
+ // Map a C datatype to a 0-based index running up to c_ndatatypes().
+ /////////////////////////////////////////////////////////////////////////
+ inline unsigned int c_datatype (const char&)
+ { return 0; }
+
+ inline unsigned int c_datatype (const signed char&)
+ { return 1; }
+
+ inline unsigned int c_datatype (const unsigned char&)
+ { return 2; }
+
+ inline unsigned int c_datatype (const short&)
+ { return 3; }
+
+ inline unsigned int c_datatype (const unsigned short&)
+ { return 4; }
+
+ inline unsigned int c_datatype (const int&)
+ { return 5; }
+
+ inline unsigned int c_datatype (const unsigned int&)
+ { return 6; }
+
+ inline unsigned int c_datatype (const long&)
+ { return 7; }
+
+ inline unsigned int c_datatype (const unsigned long&)
+ { return 8; }
+
+ inline unsigned int c_datatype (const long long&)
+ { return 9; }
+
+ inline unsigned int c_datatype (const float&)
+ { return 10; }
+
+ inline unsigned int c_datatype (const double&)
+ { return 11; }
+
+ inline unsigned int c_datatype (const long double&)
+ { return 12; }
+
+#if 0
+
+ inline unsigned int c_datatype (const complex<float>&)
+ { return 13; }
+
+ inline unsigned int c_datatype (const complex<double>&)
+ { return 14; }
+
+ inline unsigned int c_datatype (const complex<long double>&)
+ { return 15; }
+
+#else
+
+# ifdef CCTK_REAL4
+ inline unsigned int c_datatype (const CCTK_COMPLEX8&)
+ { return 13; }
+# endif
+
+# ifdef CCTK_REAL8
+ inline unsigned int c_datatype (const CCTK_COMPLEX16&)
+ { return 14; }
+# endif
+
+# ifdef CCTK_REAL16
+ inline unsigned int c_datatype (const CCTK_COMPLEX32&)
+ { return 15; }
+# endif
+
+#endif
+
+ // keep this function's return code consistent with functions above
+ inline unsigned int c_ndatatypes ()
+ { return 16; }
+
+
+ /////////////////////////////////////////////////////////////////
+ // MPI Datatype helpers
+ // Map a C datatype to its corresponding MPI datatype.
+ /////////////////////////////////////////////////////////////////
inline MPI_Datatype datatype (const char&)
{ return MPI_CHAR; }
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index 3b50abbee..0d85915d2 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -37,9 +37,11 @@ static int nexttag ()
// Constructors
-gdata::gdata (const int varindex_, const operator_type transport_operator_,
+gdata::gdata (const int varindex_,
+ const operator_type transport_operator_,
const int tag_)
- : varindex(varindex_), transport_operator(transport_operator_),
+ : varindex(varindex_),
+ transport_operator(transport_operator_),
_has_storage(false),
comm_active(false),
tag(tag_ >= 0 ? tag_ : nexttag())
@@ -75,7 +77,7 @@ void gdata::change_processor (comm_state& state,
change_processor_wait (state, newproc, mem);
break;
default:
- assert(0 && "invalid state");
+ assert(0 and "invalid state");
}
}
@@ -85,18 +87,18 @@ void gdata::change_processor (comm_state& state,
void gdata::copy_from (comm_state& state,
const gdata* src, const ibbox& box)
{
- assert (has_storage() && src->has_storage());
+ assert (has_storage() and src->has_storage());
assert (all(box.lower()>=extent().lower()
- && box.lower()>=src->extent().lower()));
+ and box.lower()>=src->extent().lower()));
assert (all(box.upper()<=extent().upper()
- && box.upper()<=src->extent().upper()));
+ and box.upper()<=src->extent().upper()));
assert (all(box.stride()==extent().stride()
- && box.stride()==src->extent().stride()));
+ and box.stride()==src->extent().stride()));
assert (all((box.lower()-extent().lower())%box.stride() == 0
- && (box.lower()-src->extent().lower())%box.stride() == 0));
+ and (box.lower()-src->extent().lower())%box.stride() == 0));
if (box.empty()) return;
- if (dist::rank() != proc() && dist::rank() != src->proc()) return;
+ if (dist::rank() != proc() and dist::rank() != src->proc()) return;
switch (state.thestate) {
case state_post:
@@ -117,12 +119,16 @@ void gdata::copy_from (comm_state& state,
// don't count processor-local copies
if (proc() != src->proc()) {
// if this is a destination processor: advance its recv buffer size
+ vector<comm_state::procbufdesc>& procbufs =
+ state.typebufs.at(c_datatype()).procbufs;
if (proc() == dist::rank()) {
- state.collbufs.at(src->proc()).recvbufsize += box.size();
+ procbufs.at(src->proc()).recvbufsize += box.size();
+ state.typebufs.at(c_datatype()).in_use = true;
}
// if this is a source processor: increment its send buffer size
if (src->proc() == dist::rank()) {
- state.collbufs.at(proc()).sendbufsize += box.size();
+ procbufs.at(proc()).sendbufsize += box.size();
+ state.typebufs.at(c_datatype()).in_use = true;
}
}
break;
@@ -142,13 +148,14 @@ void gdata::copy_from (comm_state& state,
case state_empty_recv_buffers:
// if this is a destination processor and data has already been received
// from the source processor: copy it from the recv buffer
- if (proc() == dist::rank() && state.recvbuffers_ready.at(src->proc())) {
+ if (proc() == dist::rank() and
+ state.recvbuffers_ready.at(dist::size()*c_datatype() + src->proc())) {
copy_from_recvbuffer (state, src, box);
}
break;
default:
- assert(0 && "invalid state");
+ assert(0 and "invalid state");
}
}
@@ -187,24 +194,21 @@ void gdata::copy_from_post (comm_state& state,
ivect items = (box.upper() - box.lower()) / box.stride() + 1;
ivect offs = (box.lower() - ext.lower()) / ext.stride();
char* send_buffer = (char*) b->pointer();
- const int vartype = CCTK_VarTypeI(varindex);
- const int vartypesize = CCTK_VarTypeSize(vartype);
- assert(vartypesize >= 0);
+ int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
for (int k = 0; k < items[2]; k++) {
for (int j = 0; j < items[1]; j++) {
int i = offs[0] + shape[0]*((j+offs[1]) + shape[1]*(k+offs[2]));
- memcpy (send_buffer,
- ((char*) src->storage()) + vartypesize*i,
- vartypesize*items[0]);
- send_buffer += vartypesize*items[0];
+ memcpy (send_buffer, ((char*) src->storage()) + datatypesize*i,
+ datatypesize * items[0]);
+ send_buffer += datatypesize * items[0];
}
}
wtime_copyfrom_sendinner_copy.stop();
wtime_copyfrom_sendinner_send.start();
MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
- tag, dist::comm, &b->request);
+ tag, dist::comm, &b->request);
wtime_copyfrom_sendinner_send.stop();
state.requests.push_back (b->request);
state.sendbufs.push (b);
@@ -220,7 +224,7 @@ void gdata::copy_from_wait (comm_state& state,
wtime_copyfrom_wait.start();
wtime_copyfrom_recvwaitinner_wait.start();
- if (! state.requests.empty()) {
+ if (not state.requests.empty()) {
// wait for all requests at once
MPI_Waitall (state.requests.size(), &state.requests.front(),
MPI_STATUSES_IGNORE);
@@ -240,17 +244,14 @@ void gdata::copy_from_wait (comm_state& state,
ivect items = (box.upper() - box.lower()) / box.stride() + 1;
ivect offs = (box.lower() - ext.lower()) / ext.stride();
const char* recv_buffer = (const char*) b->pointer();
- const int vartype = CCTK_VarTypeI(varindex);
- const int vartypesize = CCTK_VarTypeSize(vartype);
- assert(vartypesize >= 0);
+ int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
for (int k = 0; k < items[2]; k++) {
for (int j = 0; j < items[1]; j++) {
int i = offs[0] + shape[0]*((j+offs[1]) + shape[1]*(k+offs[2]));
- memcpy (((char*) storage()) + vartypesize*i,
- recv_buffer,
- vartypesize*items[0]);
- recv_buffer += vartypesize*items[0];
+ memcpy (((char*) storage()) + datatypesize*i, recv_buffer,
+ datatypesize * items[0]);
+ recv_buffer += datatypesize * items[0];
}
}
wtime_copyfrom_recvwaitinner_copy.stop();
@@ -276,15 +277,14 @@ void gdata::copy_into_sendbuffer (comm_state& state,
} else {
// copy to remote processor
assert (src->_has_storage);
- assert (state.collbufs.at(proc()).sendbuf -
- state.collbufs.at(proc()).sendbufbase <=
- (state.collbufs.at(proc()).sendbufsize - box.size()) *
- state.vartypesize);
- assert (proc() < state.collbufs.size());
- int fillstate = (state.collbufs[proc()].sendbuf +
- box.size()*state.vartypesize) -
- state.collbufs[proc()].sendbufbase;
- assert (fillstate <= state.collbufs[proc()].sendbufsize*state.vartypesize);
+ int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
+ comm_state::procbufdesc& procbuf =
+ state.typebufs.at(c_datatype()).procbufs.at(proc());
+ assert (procbuf.sendbuf - procbuf.sendbufbase <=
+ (procbuf.sendbufsize - box.size()) * datatypesize);
+ int fillstate = procbuf.sendbuf + box.size()*datatypesize -
+ procbuf.sendbufbase;
+ assert (fillstate <= procbuf.sendbufsize * datatypesize);
// copy this processor's data into the send buffer
const ibbox& ext = src->extent();
@@ -292,22 +292,23 @@ void gdata::copy_into_sendbuffer (comm_state& state,
ivect items = (box.upper() - box.lower()) / box.stride() + 1;
ivect offs = (box.lower() - ext.lower()) / ext.stride();
+ assert (dim == 3);
for (int k = 0; k < items[2]; k++) {
for (int j = 0; j < items[1]; j++) {
int i = offs[0] + shape[0]*((j+offs[1]) + shape[1]*(k+offs[2]));
- memcpy (state.collbufs[proc()].sendbuf,
- ((const char*) src->storage()) + state.vartypesize*i,
- state.vartypesize*items[0]);
- state.collbufs[proc()].sendbuf += state.vartypesize*items[0];
+ memcpy (procbuf.sendbuf,
+ ((const char*) src->storage()) + datatypesize*i,
+ datatypesize * items[0]);
+ procbuf.sendbuf += datatypesize * items[0];
}
}
// post the send if the buffer is full
- if (fillstate == state.collbufs[proc()].sendbufsize*state.vartypesize) {
- MPI_Isend (state.collbufs[proc()].sendbufbase,
- state.collbufs[proc()].sendbufsize,
- state.datatype, proc(), 0, dist::comm,
- &state.srequests[proc()]);
+ if (fillstate == procbuf.sendbufsize * datatypesize) {
+ MPI_Isend (procbuf.sendbufbase, procbuf.sendbufsize,
+ state.typebufs.at(c_datatype()).mpi_datatype,
+ proc(), c_datatype(), dist::comm,
+ &state.srequests.at(dist::size()*c_datatype() + proc()));
}
}
}
@@ -318,11 +319,11 @@ void gdata::copy_into_sendbuffer (comm_state& state,
void gdata::copy_from_recvbuffer (comm_state& state,
const gdata* src, const ibbox& box)
{
- assert (src->proc() < state.collbufs.size());
- assert (state.collbufs[src->proc()].recvbuf -
- state.collbufs[src->proc()].recvbufbase <=
- (state.collbufs[src->proc()].recvbufsize-box.size()) *
- state.vartypesize);
+ int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
+ comm_state::procbufdesc& procbuf =
+ state.typebufs.at(c_datatype()).procbufs.at(src->proc());
+ assert (procbuf.recvbuf - procbuf.recvbufbase <=
+ (procbuf.recvbufsize-box.size()) * datatypesize);
// copy this processor's data from the recv buffer
const ibbox& ext = extent();
@@ -330,13 +331,13 @@ void gdata::copy_from_recvbuffer (comm_state& state,
ivect items = (box.upper() - box.lower()) / box.stride() + 1;
ivect offs = (box.lower() - ext.lower()) / ext.stride();
+ assert (dim == 3);
for (int k = 0; k < items[2]; k++) {
for (int j = 0; j < items[1]; j++) {
int i = offs[0] + shape[0]*((j+offs[1]) + shape[1]*(k+offs[2]));
- memcpy (((char*) storage()) + state.vartypesize*i,
- state.collbufs[src->proc()].recvbuf,
- state.vartypesize*items[0]);
- state.collbufs[src->proc()].recvbuf += state.vartypesize*items[0];
+ memcpy (((char*) storage()) + datatypesize*i,
+ procbuf.recvbuf, datatypesize * items[0]);
+ procbuf.recvbuf += datatypesize * items[0];
}
}
}
@@ -358,15 +359,15 @@ void gdata
assert (all(box.upper()<=extent().upper()));
assert (all(box.stride()==extent().stride()));
assert (all((box.lower()-extent().lower())%box.stride() == 0));
- assert (srcs.size() == times.size() && srcs.size()>0);
+ assert (srcs.size() == times.size() and srcs.size()>0);
for (int t=0; t<(int)srcs.size(); ++t) {
assert (srcs.at(t)->has_storage());
assert (all(box.lower()>=srcs.at(t)->extent().lower()));
assert (all(box.upper()<=srcs.at(t)->extent().upper()));
}
- assert (! box.empty());
+ assert (not box.empty());
const gdata* src = srcs.at(0);
- if (dist::rank() != proc() && dist::rank() != src->proc()) return;
+ if (dist::rank() != proc() and dist::rank() != src->proc()) return;
switch (state.thestate) {
case state_post:
@@ -387,12 +388,16 @@ void gdata
// don't count processor-local interpolations
if (proc() != src->proc()) {
// if this is a destination processor: increment its recv buffer size
+ vector<comm_state::procbufdesc>& procbufs =
+ state.typebufs.at(c_datatype()).procbufs;
if (proc() == dist::rank()) {
- state.collbufs.at(src->proc()).recvbufsize += box.size();
+ procbufs.at(src->proc()).recvbufsize += box.size();
+ state.typebufs.at(c_datatype()).in_use = true;
}
// if this is a source processor: increment its send buffer size
if (src->proc() == dist::rank()) {
- state.collbufs.at(proc()).sendbufsize += box.size();
+ procbufs.at(proc()).sendbufsize += box.size();
+ state.typebufs.at(c_datatype()).in_use = true;
}
}
break;
@@ -413,12 +418,13 @@ void gdata
// if this is a destination processor and data has already been received
// from the source processor: copy it from the recv buffer
// (the processor-local case is not handled here)
- if (proc() == dist::rank() && state.recvbuffers_ready.at(src->proc())) {
+ if (proc() == dist::rank() and
+ state.recvbuffers_ready.at(dist::size()*c_datatype() + src->proc())) {
copy_from_recvbuffer(state, src, box);
}
break;
default:
- assert(0 && "invalid state");
+ assert(0 and "invalid state");
}
}
@@ -482,33 +488,32 @@ void gdata
// interpolate to remote processor
const gdata* src = srcs.at(0);
assert (src->_has_storage);
- assert (state.collbufs.at(proc()).sendbuf -
- state.collbufs.at(proc()).sendbufbase <=
- (state.collbufs.at(proc()).sendbufsize - box.size()) *
- state.vartypesize);
+ int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
+ comm_state::procbufdesc& procbuf =
+ state.typebufs.at(c_datatype()).procbufs.at(proc());
+ assert (procbuf.sendbuf - procbuf.sendbufbase <=
+ (procbuf.sendbufsize - box.size()) * datatypesize);
assert (src->has_storage());
- assert (proc() < state.collbufs.size());
- int fillstate = (state.collbufs[proc()].sendbuf +
- box.size()*state.vartypesize) -
- state.collbufs[proc()].sendbufbase;
- assert (fillstate <= state.collbufs[proc()].sendbufsize*state.vartypesize);
+ int fillstate = (procbuf.sendbuf + box.size()*datatypesize) -
+ procbuf.sendbufbase;
+ assert (fillstate <= procbuf.sendbufsize * datatypesize);
// interpolate this processor's data into the send buffer
gdata* tmp = src->make_typed (varindex, transport_operator, tag);
- tmp->allocate (box, src->proc(), state.collbufs[proc()].sendbuf);
+ tmp->allocate (box, src->proc(), procbuf.sendbuf);
tmp->interpolate_from_innerloop (srcs, times, box, time,
order_space, order_time);
delete tmp;
// advance send buffer to point to the next ibbox slot
- state.collbufs[proc()].sendbuf += state.vartypesize * box.size();
+ procbuf.sendbuf += datatypesize * box.size();
// post the send if the buffer is full
- if (fillstate == state.collbufs[proc()].sendbufsize*state.vartypesize) {
- MPI_Isend (state.collbufs[proc()].sendbufbase,
- state.collbufs[proc()].sendbufsize,
- state.datatype, proc(), 0, dist::comm,
- &state.srequests[proc()]);
+ if (fillstate == procbuf.sendbufsize*datatypesize) {
+ MPI_Isend (procbuf.sendbufbase, procbuf.sendbufsize,
+ state.typebufs.at(c_datatype()).mpi_datatype,
+ proc(), c_datatype(), dist::comm,
+ &state.srequests.at(dist::size()*c_datatype() + proc()));
}
}
}
diff --git a/Carpet/CarpetLib/src/gdata.hh b/Carpet/CarpetLib/src/gdata.hh
index 8f2b66035..7845d8fd9 100644
--- a/Carpet/CarpetLib/src/gdata.hh
+++ b/Carpet/CarpetLib/src/gdata.hh
@@ -29,6 +29,7 @@ protected: // should be readonly
// Fields
const int varindex; // Cactus variable index, or -1
+
operator_type transport_operator;
bool _has_storage; // has storage associated (on some processor)
@@ -91,7 +92,6 @@ public:
virtual void free () = 0;
// Accessors
-
bool has_storage () const {
return _has_storage;
}
@@ -134,6 +134,11 @@ public:
return dot(ind, stride());
}
+private:
+ // Datatype accessors
+ // maps the C datatype of a data class object to a 0-based index
+ virtual unsigned int c_datatype () const = 0;
+
// Data manipulators
private:
virtual comm_state::gcommbuf *
diff --git a/Carpet/CarpetLib/src/ggf.cc b/Carpet/CarpetLib/src/ggf.cc
index 360ea58db..d360ddf23 100644
--- a/Carpet/CarpetLib/src/ggf.cc
+++ b/Carpet/CarpetLib/src/ggf.cc
@@ -331,7 +331,6 @@ void ggf::copycat (comm_state& state,
assert (ml1>=0 and ml1<h.mglevels());
assert (tl2>=0 and tl2<timelevels());
assert (rl2>=0 and rl2<h.reflevels());
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
const int c2=c1;
assert (ml2<h.mglevels());
const ibbox recv = d.boxes.at(ml1).at(rl1).at(c1).*recv_box;
@@ -354,12 +353,10 @@ void ggf::copycat (comm_state& state,
assert ( ml2<h.mglevels());
assert (tl2>=0 and tl2<timelevels());
assert (rl2>=0 and rl2<h.reflevels());
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
const int c2=c1;
const iblist recv = d.boxes.at(ml1).at(rl1).at(c1).*recv_list;
// walk all boxes
- for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r)
- {
+ for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r) {
// (use the send boxes for communication)
// copy the content
gdata* const dst = storage.at(ml1).at(rl1).at(c1).at(tl1);
@@ -381,7 +378,6 @@ void ggf::copycat (comm_state& state,
assert ( ml2<h.mglevels());
assert (tl2>=0 and tl2<timelevels());
assert (rl2>=0 and rl2<h.reflevels());
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
// walk all components
for (int c2=0; c2<h.components(rl2); ++c2) {
const iblist recv = (d.boxes.at(ml1).at(rl1).at(c1).*recv_listvect).at(c2);
@@ -418,7 +414,6 @@ void ggf::intercat (comm_state& state,
vector<CCTK_REAL> times(tl2s.size());
for (int i=0; i<(int)gsrcs.size(); ++i) {
gsrcs.at(i) = storage.at(ml2).at(rl2).at(c2).at(tl2s.at(i));
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
times.at(i) = t.time(tl2s.at(i),rl2,ml2);
}
@@ -451,7 +446,6 @@ void ggf::intercat (comm_state& state,
vector<CCTK_REAL> times(tl2s.size());
for (int i=0; i<(int)gsrcs.size(); ++i) {
gsrcs.at(i) = storage.at(ml2).at(rl2).at(c2).at(tl2s.at(i));
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
times.at(i) = t.time(tl2s.at(i),rl2,ml2);
}
@@ -490,7 +484,6 @@ void ggf::intercat (comm_state& state,
vector<CCTK_REAL> times(tl2s.size());
for (int i=0; i<(int)gsrcs.size(); ++i) {
gsrcs.at(i) = storage.at(ml2).at(rl2).at(c2).at(tl2s.at(i));
- assert (state.vartype < 0 || state.vartype == CCTK_VarTypeI (varindex));
times.at(i) = t.time(tl2s.at(i),rl2,ml2);
}
@@ -540,6 +533,7 @@ void ggf::ref_bnd_prolongate (comm_state& state,
vector<int> tl2s;
if (transport_operator != op_copy) {
// Interpolation in time
+if (timelevels() < prolongation_order_time+1) fprintf (stderr, "got '%s' with %d timelevels order %d\n", CCTK_FullName (varindex), timelevels(), prolongation_order_time);
assert (timelevels() >= prolongation_order_time+1);
tl2s.resize(prolongation_order_time+1);
for (int i=0; i<=prolongation_order_time; ++i) tl2s.at(i) = i;
diff --git a/Carpet/CarpetSlab/src/GetHyperslab.cc b/Carpet/CarpetSlab/src/GetHyperslab.cc
index 4f6bb34a4..1c9e687cc 100644
--- a/Carpet/CarpetSlab/src/GetHyperslab.cc
+++ b/Carpet/CarpetSlab/src/GetHyperslab.cc
@@ -177,7 +177,7 @@ namespace CarpetSlab {
// Done with the temporary stuff
mydata = 0;
- for (comm_state state(gp.vartype); !state.done(); state.step()) {
+ for (comm_state state; !state.done(); state.step()) {
// Loop over all components, copying data from them
BEGIN_LOCAL_COMPONENT_LOOP (cgh, gp.grouptype) {
diff --git a/Carpet/CarpetSlab/src/slab.cc b/Carpet/CarpetSlab/src/slab.cc
index be0e39674..70f97f257 100644
--- a/Carpet/CarpetSlab/src/slab.cc
+++ b/Carpet/CarpetSlab/src/slab.cc
@@ -209,7 +209,7 @@ namespace CarpetSlab {
// Done with the temporary stuff
mydata = 0;
- for (comm_state state(gp.vartype); !state.done(); state.step()) {
+ for (comm_state state; !state.done(); state.step()) {
// Loop over all components, copying data from them
BEGIN_COMPONENT_LOOP (cgh, gp.grouptype) {
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-06 13:50:54 +0000