aboutsummaryrefslogtreecommitdiff
path: root/Carpet/CarpetLib/src/gdata.cc
diff options
context:
space:
mode:
Diffstat (limited to 'Carpet/CarpetLib/src/gdata.cc')
-rw-r--r--Carpet/CarpetLib/src/gdata.cc438
1 files changed, 438 insertions, 0 deletions
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
new file mode 100644
index 000000000..f05c88122
--- /dev/null
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -0,0 +1,438 @@
+// $Header: /home/eschnett/C/carpet/Carpet/Carpet/CarpetLib/src/gdata.cc,v 1.29 2004/04/07 16:58:07 schnetter Exp $
+
+#include <assert.h>
+#include <stdlib.h>
+
+#include <iostream>
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+#include "util_ErrorCodes.h"
+#include "util_Table.h"
+
+#include "bbox.hh"
+#include "defs.hh"
+#include "dist.hh"
+#include "vect.hh"
+
+#include "gdata.hh"
+
+using namespace std;
+
+
+
+// Communication state control
+template<int D>
+comm_state<D>::comm_state ()
+ : thestate(state_recv),
+ current(0)
+{
+}
+
+template<int D>
+void comm_state<D>::step ()
+{
+ assert (thestate!=state_done);
+ assert (current==tmps.size());
+ thestate=astate(size_t(thestate)+1);
+ current=0;
+}
+
+template<int D>
+bool comm_state<D>::done ()
+{
+ return thestate==state_done;
+}
+
+template<int D>
+comm_state<D>::~comm_state ()
+{
+ assert (thestate==state_recv || thestate==state_done);
+}
+
+
+
+// Hand out the next MPI tag
+static int nexttag ()
+{
+ static int last = 100;
+ ++last;
+ if (last > 30000) last = 100;
+ return last;
+}
+
+
+
+// Constructors
+template<int D>
+gdata<D>::gdata (const int varindex_, const operator_type transport_operator_)
+ : varindex(varindex_), transport_operator(transport_operator_),
+ wtime_isend(0.0), wtime_isendwait(0.0),
+ wtime_irecv(0.0), wtime_irecvwait(0.0),
+ _has_storage(false),
+ comm_active(false),
+ tag(nexttag())
+{
+ DECLARE_CCTK_PARAMETERS;
+ if (barriers) {
+ MPI_Barrier (dist::comm);
+ }
+}
+
+// Destructors
+template<int D>
+gdata<D>::~gdata ()
+{
+ DECLARE_CCTK_PARAMETERS;
+ if (barriers) {
+ MPI_Barrier (dist::comm);
+ }
+}
+
+
+
+// Data manipulators
+template<int D>
+void gdata<D>::copy_from (comm_state<D>& state,
+ const gdata* src, const ibbox& box)
+{
+ switch (state.thestate) {
+ case state_recv:
+ copy_from_recv (state, src, box);
+ break;
+ case state_send:
+ copy_from_send (state, src, box);
+ break;
+ case state_wait:
+ copy_from_wait (state, src, box);
+ break;
+ default:
+ assert(0);
+ }
+}
+
+
+
+template<int D>
+void gdata<D>::copy_from_nocomm (const gdata* src, const ibbox& box)
+{
+ assert (has_storage() && src->has_storage());
+ assert (all(box.lower()>=extent().lower()
+ && box.lower()>=src->extent().lower()));
+ assert (all(box.upper()<=extent().upper()
+ && box.upper()<=src->extent().upper()));
+ assert (all(box.stride()==extent().stride()
+ && box.stride()==src->extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0
+ && (box.lower()-src->extent().lower())%box.stride() == 0));
+
+ if (box.empty()) return;
+
+ assert (proc() == src->proc());
+
+ // copy on same processor
+ int rank;
+ MPI_Comm_rank (dist::comm, &rank);
+ if (rank == proc()) {
+ copy_from_innerloop (src, box);
+ }
+}
+
+
+
+template<int D>
+void gdata<D>::copy_from_recv (comm_state<D>& state,
+ const gdata* src, const ibbox& box)
+{
+ assert (has_storage() && src->has_storage());
+ assert (all(box.lower()>=extent().lower()
+ && box.lower()>=src->extent().lower()));
+ assert (all(box.upper()<=extent().upper()
+ && box.upper()<=src->extent().upper()));
+ assert (all(box.stride()==extent().stride()
+ && box.stride()==src->extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0
+ && (box.lower()-src->extent().lower())%box.stride() == 0));
+
+ if (box.empty()) return;
+
+ if (proc() == src->proc()) {
+ // copy on same processor
+
+ } else {
+
+ // copy to different processor
+ gdata<D>* const tmp = make_typed(varindex, transport_operator);
+ // TODO: is this efficient?
+ state.tmps.push_back (tmp);
+ ++state.current;
+ tmp->allocate (box, src->proc());
+ tmp->change_processor_recv (proc());
+
+ }
+}
+
+
+
+template<int D>
+void gdata<D>::copy_from_send (comm_state<D>& state,
+ const gdata* src, const ibbox& box)
+{
+ assert (has_storage() && src->has_storage());
+ assert (all(box.lower()>=extent().lower()
+ && box.lower()>=src->extent().lower()));
+ assert (all(box.upper()<=extent().upper()
+ && box.upper()<=src->extent().upper()));
+ assert (all(box.stride()==extent().stride()
+ && box.stride()==src->extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0
+ && (box.lower()-src->extent().lower())%box.stride() == 0));
+
+ if (box.empty()) return;
+
+ if (proc() == src->proc()) {
+ // copy on same processor
+
+ copy_from_nocomm (src, box);
+
+ } else {
+
+ // copy to different processor
+ gdata<D>* const tmp = state.tmps.at(state.current++);
+ assert (tmp);
+ tmp->copy_from_nocomm (src, box);
+ tmp->change_processor_send (proc());
+
+ }
+}
+
+
+
+template<int D>
+void gdata<D>::copy_from_wait (comm_state<D>& state,
+ const gdata* src, const ibbox& box)
+{
+ assert (has_storage() && src->has_storage());
+ assert (all(box.lower()>=extent().lower()
+ && box.lower()>=src->extent().lower()));
+ assert (all(box.upper()<=extent().upper()
+ && box.upper()<=src->extent().upper()));
+ assert (all(box.stride()==extent().stride()
+ && box.stride()==src->extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0
+ && (box.lower()-src->extent().lower())%box.stride() == 0));
+
+ if (box.empty()) return;
+
+ if (proc() == src->proc()) {
+ // copy on same processor
+
+ } else {
+
+ // copy to different processor
+ gdata<D>* const tmp = state.tmps.at(state.current++);
+ assert (tmp);
+ tmp->change_processor_wait (proc());
+ copy_from_nocomm (tmp, box);
+ delete tmp;
+
+ }
+}
+
+
+
+template<int D>
+void gdata<D>
+::interpolate_from (comm_state<D>& state,
+ const vector<const gdata*> srcs,
+ const vector<CCTK_REAL> times,
+ const ibbox& box, const CCTK_REAL time,
+ const int order_space,
+ const int order_time)
+{
+ assert (transport_operator != op_error);
+ if (transport_operator == op_none) return;
+ switch (state.thestate) {
+ case state_recv:
+ interpolate_from_recv (state, srcs, times, box, time, order_space, order_time);
+ break;
+ case state_send:
+ interpolate_from_send (state, srcs, times, box, time, order_space, order_time);
+ break;
+ case state_wait:
+ interpolate_from_wait (state, srcs, times, box, time, order_space, order_time);
+ break;
+ default:
+ assert(0);
+ }
+}
+
+
+
+template<int D>
+void gdata<D>
+::interpolate_from_nocomm (const vector<const gdata*> srcs,
+ const vector<CCTK_REAL> times,
+ const ibbox& box, const CCTK_REAL time,
+ const int order_space,
+ const int order_time)
+{
+ assert (has_storage());
+ assert (all(box.lower()>=extent().lower()));
+ assert (all(box.upper()<=extent().upper()));
+ assert (all(box.stride()==extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0));
+ assert (srcs.size() == times.size() && srcs.size()>0);
+ for (int t=0; t<(int)srcs.size(); ++t) {
+ assert (srcs.at(t)->has_storage());
+ assert (all(box.lower()>=srcs.at(t)->extent().lower()));
+ assert (all(box.upper()<=srcs.at(t)->extent().upper()));
+ }
+
+ assert (! box.empty());
+ if (box.empty()) return;
+
+ assert (proc() == srcs.at(0)->proc());
+
+ assert (transport_operator != op_error);
+ assert (transport_operator != op_none);
+
+ // interpolate on same processor
+ int rank;
+ MPI_Comm_rank (dist::comm, &rank);
+ if (rank == proc()) {
+ interpolate_from_innerloop
+ (srcs, times, box, time, order_space, order_time);
+ }
+}
+
+
+
+template<int D>
+void gdata<D>
+::interpolate_from_recv (comm_state<D>& state,
+ const vector<const gdata*> srcs,
+ const vector<CCTK_REAL> times,
+ const ibbox& box, const CCTK_REAL time,
+ const int order_space,
+ const int order_time)
+{
+ assert (has_storage());
+ assert (all(box.lower()>=extent().lower()));
+ assert (all(box.upper()<=extent().upper()));
+ assert (all(box.stride()==extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0));
+ assert (srcs.size() == times.size() && srcs.size()>0);
+ for (int t=0; t<(int)srcs.size(); ++t) {
+ assert (srcs.at(t)->has_storage());
+ assert (all(box.lower()>=srcs.at(t)->extent().lower()));
+ assert (all(box.upper()<=srcs.at(t)->extent().upper()));
+ }
+
+ assert (! box.empty());
+ if (box.empty()) return;
+
+ if (proc() == srcs.at(0)->proc()) {
+ // interpolate on same processor
+
+ } else {
+ // interpolate from other processor
+
+ gdata<D>* const tmp = make_typed(varindex, transport_operator);
+ // TODO: is this efficient?
+ state.tmps.push_back (tmp);
+ ++state.current;
+ tmp->allocate (box, srcs.at(0)->proc());
+ tmp->change_processor_recv (proc());
+
+ }
+}
+
+
+
+template<int D>
+void gdata<D>
+::interpolate_from_send (comm_state<D>& state,
+ const vector<const gdata*> srcs,
+ const vector<CCTK_REAL> times,
+ const ibbox& box, const CCTK_REAL time,
+ const int order_space,
+ const int order_time)
+{
+ assert (has_storage());
+ assert (all(box.lower()>=extent().lower()));
+ assert (all(box.upper()<=extent().upper()));
+ assert (all(box.stride()==extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0));
+ assert (srcs.size() == times.size() && srcs.size()>0);
+ for (int t=0; t<(int)srcs.size(); ++t) {
+ assert (srcs.at(t)->has_storage());
+ assert (all(box.lower()>=srcs.at(t)->extent().lower()));
+ assert (all(box.upper()<=srcs.at(t)->extent().upper()));
+ }
+
+ assert (! box.empty());
+ if (box.empty()) return;
+
+ if (proc() == srcs.at(0)->proc()) {
+ // interpolate on same processor
+
+ interpolate_from_nocomm (srcs, times, box, time, order_space, order_time);
+
+ } else {
+ // interpolate from other processor
+
+ gdata<D>* const tmp = state.tmps.at(state.current++);
+ assert (tmp);
+ tmp->interpolate_from_nocomm (srcs, times, box, time, order_space, order_time);
+ tmp->change_processor_send (proc());
+
+ }
+}
+
+
+
+template<int D>
+void gdata<D>
+::interpolate_from_wait (comm_state<D>& state,
+ const vector<const gdata*> srcs,
+ const vector<CCTK_REAL> times,
+ const ibbox& box, const CCTK_REAL time,
+ const int order_space,
+ const int order_time)
+{
+ assert (has_storage());
+ assert (all(box.lower()>=extent().lower()));
+ assert (all(box.upper()<=extent().upper()));
+ assert (all(box.stride()==extent().stride()));
+ assert (all((box.lower()-extent().lower())%box.stride() == 0));
+ assert (srcs.size() == times.size() && srcs.size()>0);
+ for (int t=0; t<(int)srcs.size(); ++t) {
+ assert (srcs.at(t)->has_storage());
+ assert (all(box.lower()>=srcs.at(t)->extent().lower()));
+ assert (all(box.upper()<=srcs.at(t)->extent().upper()));
+ }
+
+ assert (! box.empty());
+ if (box.empty()) return;
+
+ if (proc() == srcs.at(0)->proc()) {
+ // interpolate on same processor
+
+ } else {
+ // interpolate from other processor
+
+ gdata<D>* const tmp = state.tmps.at(state.current++);
+ assert (tmp);
+ tmp->change_processor_wait (proc());
+ copy_from_nocomm (tmp, box);
+ delete tmp;
+
+ }
+}
+
+
+
+template class comm_state<3>;
+template class gdata<3>;
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-13 04:15:33 +0000