1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
#ifndef DISTRIBUTE_HH
#define DISTRIBUTE_HH
#include <cctk.h>
#include <defs.hh>
#include <vect.hh>
#include <list>
#include <mpi.h>
namespace CarpetIOF5 {
using namespace std;
// Fragment descriptor
struct fragdesc_t {
// only integer entries
int varindex;
static int const mglevel = 0;
int reflevel, map, src_component, timelevel;
ivect imin, imax; // upper bound is inclusive
// destination
int component, process;
int tag;
// meta-messages
enum state_t { state_normal, state_sent_all, state_all_sent_all };
int state;
fragdesc_t(): state(state_normal) {}
int num_ints() const { return sizeof(fragdesc_t) / sizeof(int); }
int npoints() const;
int vartypesize() const;
MPI_Datatype datatype() const;
};
// Scatter (distribute) Cactus variables
class scatter_t {
cGH const *const cctkGH;
static int const max_concurrent_sends = 100;
enum tags_t {
tag_desc,
tag_data_min,
tag_data_max = tag_data_min + max_concurrent_sends
};
class busy_tags_t {
vector<bool> tags;
public:
busy_tags_t();
~busy_tags_t();
int allocate_tag();
void free_tag(int tag);
};
vector<busy_tags_t> busy_tags;
struct transmission_t {
fragdesc_t fragdesc;
vector<char> data;
MPI_Request request;
transmission_t(): request(MPI_REQUEST_NULL) {}
~transmission_t() { assert(request==MPI_REQUEST_NULL); }
};
// Desired number of public receives to keep posted at all times
static int const num_public_recvs = 10;
list<transmission_t*> public_recvs, recvs, sends;
int num_received, num_sent;
ptrdiff_t bytes_allocated;
bool did_send_all;
int did_receive_sent_all;
bool did_receive_all_sent_all;
fragdesc_t to_parent, to_children;
public:
scatter_t(cGH const *cctkGH_);
~scatter_t();
void send(fragdesc_t const& fd, void const *data);
private:
void post_public_recvs();
void do_some_work(bool do_wait = false);
void maybe_send_did_send();
void send_all_did_send();
void set_did_send_all();
void set_did_receive_sent_all();
void set_did_receive_all_sent_all();
void handle_state_transition(fragdesc_t const& fd);
list<transmission_t*> split_for_sending(fragdesc_t const& fd,
void const *data);
void write_data(transmission_t *const tm);
};
} // namespace CarpetIOF5
#endif // #ifndef DISTRIBUTE_HH
|
