1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
#ifndef COMMSTATE_HH
#define COMMSTATE_HH
#include <cctk.h>
#include <cctk_Parameters.h>
#include <cstdlib>
#include <iostream>
#include <vector>
#ifdef CCTK_MPI
# include <mpi.h>
#else
# include "nompi.h"
#endif
#include "dist.hh"
#include "timestat.hh"
using namespace std;
using namespace CarpetLib;
// State information for communications
// A comm state object will step through the state transitions in the
// given order:
enum astate {
state_get_buffer_sizes,
state_fill_send_buffers,
state_do_some_work,
state_empty_recv_buffers,
state_done
};
char const * tostring (astate const & thestate);
inline ostream& operator<< (ostream& os, astate const & thestate)
{
return os << tostring(thestate);
}
struct comm_state {
astate thestate;
comm_state ();
void step ();
bool done () const;
~comm_state ();
private:
// Forbid copying and passing by value
comm_state (comm_state const &);
comm_state& operator= (comm_state const &);
// structure describing a per-process buffer
struct procbufdesc {
// allocated communication buffers
vector<char> sendbufbase;
vector<char> recvbufbase;
// sizes of the communication buffers (in elements of type <datatype>)
size_t sendbufsize;
size_t recvbufsize;
// pointers to step through the communication buffers
// (these get advanced by the routines which fill/empty the buffers)
char* sendbuf;
char* recvbuf;
bool did_post_send;
bool did_post_recv;
// constructor for an instance of this structure
procbufdesc() :
sendbufsize(0), recvbufsize(0),
sendbuf(NULL), recvbuf(NULL),
did_post_send(false), did_post_recv(false)
{
}
};
// structure describing a collective communications buffer for a C datatype
struct typebufdesc {
// flag indicating whether this buffer is in use
bool in_use;
// the MPI datatype
MPI_Datatype mpi_datatype;
// the size of this datatype (in bytes)
int datatypesize;
// per-process buffers
vector<procbufdesc> procbufs; // [dist::size()]
// constructor for an instance of this structure
typebufdesc() :
in_use(false),
mpi_datatype(MPI_DATATYPE_NULL), datatypesize(0)
{
}
};
// datatype buffers
vector<typebufdesc> typebufs; // [type]
// outstanding requests for posted send/recv communications
vector<MPI_Request> srequests;
vector<MPI_Request> rrequests;
static inline
MPI_Request & push_back (vector<MPI_Request> & reqs)
{
reqs.push_back (MPI_REQUEST_NULL);
return reqs.back();
}
public:
void
reserve_send_space (unsigned type,
int proc,
int npoints);
void
reserve_recv_space (unsigned type,
int proc,
int npoints);
void *
send_buffer (unsigned type,
int proc,
int npoints);
void *
recv_buffer (unsigned type,
int proc,
int npoints);
void
commit_send_space (unsigned type,
int proc,
int npoints);
void
commit_recv_space (unsigned type,
int proc,
int npoints);
};
#endif // COMMSTATE_HH
|
