1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
#ifndef COMMSTATE_HH
#define COMMSTATE_HH
#include <cstdlib>
#include <queue>
#include <vector>
#include <mpi.h>
#include "cctk_Parameters.h"
#include "dist.hh"
#include "timestat.hh"
using namespace std;
using namespace CarpetLib;
// State information for communications
// A comm state object will step through the state transitions in the
// given order:
enum astate {
state_get_buffer_sizes,
state_fill_send_buffers,
state_do_some_work,
state_empty_recv_buffers,
state_done
};
struct comm_state {
astate thestate;
comm_state ();
void step ();
bool done ();
~comm_state ();
private:
// Forbid copying and passing by value
comm_state (comm_state const &);
comm_state& operator= (comm_state const &);
public:
//////////////////////////////////////////////////////////////////////////
// the following members are used for collective communications
//////////////////////////////////////////////////////////////////////////
public:
// structure describing a per-processor buffer for collective communications
struct procbufdesc {
// the allocated communication buffers
vector<char> sendbufbase;
vector<char> recvbufbase;
// the sizes of communication buffers (in elements of type <datatype>)
size_t sendbufsize;
size_t recvbufsize;
// pointers to step through the communication buffers
// (these get advanced by the routines which fill/empty the buffers)
char* sendbuf;
char* recvbuf;
// constructor for an instance of this structure
procbufdesc() : sendbufsize(0), recvbufsize(0),
sendbuf(NULL), recvbuf(NULL)
{
}
};
// structure describing a collective communications buffer for a C datatype
struct typebufdesc {
// flag indicating whether this buffer is in use
bool in_use;
// the size of this datatype (in bytes)
int datatypesize;
// the corresponding MPI datatype
MPI_Datatype mpi_datatype;
// per-processor buffers
vector<procbufdesc> procbufs; // [dist::size()]
// constructor for an instance of this structure
typebufdesc() : in_use(false), datatypesize(0),
mpi_datatype(MPI_DATATYPE_NULL)
{
}
};
// list of datatype buffers
vector<typebufdesc> typebufs; // [dist::c_ndatatypes()]
void
reserve_send_space (unsigned int type,
int proc,
int npoints);
void
reserve_recv_space (unsigned int type,
int proc,
int npoints);
void *
send_buffer (unsigned int type,
int proc,
int npoints);
void *
recv_buffer (unsigned int type,
int proc,
int npoints);
void
commit_send_space (unsigned int type,
int proc,
int npoints);
void
commit_recv_space (unsigned int type,
int proc,
int npoints);
private:
// lists of outstanding requests for posted send/recv communications
vector<MPI_Request> srequests; // [dist::size() * dist::c_ndatatypes()]
vector<MPI_Request> rrequests; // [dist::size() * dist::c_ndatatypes()]
// number of posted and already completed receive communications
int num_posted_recvs;
int num_completed_recvs;
// wait for completion of posted collective buffer sends/receives
bool AllPostedCommunicationsFinished();
};
#endif // COMMSTATE_HH
|
