1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
/*@@
@file PostReceiveGA.c
@date Wed Feb 2 11:28:06 1997
@author Gabrielle Allen
@desc
Routines which post all the IRecv commands for a single sync.
These allow the asyncronous model proposed in, for example,
@seeroutine PUGH_Sync to go!
@enddesc
@version $Header$
@@*/
/* #define DEBUG_PUGH 1 */
#include <stdio.h>
#include "pugh.h"
static const char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_PostReceiveGA_c);
#ifdef CCTK_MPI
void PostReceiveGA(pGH *pughGH, int dir, pComm *comm);
/*@@
@routine PostReceiveGA
@date Thu Apr 3 12:10:46 1997
@author Paul Walker
@desc
This routine posts a recieve (MPI_Irecv) of the buffer
we want to get the send from another processor, which
will be sent by @seeroutine PostSendGA and then
finalized by @seeroutoine FinishRecvGA.
<p>
Aside from a silly calculation to get a unique tag
based on neigbors and processors, this is a very
straightforward routine.
@enddesc
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GAs with storage
@endhistory
@@*/
void PostReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
pGA *GA;
int rtag;
int neighbour;
GA = (pGA *) pughGH->variables[comm->first_var][0];
/* return if no storage assigned */
if (! GA->storage)
{
CCTK_VWarn(2, __LINE__, __FILE__, CCTK_THORNSTRING,
"Trying to synchronize variable '%s' with no storage", GA->name);
return;
}
/* return if communication not required and no forced synchronisation */
if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
/* return if there is no neighbour in the given direction */
neighbour = GA->connectivity->neighbours[pughGH->myproc][dir];
if (neighbour < 0)
{
return;
}
/* note this is the complement of the stag set in PostSendGA */
rtag = 1000 + dir + 2 * (pughGH->myproc + pughGH->nprocs * GA->id);
/* mod the tag to force MPI compliance */
rtag = rtag % 32768;
#ifdef DEBUG_PUGH
printf ("PostReceiveGA: into direction %d from proc %d with rtag %d size %d "
"for %d vars starting from '%s'\n",
dir, neighbour, rtag, comm->buffer_sz[dir], comm->n_vars, GA->name);
#endif
if (pughGH->commmodel == PUGH_ALLOCATEDBUFFERS)
{
CACTUS_MPI_ERROR (MPI_Irecv (comm->recv_buffer[dir],
comm->buffer_sz[dir], comm->mpi_type,
neighbour, rtag,
pughGH->PUGH_COMM_WORLD, &comm->rreq[dir]));
}
#ifdef PUGH_WITH_DERIVED_DATATYPES
else if (pughGH->commmodel == PUGH_DERIVEDTYPES)
{
int var;
MPI_Datatype *recv_dt;
switch (GA->vtype)
{
case CCTK_VARIABLE_BYTE:
recv_dt = pughGH->recv_char_dt;
break;
case CCTK_VARIABLE_INT:
recv_dt = pughGH->recv_int_dt;
break;
case CCTK_VARIABLE_REAL:
recv_dt = pughGH->recv_real_dt;
break;
case CCTK_VARIABLE_COMPLEX:
recv_dt = pughGH->recv_complex_dt;
break;
default:
CCTK_WARN (1, "Unknown variable type in PostReceiveGA");
return;
}
for (var = comm->first_var; var < comm->first_var + comm->n_vars; var++)
{
pGA *GA = (pGA *) pughGH->variables[var][comm->sync_timelevel];
CACTUS_MPI_ERROR (MPI_Irecv (GA->data,
1, recv_dt[dir],
neighbour, (rtag + var) % 32768,
pughGH->PUGH_COMM_WORLD,
&GA->comm->rreq[dir]));
}
}
#endif
}
#endif /* CCTK_MPI */
|
