1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/*@@
@file FinishReceiveGA.c
@date Wed Feb 2 11:37:28 1997
@author Gabrielle Allen
@desc
The routine which finalize the MPI recieves for a communication.
Critically linked with @seefile PostRecvGA.c and
@seefile PostSendGA.c
@enddesc
@version $Id$
@@*/
/*#define DEBUG_PUGH*/
#include <string.h>
#include "pugh.h"
static const char *rcsid = "$Header$";
CCTK_FILEVERSION(CactusPUGH_PUGH_FinishReceiveGA_c);
#ifdef CCTK_MPI
void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm);
/*@@
@routine FinishReceiveGA
@date Thu Apr 3 11:38:07 1997
@author Paul Walker
@desc
This routine finalizes the MPI communication through a face.
It is crucially linked with @seeroutine PostRecvGA and
@seeroutine PostSendGA.
<p>
<b>Important!</b>
This routine does <b>not</b> wait on the recieves.
Before it is called, you must do the MPI_Wait or
else you will not get the right answer. for an
example of this, see @seeroutine PUGH_DisableArrayGroupComm or
@seeroutine PUGH_Sync
@enddesc
@calledby PUGH_Sync
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GAs with storage
@endhistory
@@*/
void FinishReceiveGA(pGH *pughGH, int dir, pComm *comm)
{
pGA *GA;
int i, dim, copy_bytes, offset;
char *copy_from, *copy_to;
int *istart, *iend, *iterator;
GA = (pGA *) pughGH->variables[comm->first_var][0];
/* Return if GA has no storage */
if (! GA->storage)
{
return;
}
/* Return if communication not required and no forced synchronisation */
if (! (pughGH->forceSync || comm->docomm[dir]))
{
return;
}
if (GA->connectivity->neighbours[pughGH->myproc][dir] < 0)
{
return;
}
/* Here wait one at a time, so the others can arrive while
we copy the data back in... */
#ifdef DEBUG_PUGH
printf ("FinishReceiveGA: into direction %d from proc %d for %d vars "
"starting with '%s'\n",
dir, GA->connectivity->neighbours[pughGH->myproc][dir],
comm->n_vars, GA->name);
#endif
istart = GA->extras->ghosts[GA->stagger[dir>>1]][0][dir];
iend = GA->extras->ghosts[GA->stagger[dir>>1]][1][dir];
iterator = GA->extras->iterator;
/* set iterator to the start vector */
for(dim = 0; dim < GA->extras->dim; dim++)
{
iterator[dim] = istart[dim];
}
/* get the source and the number of bytes to copy in the lowest dimension */
copy_from = comm->recv_buffer[dir];
copy_bytes = (iend[0] - istart[0]) * GA->varsize;
/* now do the nested loops starting with the innermost */
dim = 1;
while (1)
{
/* check for end of current loop */
if (GA->extras->dim > 1 && iterator[dim] >= iend[dim])
{
/* increment outermost loopers */
for (dim++; dim < GA->extras->dim; dim++)
{
iterator[dim]++;
if (iterator[dim] < iend[dim])
{
break;
}
}
/* done if beyond outermost loop */
if (dim >= GA->extras->dim)
{
break;
}
/* reset innermost loopers */
for (dim--; dim > 0; dim--)
{
iterator[dim] = istart[dim];
}
dim = 1;
}
/* get the linear offset */
offset = istart[0];
for (i = 1; i < GA->extras->dim; i++)
{
offset += iterator[i] * GA->extras->hyper_volume[i];
}
offset *= GA->varsize;
/* copy the data in dimension 0 */
for (i = comm->first_var; i < comm->first_var + comm->n_vars; i++)
{
copy_to = ((pGA *) pughGH->variables[i][comm->sync_timelevel])->data;
memcpy (copy_to + offset, copy_from, copy_bytes);
copy_from += copy_bytes;
}
if (GA->extras->dim > 1)
{
/* increment current looper */
iterator[dim]++;
}
else
{
/* exit loop if array dim is 1D */
break;
}
} /* end of nested loops over all dimensions */
}
#endif /* CCTK_MPI */
|
