1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
/*@@
@file pGF_FinishRecv.c
@date Thu Apr 3 11:37:28 1997
@author Paul Walker
@desc
The routine which finalize the MPI recieves for a grid
function. Critically linked with @seefile pGF_PostRecv.c
and @seefile pGF_PostSend.c
@enddesc
@version $Header$
@@*/
#include <stdio.h>
#include "cctk.h"
#include "pugh.h"
void pGF_FinishRecv3(pGH *GH,pGF *GF,int dir);
void pGF_FinishRecv1(pGH *GH,pGF *GF,int dir);
static char *rcsid = "$Header$";
/*@@
@routine pGF_FinishRecv
@date Thu Apr 3 11:38:07 1997
@author Paul Walker
@desc
This routine finalizes the MPI communication through a face.
It is crucially linked with @seeroutine pGF_PostRecv and
@seeroutine pGF_PostSend.
<p>
<b>Important!</b>
This routine does <b>not</b> wait on the recieves.
Before it is called, you must do the MPI_Wait or
else you will not get the right answer. for an
example of this, see @seeroutine SyncGroupGF or
@seeroutine SyncGroupGF
@enddesc
@calledby SyncGroupGF
@history
@hdate Nov 4 1998 @hauthor Gabrielle Allen
@hdesc Allow for forced synchronization of all GFs with storage
@endhistory
@@*/
void pGF_FinishRecv(pGH *GH, pGF *GF, int dir)
{
#ifdef MPI
/* Return if GF has no storage */
if (!(GF->storage)) return;
/* Return if communication not required and no forced synchronisation */
if (!(GH->forceSync || GF->docomm[dir])) return;
if (GH->neighbors[GH->myproc][dir] >= 0)
{
/* Here wait one at a time, so the others can arrive while
we copy the data back in... */
#ifdef DEBUG_PRINT
printf ("FINISHRECV: GF %s Side %d Proc %d request %d\n",
GF->name,dir,GH->myproc,GF->rreq[dir]);
#endif
if (GH->dim == 3)
{
pGF_FinishRecv3(GH,GF,dir);
}
else if (GH->dim == 1)
{
pGF_FinishRecv1(GH,GF,dir);
}
else
{
CCTK_WARN(0,"PUGH does not support this dimension grid");
}
}
#endif
}
void pGF_FinishRecv3(pGH *GH,pGF *GF,int dir)
{
int istart,iend,jstart,jend,kstart,kend;
int ii,jj,kk,xx;
CCTK_CHAR *char_data;
CCTK_INT *int_data;
CCTK_REAL *real_data;
CCTK_COMPLEX *complex_data;
/* Copy buffer onto GF Storage */
istart = GH->ghosts[GF->stagger][0][dir][0];
iend = GH->ghosts[GF->stagger][1][dir][0];
jstart = GH->ghosts[GF->stagger][0][dir][1];
jend = GH->ghosts[GF->stagger][1][dir][1];
kstart = GH->ghosts[GF->stagger][0][dir][2];
kend = GH->ghosts[GF->stagger][1][dir][2];
/* Great now copy. Note ordering is just link in PostSend */
xx=0;
switch (GF->vtype)
{
case CCTK_VARIABLE_CHAR:
char_data = (CCTK_CHAR *) GF->data;
for (kk=kstart; kk<kend; kk++)
{
for (jj=jstart; jj<jend; jj++)
{
for (ii=istart; ii<iend; ii++)
{
char_data [DATINDEX (GH,ii,jj,kk)] =
((CCTK_CHAR *) GF->recv_buffer[dir]) [xx++];
}
}
}
break;
case CCTK_VARIABLE_INT:
int_data = (CCTK_INT *) GF->data;
for (kk=kstart; kk<kend; kk++)
{
for (jj=jstart; jj<jend; jj++)
{
for (ii=istart; ii<iend; ii++)
{
int_data [DATINDEX (GH,ii,jj,kk)] =
((CCTK_INT *) GF->recv_buffer[dir]) [xx++];
}
}
}
break;
case CCTK_VARIABLE_REAL:
real_data = (CCTK_REAL *) GF->data;
for (kk=kstart; kk<kend; kk++)
{
for (jj=jstart; jj<jend; jj++)
{
for (ii=istart; ii<iend; ii++)
{
real_data [DATINDEX (GH,ii,jj,kk)] =
((CCTK_REAL *) GF->recv_buffer[dir]) [xx++];
}
}
}
break;
case CCTK_VARIABLE_COMPLEX:
complex_data = (CCTK_COMPLEX *) GF->data;
for (kk=kstart; kk<kend; kk++)
{
for (jj=jstart; jj<jend; jj++)
{
for (ii=istart; ii<iend; ii++)
{
complex_data [DATINDEX (GH,ii,jj,kk)] =
((CCTK_COMPLEX *) GF->recv_buffer[dir]) [xx++];
}
}
}
break;
default:
CCTK_WARN (1, "Unsupported variable type in pGF_FinishRecv3");
return;
}
}
void pGF_FinishRecv1(pGH *GH,pGF *GF,int dir)
{
int istart,iend;
int ii,xx;
CCTK_CHAR *char_data;
CCTK_INT *int_data;
CCTK_REAL *real_data;
CCTK_COMPLEX *complex_data;
/* Copy buffer onto GF Storage */
istart = GH->ghosts[GF->stagger][0][dir][0];
iend = GH->ghosts[GF->stagger][1][dir][0];
/* Great now copy. Note ordering is just link in PostSend */
xx=0;
switch (GF->vtype)
{
case CCTK_VARIABLE_CHAR:
char_data = (CCTK_CHAR *) GF->data;
for (ii=istart; ii<iend; ii++)
{
char_data [ii] =
((CCTK_CHAR *) GF->recv_buffer[dir]) [xx++];
}
break;
case CCTK_VARIABLE_INT:
int_data = (CCTK_INT *) GF->data;
for (ii=istart; ii<iend; ii++)
{
int_data [ii] =
((CCTK_INT *) GF->recv_buffer[dir]) [xx++];
}
break;
case CCTK_VARIABLE_REAL:
real_data = (CCTK_REAL *) GF->data;
for (ii=istart; ii<iend; ii++)
{
real_data [ii] =
((CCTK_REAL *) GF->recv_buffer[dir]) [xx++];
}
break;
case CCTK_VARIABLE_COMPLEX:
complex_data = (CCTK_COMPLEX *) GF->data;
for (ii=istart; ii<iend; ii++)
{
complex_data [ii] =
((CCTK_COMPLEX *) GF->recv_buffer[dir]) [xx++];
}
break;
default:
CCTK_WARN (1, "Unsupported variable type in pGF_FinishRecv3");
return;
}
}
|
