1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
#ifndef __MPIO_HH_
#define __MPIO_HH_
#include "IO.hh"
#include "MPIutils.hh"
#include "Timer.H"
/* Simplifies assumptions about domain decomposition */
/*
Module: MPIO
Description:
Implements slice-based Parallel IO for IEEEIO libraries.
Uses iSend/iRecv of XY slices to hide message passing latency.
This is a simplified version of PIO which makes fewer assumptions
about the layout of the data.
Will eventually be subclassed from IObase so that you can
drop-in replace a single-threaded IO system with this
multithreaded one. writeChunk() will be used to announce
the local chunks and perform the write in parallel rather
than collecting serially.
Needs to have extensions to automatically remove ghost zones
setLocalDims(origin,dims,minindex,maxindex)
Limitations:
* currently only works for 3D grids
* only understands Float32 and Float64 datatypes
* Assumes HPF layout (eg, canonical processor layout where
all PE's are aligned to grid... no partial overlaps)
* Uses XY slices (assumes fortran ordering for dimensions)
* Collects only to a single file (cannot do a multi-gather).
*/
class MPIO /* : public IObase */ {
int *gdims,*gorigins,grank;
MPIcomm &comm;
int globaldims[3]; // for master
int localorigin[3],localdims[3];
int root,myid;
IObase *file;
Timer slicewait,slicewrite,slicecollect,sliceselect;
Timer sliceallwait;
int *chunkindex,*chunkindexcount,*slicemap,*procmap;
inline int isRoot(){ return ((myid == root)?1:0);}
// MPI_Request *slice_send_requests;
protected:
inline void sendSlice(int z,float *data,MPI_Request *req);
void requestSlice(int z,float *slicebuffer,MPI_Request *req);
void waitForSlice(int z,float *slicebuffer,float *destbuffer,MPI_Request *req);
public:
MPIO(IObase *io,MPIcomm &c);
~MPIO();
void setLocalDims(int rank,int *origin, int *dims);
virtual int write(IObase::DataType type,int rank,int *dims,void *data);
void write(float *data);
};
void MPIO::sendSlice(int z,float *data,MPI_Request *req){
int iz = z - localorigin[2];
if(iz<0 || iz>=localdims[2]) return;
//printf("sendslice %u\n",z);
comm.isSend(root,z,localdims[0]*localdims[1],
data+localdims[0]*localdims[1]*iz,req[iz]);
}
#endif // __MPIO_HH_
|
