diff options
-rw-r--r-- | interface.ccl | 12 | ||||
-rw-r--r-- | schedule.ccl | 7 | ||||
-rw-r--r-- | src/CollectData1D.c | 279 | ||||
-rw-r--r-- | src/GetLocalLine.c | 43 | ||||
-rw-r--r-- | src/Hyperslab.h | 14 | ||||
-rw-r--r-- | src/Misc.c | 29 | ||||
-rw-r--r-- | src/TestSlab.c | 74 | ||||
-rw-r--r-- | src/make.code.defn | 2 |
8 files changed, 308 insertions, 152 deletions
diff --git a/interface.ccl b/interface.ccl index 9f47f5b..5c9df1b 100644 --- a/interface.ccl +++ b/interface.ccl @@ -3,3 +3,15 @@ implements: hyperslab + +public: + +real testgf type = GF +{ + tx, + ty, + tz, +} "3D Cartesian test GFs" + + + diff --git a/schedule.ccl b/schedule.ccl index e35b24c..026b76e 100644 --- a/schedule.ccl +++ b/schedule.ccl @@ -1,6 +1,13 @@ # Schedule definitions for thorn Hyperslab # $Header$ +STORAGE: testgf + +schedule InitTestGF at BASEGRID +{ + LANG:C +} "Init test GFs" + schedule TestSlab at INITIAL { LANG:C diff --git a/src/CollectData1D.c b/src/CollectData1D.c index c525746..3d006d1 100644 --- a/src/CollectData1D.c +++ b/src/CollectData1D.c @@ -7,6 +7,7 @@ #include "cctk_Parameters.h" #include "cctk_Arguments.h" +#include "Hyperslab.h" #include "CactusBase/IOUtil/src/ioGH.h" #include "CactusPUGH/PUGH/src/include/pugh.h" @@ -14,185 +15,263 @@ #define TAGBASE 1234 #define MAX_DIM 3 + int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim, - int *S_l, int *u_l, int ds, int len_l, + int *S_l, int *u_l, int *ds, int len_l, void **dptr, int *dsize) + { void *loc_dptr; - int loc_dcount=0; + int loc_dcount[1]; void *all_dptr; int all_dcount=0; - int *rem_dcount; + int *rem_dcount; int vtype, vtypesize; + int iproc,myproc,nprocs; int vmpi_type; int *displs; - int idim, i,j,k, ip; + int idim, i,j,k; int ierr; pGH *pughGH; - pughGH = PUGH_pGH (GH); - myproc = CCTK_MyProc(GH); + vtype = CCTK_VarTypeI(vindex); + vtypesize= CCTK_VarTypeSize(vtype); + + pughGH = PUGH_pGH(GH); + myproc = CCTK_MyProc(GH); nprocs = CCTK_nProcs(GH); - vmpi_type= PUGH_MPI_REAL; + vmpi_type= CCTKtoMPItype(vtype); - vtype = CCTK_GroupTypeFromVarI(vindex); - vtypesize= CCTK_VarTypeSize(vtype); + rem_dcount=(int*) malloc (nprocs * sizeof (int)); + displs =(int*) malloc (nprocs * sizeof (int)); - rem_dcount=(CCTK_INT *) malloc (nprocs * sizeof (CCTK_INT)); - - ierr = CollectLocalData1D(GH, vindex, vtype, vtypesize, vtimelvl, - vdim, S_l, u_l, len_l, - &loc_dptr, &loc_dcount); - - - /* Send local number of data elements */ - CACTUS_MPI_ERROR (MPI_Gather (&loc_dcount, 1, vmpi_type, - rem_dcount, 1, PUGH_MPI_INT, 0, pughGH->PUGH_COMM_WORLD)); + printf("Calling CollectLocal ... "); - - /* Calculate the displacements */ - displs[0] = 0; - all_dcount= rem_dcount[0]; + + ierr = CollectLocalData1D(GH, vindex, vtimelvl, vdim, + S_l, u_l, ds, len_l, + &loc_dptr, loc_dcount); + + /* Send local number of data elements */ +#ifdef CCTK_MPI + CACTUS_MPI_ERROR (MPI_Gather (loc_dcount, 1, PUGH_MPI_INT, + rem_dcount, 1, PUGH_MPI_INT, 0, pughGH->PUGH_COMM_WORLD)); +#endif + displs[0] = 0; + all_dcount = rem_dcount[0]; for (iproc=1;iproc<nprocs;iproc++) { displs[iproc] = displs[iproc-1] + rem_dcount[iproc-1]; - all_dcount += rem_dcount[ip]; + all_dcount += rem_dcount[iproc]; } + dsize[0] = all_dcount; + for (iproc=0;iproc<nprocs;iproc++) + printf(" DISPLS[%d] rem: %d displs: %d all %d \n", + iproc, rem_dcount[iproc],displs[iproc],all_dcount); + fflush(stdout); - all_dptr = malloc (vtypesize*all_dcount); - - CACTUS_MPI_ERROR (MPI_Gatherv(loc_dptr, loc_dcount, vmpi_type, - all_dptr, rem_dcount, displs, vmpi_type, + /*$dptr= malloc(all_dcount * vtypesize);$*/ + + i = loc_dcount[0]; + { + + void *st = malloc(all_dcount * vtypesize); + st = dptr; + CACTUS_MPI_ERROR (MPI_Gatherv(loc_dptr, i , vmpi_type, + st, rem_dcount, displs, vmpi_type, 0, pughGH->PUGH_COMM_WORLD)); + } + + printf(" CollectData1D done \n"); return(0); - - } int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim, - int *S_l, int *u_l, int ds, int len_l, + int *S_l, int *u_l, int *ds, int len_l, void **loc_dptr, int *dsize) { int *locstart, *locend, nlocpoints; int ipnt, gridpnt[MAX_DIM]; int ierr; - - int idim,l; - int vmpi_type; - int vtype, vtypesize; - - void *data = CCTK_VarDataPtrI (GH, vtimelvl, vindex); - - ioGH *ioUtilGH; - - ioUtilGH = (ioGH *) GH->extensions [CCTK_GHExtensionHandle ("IO")]; - - vtype = CCTK_GroupTypeFromVarI(vindex); - vtypesize= CCTK_VarTypeSize(vtype); - - switch (vtype) { - case CCTK_VARIABLE_CHAR: - vtypesize = sizeof (CCTK_CHAR); - vmpi_type = PUGH_MPI_CHAR; - break; + int gindex,vtype,vtypesize; + int idim,lincount; - case CCTK_VARIABLE_INT: - vtypesize = sizeof (CCTK_INT); - vmpi_type = PUGH_MPI_INT; - break; + void *data = CCTK_VarDataPtrI (GH, vtimelvl, vindex); - case CCTK_VARIABLE_REAL: - vtypesize = ioUtilGH->out_single ? sizeof(CCTK_REAL4):sizeof (CCTK_REAL); - vmpi_type = ioUtilGH->out_single ? PUGH_MPI_REAL4 : PUGH_MPI_REAL; - break; - default: - CCTK_WARN (1, "Unsupported variable type in IOFlexIO_Write2D"); - return(-1); - } + vtype = CCTK_VarTypeI(vindex); + vtypesize = CCTK_VarTypeSize(vtype); locstart = (int*)malloc(vdim*sizeof(int)); locend = (int*)malloc(vdim*sizeof(int)); - ierr = GetLocalLine(GH, vdim, S_l, u_l, ds, len_l, locstart, locend, &nlocpoints); + ierr = GetLocalLine(GH, vdim, S_l, u_l, ds, len_l, + locstart, locend, &nlocpoints); - printf("CollectLocal1D: %d npoints\n",nlocpoints); + dsize[0]= nlocpoints; - *loc_dptr= malloc(nlocpoints * vtypesize); - - dsize[0] = nlocpoints; + printf("CollectLocal1D: %d npoints lb: %d %d %d\n", + nlocpoints,GH->cctk_lbnd[0],GH->cctk_lbnd[1],GH->cctk_lbnd[2]); - l=0; + *loc_dptr= malloc(nlocpoints * vtypesize); - /* zero out gridpnt[] for unused dimensions */ - for (idim = 0; idim < MAX_DIM; idim++) - gridpnt [idim] = 0; + lincount=0; /* CCTK_VARIABLE_INT */ if (vtype == CCTK_VARIABLE_INT) { CCTK_INT *int_dptr = (CCTK_INT *) *loc_dptr; - for (ipnt=0; ipnt<=nlocpoints; ipnt++) - { - for (idim=0;idim<vdim;idim++) - gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; - int_dptr[l++]=((CCTK_INT *) data) - [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + if (vdim==3) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + int_dptr[lincount++]=((CCTK_INT *) data) + [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + } + } + else if (vdim==2) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + int_dptr[lincount++]=((CCTK_INT *) data) + [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])]; + } } + else if (vdim==1) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + int_dptr[lincount++]=((CCTK_INT *) data) + [CCTK_GFINDEX1D(GH,gridpnt[0])]; + } + } + else return(-1); } /* CCTK_VARIABLE_CHAR */ else if (vtype == CCTK_VARIABLE_CHAR) { CCTK_CHAR *char_dptr = (CCTK_CHAR *) *loc_dptr; - for (ipnt=0; ipnt<=nlocpoints; ipnt++) - { - for (idim=0;idim<vdim;idim++) - gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; - char_dptr[l++]=((CCTK_CHAR *) data)[CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + if (vdim==3) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + char_dptr[lincount++]=((CCTK_CHAR *) data) + [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + } + } + else if (vdim==2) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + char_dptr[lincount++]=((CCTK_CHAR *) data) + [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])]; + } + } + else if (vdim==1) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + char_dptr[lincount++]=((CCTK_CHAR *) data) + [CCTK_GFINDEX1D(GH,gridpnt[0])]; + } } } /* CCTK_VARIABLE_REAL */ - else if (vtype == CCTK_VARIABLE_REAL && !ioUtilGH->out_single) + else if (vtype == CCTK_VARIABLE_REAL) { CCTK_REAL *real_dptr = (CCTK_REAL *) *loc_dptr; - - for (ipnt=0; ipnt<=nlocpoints; ipnt++) - { - for (idim=0;idim<vdim;idim++) - gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; - real_dptr[l++]=((CCTK_REAL *) data) - [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + if (vdim==3) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + printf(" nlocpoints: %d \n",nlocpoints); + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + printf(" gridpnts: ( %d %d %d ) ", + gridpnt[0],gridpnt[1],gridpnt[2]); + + real_dptr[lincount++]=((CCTK_REAL *) data) + [CCTK_GFINDEX3D(GH,gridpnt[0]-1,gridpnt[1]-1,gridpnt[2]-1)]; + printf(" Value: %f \n",real_dptr[lincount-1]); + } + } + else if (vdim==2) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + real_dptr[lincount++]=((CCTK_REAL *) data) + [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])]; + } + } + else if (vdim==1) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + real_dptr[lincount++]=((CCTK_REAL *) data) + [CCTK_GFINDEX1D(GH,gridpnt[0])]; + } } } /* CCTK_VARIABLE_REAL4 */ - else if (vtype == CCTK_VARIABLE_REAL4 && ioUtilGH->out_single) + else if (vtype == CCTK_VARIABLE_REAL4) { CCTK_REAL4 *real4_dptr = (CCTK_REAL4*) *loc_dptr; - - for (ipnt=0; ipnt<=nlocpoints; ipnt++) - { - for (idim=0;idim<vdim;idim++) - gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; - - real4_dptr[l++]=((CCTK_REAL4 *) data) - [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + if (vdim==3) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + + real4_dptr[lincount++]=((CCTK_REAL4 *) data) + [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])]; + } + } + else if (vdim==2) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + + real4_dptr[lincount++]=((CCTK_REAL4 *) data) + [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])]; + } + } + else if (vdim==1) { + for (ipnt=0; ipnt<nlocpoints; ipnt++) + { + for (idim=0;idim<vdim;idim++) + gridpnt[idim] = locstart[idim]+ipnt * u_l[idim]; + + real4_dptr[lincount++]=((CCTK_REAL4 *) data) + [CCTK_GFINDEX1D(GH,gridpnt[0])]; + } } } -} + free(locstart); + free(locend); +} + diff --git a/src/GetLocalLine.c b/src/GetLocalLine.c index c0b1ef3..7f7d668 100644 --- a/src/GetLocalLine.c +++ b/src/GetLocalLine.c @@ -1,29 +1,3 @@ - - -/* - A B - <-----------------> - |_____|_____|_____|_____|_____|_____|_____| - 0 1 2 3 4 5 6 #proc - - These are the case we have to deal with. For downsampling we need - to calulate the offset that a proc needs to start on gridpoint, that - is actually reached by downsampling. Eg. see the "X" below. - where downsampl. = 4, start at global index 2, 3 procs, shown with - ghostzones, where they overlap. - 1 1 1 1 1 1 1 1 1 1 - globalIDX: 0 1 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 - - localIDX: - proc0/2: 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 - proc1: 0 1 2 3 4 5 6 7 - downsampl: X X X X - All the stuff below is about starting at "4" on proc one and - not 0 or 1 or what else. (0 on proc 2 is a ghost). - -*/ - - #include <stdio.h> #include <stdlib.h> #include <math.h> @@ -32,6 +6,8 @@ #include "cctk_Parameters.h" #include "cctk_Arguments.h" +#include "Hyperslab.h" + #define MAX_DIM 3 #define MAX_FACE 6 #define EPS 1e-10 @@ -72,7 +48,7 @@ int IsPointOnProc_REAL(int dim, int *gownership, CCTK_REAL *pnt) } int GetLocalLine(cGH *GH, - int vardim, int *S_l, int *u_l, int *ds, int len_l, + int vardim, int *S_l, int *u_l, int *ds, int len_l, int *locstart, int *locend, int *nlocpoints) { /* variables for proc. surfaces */ @@ -313,8 +289,8 @@ int GetLocalLine(cGH *GH, nlocpoints[0]= ABS(lmd[0]-lmd[1])+1; for (idim=0;idim<vardim;idim++) { - locstart[idim] = S_l[idim]+lmd[0]*u_l[idim]; - locend[idim] = S_l[idim]+lmd[1]*u_l[idim]; + locstart[idim] = (S_l[idim]+lmd[0]*u_l[idim]); + locend[idim] = (S_l[idim]+lmd[1]*u_l[idim]); } } else @@ -330,6 +306,15 @@ int GetLocalLine(cGH *GH, printf("\n>>>>> GLOBAL START/END: (%d %d %d) -> (%d %d %d) = %d\n", locstart[0],locstart[1],locstart[2], locend[0],locend[1],locend[2],nlocpoints[0]); + + for (idim=0;idim<vardim;idim++) + { + locstart[idim] -= GH->cctk_lbnd[idim]; + locend[idim] -= GH->cctk_lbnd[idim]; + } + printf(">>>>> LOCAL START/END : (%d %d %d) -> (%d %d %d) = %d\n", + locstart[0],locstart[1],locstart[2], + locend[0],locend[1],locend[2],nlocpoints[0]); return(0); diff --git a/src/Hyperslab.h b/src/Hyperslab.h new file mode 100644 index 0000000..599c0c3 --- /dev/null +++ b/src/Hyperslab.h @@ -0,0 +1,14 @@ + + +int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim, + int *S_l, int *u_l, int *ds, int len_l, + void **dptr, int *dsize); + +int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim, + int *S_l, int *u_l, int *ds, int len_l, + void **loc_dptr, int *dsize); + +int GetLocalLine(cGH *GH, + int vardim, int *S_l, int *u_l, int *ds, int len_l, + int *locstart, int *locend, int *nlocpoints); + diff --git a/src/Misc.c b/src/Misc.c new file mode 100644 index 0000000..9c77b8c --- /dev/null +++ b/src/Misc.c @@ -0,0 +1,29 @@ + + +#include "CactusPUGH/PUGH/src/include/pugh.h" + + +int CCTKtoMPItype(int cctktype) +{ + int mpitype; + switch (cctktype) { + + case CCTK_VARIABLE_CHAR: + mpitype = PUGH_MPI_CHAR; + break; + + case CCTK_VARIABLE_INT: + mpitype = PUGH_MPI_INT; + break; + + case CCTK_VARIABLE_REAL: + mpitype = PUGH_MPI_REAL; + break; + + default: + printf("Unsupported MPI variable type in Hyperslab\n"); + mpitype = -1; + } + + return(mpitype); +} diff --git a/src/TestSlab.c b/src/TestSlab.c index c812cb3..2a3b966 100644 --- a/src/TestSlab.c +++ b/src/TestSlab.c @@ -6,6 +6,8 @@ #include "cctk.h" #include "cctk_Parameters.h" #include "cctk_Arguments.h" + +#include "Hyperslab.h" #include "CactusPUGH/PUGH/src/include/pugh.h" #include "CactusBase/IOUtil/src/ioGH.h" @@ -16,46 +18,74 @@ #define ABS(a) ((a)<0 ? -(a) : (a)) -int IsPointOnProc(int dim, int *gownership, int *pnt); -int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim, - int *S_l, int *u_l, int ds, int len_l, - void **loc_dptr, int *dsize); +void InitTestGF(CCTK_ARGUMENTS) +{ + DECLARE_CCTK_ARGUMENTS + int i,j,k; + + + for(i=0; i < cctk_lsh[0]; i++) + { + for(j=0; j < cctk_lsh[1]; j++) + { + for(k=0; k < cctk_lsh[2]; k++) + { + tx[CCTK_GFINDEX3D(cctkGH,i,j,k)] = i; + ty[CCTK_GFINDEX3D(cctkGH,i,j,k)] = j; + tz[CCTK_GFINDEX3D(cctkGH,i,j,k)] = k; + } + } + } +} -int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim, - int *S_l, int *u_l, int ds, int len_l, - void **loc_dptr, int *dsize); void TestSlab(CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS - int gs[3], di[3], ds; /* global start/direction/downsampling*/ + int gs[3], di[3], ds[1]; /* global start/direction/downsampling*/ int ls[3], le[3]; /* local start/end */ - int nlocpoints; /* number of points */ + int dsize[1]; /* number of points */ int nprocs,iproc; /* number of procs */ - int *c_nlocpoints; /* collected number of points per proc (size nproc) */ - pGH *pughGH; - int vindex, gindex; - int vdim, vtimelvl; - void *loc_dptr; - vindex = CCTK_VarIndex("grid::x"); + int l; + + + int vindex, vdim, vtimelvl; + CCTK_REAL *dptr; + + /*$vindex = CCTK_VarIndex("hyperslab::tz");$*/ + vindex = CCTK_VarIndex("grid::z"); vtimelvl = CCTK_NumTimeLevelsFromVarI (vindex) - 1; - vdim = CCTK_GroupDimI(gindex); + vdim = CCTK_GroupDimFromVarI(vindex); - gs[0]=1; + /*$gs[0]=1; gs[1]=5; gs[2]=9; di[0]= 0; di[1]=-1; - di[2]=-2; - - ds=1; + di[2]=-2;$*/ + + gs[0]=1; + gs[1]=1; + gs[2]=1; + + di[0]= 0; + di[1]= 0; + di[2]= 1; + + + ds[0]=1; - CollectLocalData1D(cctkGH, vindex, vtimelvl, vdim, - gs, di, ds, -1, &loc_dptr, &nlocpoints); + printf("Calling CollectData \n"); + CollectData1D(cctkGH, vindex, vtimelvl, vdim, + gs, di, ds, 10, + &dptr, dsize); + printf("TestSlab: %d \n",dsize[0]); + for (l=0;l<dsize[0];l++) + printf("%f \n",dptr[l]); } diff --git a/src/make.code.defn b/src/make.code.defn index f7f22ae..77b30bc 100644 --- a/src/make.code.defn +++ b/src/make.code.defn @@ -2,7 +2,7 @@ # $Header$ # Source files in this directory -SRCS = GetLocalLine.c TestSlab.c CollectData1D.c +SRCS = GetLocalLine.c TestSlab.c CollectData1D.c Misc.c # Subdirectories containing source files SUBDIRS = |