Fixed MPI communication stuff.

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGHSlab/trunk@12 10716dce-81a3-4424-a2c8-48026a0d3035

4 files changed, 153 insertions, 168 deletions

diff --git a/src/CollectData1D.c b/src/CollectData1D.c
index f731743..c8ab8de 100644
--- a/src/CollectData1D.c
+++ b/src/CollectData1D.c
@@ -17,9 +17,8 @@
 
 
 int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
-		  int *S_l, int *u_l, int *ds, int len_l, 
-		  void **dptr, int *dsize)
-
+                  int *S_l, int *u_l, int *ds, int len_l, 
+                  void **dptr, int *dsize, int proc)
 {
 
   void *loc_dptr;
@@ -28,80 +27,126 @@ int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
   void *all_dptr;
   int   all_dcount=0;
 
-  int  *rem_dcount;
-
   int vtype, vtypesize;
-
   int iproc,myproc,nprocs;
-  int vmpi_type;
-  int *displs;
+
+#ifdef CCTK_MPI
+  pGH *pughGH;   
+  CCTK_INT  tmp, *rem_dcount;
+
+  MPI_Datatype vmpi_type;
+  int *recvcnts, *displs;
+#endif
 
   int idim, i,j,k;
   int ierr;
 
-  pGH *pughGH;   
 
   vtype    = CCTK_VarTypeI(vindex);
   vtypesize= CCTK_VarTypeSize(vtype);
 
-  pughGH   = PUGH_pGH(GH);
   myproc   = CCTK_MyProc(GH); 
   nprocs   = CCTK_nProcs(GH);
   vmpi_type= CCTKtoMPItype(vtype);
 
-  rem_dcount=(int*) malloc (nprocs * sizeof (int));
-  displs    =(int*) malloc (nprocs * sizeof (int));
-
   printf("Calling CollectLocal ... ");  
 
 
   ierr = CollectLocalData1D(GH, vindex, vtimelvl, vdim, 
-			    S_l, u_l, ds, len_l, 
-			    &loc_dptr, loc_dcount);
+                            S_l, u_l, ds, len_l, 
+                            &loc_dptr, loc_dcount);
 
-  /* Send local number of data elements */
 #ifdef CCTK_MPI
-  CACTUS_MPI_ERROR (MPI_Gather (loc_dcount, 1, PUGH_MPI_INT,
-				rem_dcount, 1, PUGH_MPI_INT, 0, pughGH->PUGH_COMM_WORLD));
-#endif
-  displs[0]  = 0;
-  all_dcount = rem_dcount[0];
-  for (iproc=1;iproc<nprocs;iproc++)
+
+  pughGH   = PUGH_pGH(GH);
+
+  /* Send local number of data elements */
+  tmp = (CCTK_INT) loc_dcount [0];
+
+  if (proc < 0 || proc == myproc)
+  {
+    rem_dcount= (CCTK_INT *) malloc (nprocs * sizeof (CCTK_INT));
+    recvcnts  = (int *) malloc (nprocs * sizeof (int));
+    displs    = (int *) malloc (nprocs * sizeof (int));
+  }
+
+  if (proc < 0)
   {
-    displs[iproc] = displs[iproc-1] + rem_dcount[iproc-1];
-    all_dcount   += rem_dcount[iproc];
+    CACTUS_MPI_ERROR (MPI_Allgather (&tmp, 1, PUGH_MPI_INT,
+                                     rem_dcount, 1, PUGH_MPI_INT,
+                                     pughGH->PUGH_COMM_WORLD));
+  } else {
+    CACTUS_MPI_ERROR (MPI_Gather (&tmp, 1, PUGH_MPI_INT,
+                                  rem_dcount, 1, PUGH_MPI_INT,
+                                  proc, pughGH->PUGH_COMM_WORLD));
   }
-  dsize[0] = all_dcount;
-  for (iproc=0;iproc<nprocs;iproc++)
-    printf(" DISPLS[%d] rem: %d   displs: %d   all %d \n",
-	   iproc, rem_dcount[iproc],displs[iproc],all_dcount);
-  fflush(stdout);
-
-  /*$dptr= malloc(all_dcount * vtypesize);$*/
-  
-  i = loc_dcount[0];
+
+  if (proc < 0 || proc == myproc)
   {
-    
-  void *st = malloc(all_dcount * vtypesize);
-  st = dptr;
-  CACTUS_MPI_ERROR (MPI_Gatherv(loc_dptr, i , vmpi_type, 
-				st,     rem_dcount,    displs,    vmpi_type,
-				0, pughGH->PUGH_COMM_WORLD));
+    displs[0]  = 0;
+    all_dcount = recvcnts[0] = (int) rem_dcount[0];
+    for (iproc=1;iproc<nprocs;iproc++)
+    {
+      recvcnts[iproc] = (int) rem_dcount[iproc];
+      displs[iproc] = displs[iproc-1] + recvcnts[iproc-1];
+      all_dcount   += recvcnts[iproc];
+    }
+
+    printf ("  dsize: %d\n", dsize [0]);
+    for (iproc=0;iproc<nprocs;iproc++)
+      printf(" DISPLS[%d] rem: %d   displs: %d   all %d \n",
+             iproc, recvcnts[iproc],displs[iproc],all_dcount);
+    fflush(stdout);
+
+    all_dptr = malloc (all_dcount * vtypesize);
   }
-  
-  
+  else
+  {
+    all_dptr = NULL;
+  }
+
+  if (proc < 0)
+  {
+    CACTUS_MPI_ERROR (MPI_Allgatherv(loc_dptr, loc_dcount [0], vmpi_type, 
+                                     all_dptr, recvcnts, displs, vmpi_type,
+                                     pughGH->PUGH_COMM_WORLD));
+  } else {
+    CACTUS_MPI_ERROR (MPI_Gatherv(loc_dptr, loc_dcount [0], vmpi_type, 
+                                  all_dptr, recvcnts, displs, vmpi_type,
+                                  proc, pughGH->PUGH_COMM_WORLD));
+  }
+
+  if (proc < 0 || proc == myproc)
+  {
+    free (rem_dcount);
+    free (recvcnts);
+    free (displs);
+  }
+
+#else
+  all_dptr = loc_dptr;
+  all_dcount = loc_dcount;
+#endif
+
+  /* assign return values */
+  if (proc < 0 || proc == myproc)
+  {
+    *dptr = all_dptr;
+    dsize[0] = all_dcount;
+  }
+
   printf(" CollectData1D done \n");
   return(0);
 }
 
 
 int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim, 
-		       int *S_l, int *u_l, int *ds, int len_l,
-		       void **loc_dptr, int *dsize)
+                       int *S_l, int *u_l, int *ds, int len_l,
+                       void **loc_dptr, int *dsize)
 {
   int *locstart, *locend, nlocpoints;
   int ipnt, gridpnt[MAX_DIM];
-  int ierr;
+  int ret_val;
 
   int gindex,vtype,vtypesize;
   int idim,lincount;
@@ -109,89 +154,59 @@ int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
   void *data = CCTK_VarDataPtrI (GH, vtimelvl, vindex);
 
 
+  if (vdim > MAX_DIM)
+    return (-1);
+
   vtype      = CCTK_VarTypeI(vindex);
   vtypesize  = CCTK_VarTypeSize(vtype);
 
   locstart = (int*)malloc(vdim*sizeof(int));
   locend   = (int*)malloc(vdim*sizeof(int));
 
-  ierr    = GetLocalLine(GH, vdim, S_l, u_l, ds[0], len_l, 
-			 locstart, locend, &nlocpoints);
+  ret_val  = GetLocalLine(GH, vdim, S_l, u_l, ds[0], len_l, 
+                          locstart, locend, &nlocpoints);
+  if (ret_val)
+  {
+    CCTK_WARN (1, "GetLocalLine() failed");
+    return (ret_val);
+  }
 
   dsize[0]= nlocpoints;
 
   printf("CollectLocal1D: %d npoints   lb: %d %d %d\n",
-	 nlocpoints,GH->cctk_lbnd[0],GH->cctk_lbnd[1],GH->cctk_lbnd[2]);
+         nlocpoints,GH->cctk_lbnd[0],GH->cctk_lbnd[1],GH->cctk_lbnd[2]);
 
   *loc_dptr= malloc(nlocpoints * vtypesize);
 
   lincount=0; 
 
+  /* zero out index array so that unused dimensions are 0 */
+  memset (gridpnt, 0, sizeof (gridpnt));
+
   /* CCTK_VARIABLE_INT */
   if (vtype == CCTK_VARIABLE_INT)
   {
     CCTK_INT *int_dptr = (CCTK_INT *) *loc_dptr;
 
-    if (vdim==3) {    
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-      { 
-	for (idim=0;idim<vdim;idim++)
-	  gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	int_dptr[lincount++]=((CCTK_INT *) data)
-	  [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
-      }
-    }
-    else if (vdim==2) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-      { 
-	for (idim=0;idim<vdim;idim++)
-	  gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	int_dptr[lincount++]=((CCTK_INT *) data)
-	  [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])];
-      }
-    }
-    else if (vdim==1) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-      { 
-	for (idim=0;idim<vdim;idim++)
-	  gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	int_dptr[lincount++]=((CCTK_INT *) data)
-	    [CCTK_GFINDEX1D(GH,gridpnt[0])];
-      }
+    for (ipnt=0; ipnt<nlocpoints; ipnt++)
+    { 
+      for (idim=0;idim<vdim;idim++)
+        gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
+      int_dptr[lincount++]=((CCTK_INT *) data)
+        [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
     }
-    else return(-1);
   }   
   /* CCTK_VARIABLE_CHAR */
   else if (vtype == CCTK_VARIABLE_CHAR)
   {
     CCTK_CHAR *char_dptr = (CCTK_CHAR *) *loc_dptr;
 
-    if (vdim==3) {   
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  char_dptr[lincount++]=((CCTK_CHAR *) data)
-	    [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
-	}
-    }
-    else if (vdim==2) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  char_dptr[lincount++]=((CCTK_CHAR *) data)
-	    [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])];
-	}
-    }
-    else if (vdim==1) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  char_dptr[lincount++]=((CCTK_CHAR *) data)
-	    [CCTK_GFINDEX1D(GH,gridpnt[0])];
-	}
+    for (ipnt=0; ipnt<nlocpoints; ipnt++)
+    { 
+      for (idim=0;idim<vdim;idim++)
+        gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
+      char_dptr[lincount++]=((CCTK_CHAR *) data)
+        [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
     }
   }
  
@@ -200,37 +215,17 @@ int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
   {
     CCTK_REAL *real_dptr = (CCTK_REAL *) *loc_dptr;
 
-    if (vdim==3) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  printf(" nlocpoints: %d \n",nlocpoints);
-	  for (idim=0;idim<vdim;idim++) 
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	    printf(" gridpnts: ( %d %d %d ) ", 
-		   gridpnt[0],gridpnt[1],gridpnt[2]);
-	    
-	  real_dptr[lincount++]=((CCTK_REAL *) data)
-	    [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
-	  printf(" Value:  %f \n",real_dptr[lincount-1]);
-	}
-    }
-    else if (vdim==2) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  real_dptr[lincount++]=((CCTK_REAL *) data)
-	    [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])];
-	}
-    }
-    else if (vdim==1) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  real_dptr[lincount++]=((CCTK_REAL *) data)
-	    [CCTK_GFINDEX1D(GH,gridpnt[0])];
-	}
+    for (ipnt=0; ipnt<nlocpoints; ipnt++)
+    { 
+      printf(" nlocpoints: %d \n",nlocpoints);
+      for (idim=0;idim<vdim;idim++) 
+        gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
+      printf(" gridpnts: ( %d %d %d ) ", 
+             gridpnt[0],gridpnt[1],gridpnt[2]);
+        
+      real_dptr[lincount++]=((CCTK_REAL *) data)
+        [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
+      printf(" Value:  %f \n",real_dptr[lincount-1]);
     }
   }   
   /* CCTK_VARIABLE_REAL4 */
@@ -238,40 +233,27 @@ int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
   {    
     CCTK_REAL4 *real4_dptr = (CCTK_REAL4*) *loc_dptr;
 
-    if (vdim==3) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  
-	  real4_dptr[lincount++]=((CCTK_REAL4 *) data)
-	    [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
-	}
-    }
-    else if (vdim==2) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	    gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	    
-	  real4_dptr[lincount++]=((CCTK_REAL4 *) data)
-	    [CCTK_GFINDEX2D(GH,gridpnt[0],gridpnt[1])];
-	}
-    }
-    else if (vdim==1) {
-      for (ipnt=0; ipnt<nlocpoints; ipnt++)
-	{ 
-	  for (idim=0;idim<vdim;idim++)
-	      gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
-	  
-	  real4_dptr[lincount++]=((CCTK_REAL4 *) data)
-	    [CCTK_GFINDEX1D(GH,gridpnt[0])];
-	}
+    for (ipnt=0; ipnt<nlocpoints; ipnt++)
+    { 
+      for (idim=0;idim<vdim;idim++)
+        gridpnt[idim] = locstart[idim]+ipnt * u_l[idim];
+      
+      real4_dptr[lincount++]=((CCTK_REAL4 *) data)
+        [CCTK_GFINDEX3D(GH,gridpnt[0],gridpnt[1],gridpnt[2])];
     }
   }
+  else
+  {
+    char *fullname = CCTK_FullName (vindex);
+
+    CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                "Unsupported type for variable '%s'", fullname);
+    free (fullname);
+    ret_val = -1;
+  }
 
   free(locstart);
   free(locend);
 
+  return (ret_val);
 }
-  
diff --git a/src/Hyperslab.h b/src/Hyperslab.h
index 3a894e7..2061e49 100644
--- a/src/Hyperslab.h
+++ b/src/Hyperslab.h
@@ -2,7 +2,7 @@
 
 int CollectData1D(cGH *GH, int vindex, int vtimelvl, int vdim,
 		  int *S_l, int *u_l, int *ds, int len_l, 
-		  void **dptr, int *dsize);
+		  void **dptr, int *dsize, int proc);
 
 int CollectLocalData1D(cGH *GH, int vindex, int vtimelvl, int vdim, 
 		       int *S_l, int *u_l, int *ds, int len_l,
diff --git a/src/Misc.c b/src/Misc.c
index e5e905c..7c6747a 100644
--- a/src/Misc.c
+++ b/src/Misc.c
@@ -18,9 +18,9 @@ int IsOnGlobGrid(cGH *GH, int vdim, int *grdpnt)
 }
       
 
-int CCTKtoMPItype(int cctktype)
+MPI_Datatype CCTKtoMPItype(int cctktype)
 {
-  int mpitype;
+  MPI_Datatype mpitype;
   switch (cctktype) {
 
   case CCTK_VARIABLE_CHAR:
diff --git a/src/TestSlab.c b/src/TestSlab.c
index 2a3b966..ac6e72f 100644
--- a/src/TestSlab.c
+++ b/src/TestSlab.c
@@ -82,10 +82,13 @@ void TestSlab(CCTK_ARGUMENTS) {
   printf("Calling CollectData \n");
   CollectData1D(cctkGH, vindex, vtimelvl, vdim, 
 		gs, di, ds, 10, 
-		&dptr, dsize);
+		&dptr, dsize, 0);
 
-  printf("TestSlab: %d \n",dsize[0]);
-  for (l=0;l<dsize[0];l++)
-    printf("%f \n",dptr[l]); 
+  if (CCTK_MyProc (cctkGH) == 0)
+  {
+    printf("TestSlab: %d \n",dsize[0]);
+    for (l=0;l<dsize[0];l++)
+      printf("%f \n",dptr[l]); 
+  }
 
 }