Removing mpi statements from the AHFinder routines. Successful for the scalars,

the 1D arrays are still in mpi as the reduction operation for 1d arrays is harmonized.

Successfully tested against screen output since there is no testsuite for now.


gerd


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinder/trunk@14 89daf98e-ef62-4674-b946-b8ff9de2216c

1 files changed, 117 insertions, 47 deletions

diff --git a/src/AHFinder_int.F b/src/AHFinder_int.F
index 83e73cb..b93f4eb 100644
--- a/src/AHFinder_int.F
+++ b/src/AHFinder_int.F
@@ -62,6 +62,9 @@ c@@*/
       CCTK_REAL, allocatable, dimension(:,:) :: rr,xa,ya,za,da,exp,gradn
       CCTK_REAL, allocatable, dimension(:,:) :: txx,tyy,tzz,txy,txz,tyz
 
+c     Reduction related things
+      INTEGER reduction_handle
+
       save xmn,ymn,zmn,xmx,ymx,zmx
       save dtheta,dphi,dtp,idtheta,idphi
       save npt,npp,l_ntheta,l_nphi,theta0,phi0
@@ -140,7 +143,14 @@ c@@*/
 !
 !     da         Area element.
 
-      
+
+
+c     Get the reduction handel for the sum operation
+      call CCTK_ReductionArrayHandle(reduction_handle,"sum")
+      if (reduction_handle.lt.0) then 
+        call CCTK_WARN(1,"Cannot get reduction handle for SUM operation.")
+      endif
+
 
 !     **************************
 !     ***   DEFINE NUMBERS   ***
@@ -559,17 +569,30 @@ c@@*/
       end if
 
 !     Reduce the errors across processors.
-
-#ifdef MPI
-
-      call mpi_allreduce(interror1,i,1,MPI_INTEGER,
-     .     MPI_SUM,MPI_COMM_WORLD,ierror)
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     interror1, i, CCTK_VARIABLE_INT) 
+      if (ierror.ne.0) then
+         call CCTK_WARN(1,"Reduction of norm failed!");
+      endif
       interror1 = i
-      call mpi_allreduce(interror2,i,1,MPI_INTEGER,
-     .     MPI_SUM,MPI_COMM_WORLD,ierror)
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     interror2, i, CCTK_VARIABLE_INT) 
+      if (ierror.ne.0) then
+         call CCTK_WARN(1,"Reduction of norm failed!");
+      endif
       interror2 = i
 
-#endif
+
+c#ifdef MPI
+c
+c      call mpi_allreduce(interror1,i,1,MPI_INTEGER,
+c     .     MPI_SUM,MPI_COMM_WORLD,ierror)
+c      interror1 = i
+c      call mpi_allreduce(interror2,i,1,MPI_INTEGER,
+c     .     MPI_SUM,MPI_COMM_WORLD,ierror)
+c      interror2 = i
+c
+c#endif
 
 !     If there was an error on any processor then assign
 !     large values to the integrals and return from the
@@ -923,31 +946,54 @@ c@@*/
 
 !     Reduce the integrals across processors.
 
-#ifdef MPI
+!     Area and expansion.
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     intarea, aux, CCTK_VARIABLE_REAL) 
+      intarea=aux
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     intexp, aux, CCTK_VARIABLE_REAL) 
+      intexp=aux
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     intexp2, aux, CCTK_VARIABLE_REAL) 
+      intexp2=aux
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     intexpdel2, aux, CCTK_VARIABLE_REAL) 
+      intexpdel2=aux
+
+!     negative expansion elements on surfac
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     inside_min_neg_count, aux, CCTK_VARIABLE_REAL) 
+      inside_min_neg_count=aux
+      call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, 
+     $     inside_min_neg_count, aux, CCTK_VARIABLE_REAL) 
+      inside_min_count=aux
+ 
+
+c#ifdef MPI
 
 !     Area and expansion.
 
-      call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      intarea = aux
-      call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      intexp  = aux
-      call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      intexp2 = aux
-      call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      intexpdel2 = aux
+CCC#       call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       intarea = aux
+CCC#       call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       intexp  = aux
+CCC#       call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       intexp2 = aux
+CCC#       call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       intexpdel2 = aux
 
 !     negative expansion elements on surface
 
-      call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      inside_min_count = aux
-      call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION,
-     .MPI_SUM,MPI_COMM_WORLD,ierror)
-      inside_min_neg_count = aux
+CCC#       call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       inside_min_count = aux
+CCC#       call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION,
+CCC#      .MPI_SUM,MPI_COMM_WORLD,ierror)
+CCC#       inside_min_neg_count = aux
 
 !     Spectral components for flow.
 
@@ -958,19 +1004,31 @@ c@@*/
          auxi = lmax+1
 
 	 if (hw.ne.zero) then
+c            call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c     $           hflow0, tempv, auxi, CCTK_VARIABLE_REAL) 
             call mpi_allreduce(hflow0,tempv,auxi,MPI_DOUBLE_PRECISION,
-     .      MPI_SUM,MPI_COMM_WORLD,ierror)
+     .           MPI_SUM,MPI_COMM_WORLD,ierror)
             hflow0 = tempv
 	 end if
-         if (cw.ne.zero) then
+         if (cw.ne.zero) then      
+c           write(*,*) "AHFinder_int1 cflow0:", cflow0
+c           write(*,*) "AHFinder_int1 tempv:", tempv
+c           call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $           cflow0, tempv, auxi, CCTK_VARIABLE_REAL) 
             call mpi_allreduce(cflow0,tempv,auxi,MPI_DOUBLE_PRECISION,
-     .      MPI_SUM,MPI_COMM_WORLD,ierror)
+     .           MPI_SUM,MPI_COMM_WORLD,ierror)
+c           write(*,*) "AHFinder_int2 cflow0:", cflow0
+c           write(*,*) "AHFinder_int2 tempv:", tempv
             cflow0 = tempv
+c           write(*,*) "AHFinder_int3 cflow0:", cflow0
+c           write(*,*) "AHFinder_int3 tempv:", tempv
          end if
          if (nw.ne.zero) then
-            call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION,
-     .      MPI_SUM,MPI_COMM_WORLD,ierror)
-            nflow0 = tempv
+c           call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $           nflow0, tempv, auxi, CCTK_VARIABLE_REAL) 
+             call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION,
+     .           MPI_SUM,MPI_COMM_WORLD,ierror)
+             nflow0 = tempv
          end if
 
 !        Non-axisymmetric.
@@ -980,36 +1038,48 @@ c@@*/
             auxi = lmax*lmax
 
 	    if (nw.ne.zero) then
-               call mpi_allreduce(hflowc,tempm,auxi,
-     .         MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
-               hflowc = tempm
+c              call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $              hflowc, tempm, auxi, CCTK_VARIABLE_REAL) 
+                call mpi_allreduce(hflowc,tempm,auxi,
+     .              MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+                hflowc = tempm
 	    end if
             if (cw.ne.zero) then
-               call mpi_allreduce(cflowc,tempm,auxi,
-     .         MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
-               cflowc = tempm
+c               call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c     $              cflowc, tempm, auxi, CCTK_VARIABLE_REAL) 
+                call mpi_allreduce(cflowc,tempm,auxi,
+     .              MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+                cflowc = tempm
             end if
             if (nw.ne.zero) then
-               call mpi_allreduce(nflowc,tempm,auxi,
-     .         MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
-               nflowc = tempm
+c              call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $              nflowc, tempm, auxi, CCTK_VARIABLE_REAL) 
+                call mpi_allreduce(nflowc,tempm,auxi,
+     .              MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+                nflowc = tempm
             end if
 
             if (.not.refy) then
 
 	       if (hw.ne.zero) then
+c                 call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c     $                 hflows, tempm, auxi, CCTK_VARIABLE_REAL) 
                   call mpi_allreduce(hflows,tempm,auxi,
-     .            MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+     .                 MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
                   hflows = tempm
 	       end if
                if (cw.ne.zero) then
+c                 call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $                 cflows, tempm, auxi, CCTK_VARIABLE_REAL) 
                   call mpi_allreduce(cflows,tempm,auxi,
-     .            MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+     .                 MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
                   cflows = tempm
                end if
                if (nw.ne.zero) then
+c                 call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, 
+c    $                 nflows, tempm, auxi, CCTK_VARIABLE_REAL) 
                   call mpi_allreduce(nflows,tempm,auxi,
-     .            MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
+     .                 MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror)
                   nflows = tempm
                end if
             end if
@@ -1018,7 +1088,7 @@ c@@*/
 
       end if
 
-#endif
+c#endif
 
 !     For cartoon multiply the integrals with 2*pi.