From b74b2a738738bd33ce1c8819d0f71dbbc865910b Mon Sep 17 00:00:00 2001 From: lanfer Date: Tue, 19 Oct 1999 12:19:16 +0000 Subject: Removing mpi statements from the AHFinder routines. Successful for the scalars, the 1D arrays are still in mpi as the reduction operation for 1d arrays is harmonized. Successfully tested against screen output since there is no testsuite for now. gerd git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinder/trunk@14 89daf98e-ef62-4674-b946-b8ff9de2216c --- src/AHFinder.F | 2 +- src/AHFinder_gau.F | 45 ++++++++++++--- src/AHFinder_int.F | 164 ++++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 154 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/AHFinder.F b/src/AHFinder.F index 53cbd3b..f45ad29 100644 --- a/src/AHFinder.F +++ b/src/AHFinder.F @@ -241,7 +241,7 @@ c Note thadmcevol_onpunc_0.250_128_neu2_drop/at including cactus.h will also inc ! write(*,*) 'filestr =', filestr ! write(*,*) 'nfile =',nfile ! - write(*,*) 'jetzt bin ich da, wo ich nicht sein sollte' + write(*,*) 'jetzt bin ich da, wo ich nicht sein sollte ( ?? sounds like Jim Kopf und die Wilde 13 - gl)' if (find3) then if (mfind.eq.0) then diff --git a/src/AHFinder_gau.F b/src/AHFinder_gau.F index 3261bc7..a18e195 100644 --- a/src/AHFinder_gau.F +++ b/src/AHFinder_gau.F @@ -28,9 +28,9 @@ c Note that including cactus.h will also include AHFinder.h DECLARE_CCTK_FARGUMENTS DECLARE_CCTK_PARAMETERS -#ifdef MPI - include 'mpif.h' -#endif +c#ifdef MPI +c include 'mpif.h' +c#endif logical firstgau logical firstcal(4) @@ -71,6 +71,9 @@ c Note that including cactus.h will also include AHFinder.h character*200 gaussf +c Reduction related things + INTEGER reduction_handle + ! Declarations for macros. #include "CactusEinstein/Einstein/src/Einstein.h" @@ -200,6 +203,14 @@ c Note that including cactus.h will also include AHFinder.h pi = 3.141592654D0 +! ********************************************** +! * Get the reduction handle for sum operation * +! ********************************************** + call CCTK_ReductionArrayHandle(reduction_handle,"sum"); + if (reduction_handle.lt.0) then + call CCTK_WARN(1,"Cannot get reduction handle.") + endif + ! ************************************** ! *** ALLOCATE MEMORY FOR ARRAYS *** @@ -589,15 +600,31 @@ c Note that including cactus.h will also include AHFinder.h ! Reduce the errors across processors (all processors must ! know about this since all will participate on the interpolation ! below). + + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ error1, i, CCTK_VARIABLE_REAL) -#ifdef MPI - call mpi_allreduce(error1,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction failed!"); + endif error1 = i - call mpi_allreduce(error2,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ error2, i, CCTK_VARIABLE_REAL) + + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction failed!"); + endif error2 = i -#endif + +cc#ifdef MPI +cc call mpi_allreduce(error1,i,1,MPI_INTEGER, +cc . MPI_SUM,MPI_COMM_WORLD,ierror) +cc error1 = i +cc call mpi_allreduce(error2,i,1,MPI_INTEGER, +cc . MPI_SUM,MPI_COMM_WORLD,ierror) +cc error2 = i +cc#endif ! If there was an error then return from the subroutine ! (but remember to deallocate the arrays first!). diff --git a/src/AHFinder_int.F b/src/AHFinder_int.F index 83e73cb..b93f4eb 100644 --- a/src/AHFinder_int.F +++ b/src/AHFinder_int.F @@ -62,6 +62,9 @@ c@@*/ CCTK_REAL, allocatable, dimension(:,:) :: rr,xa,ya,za,da,exp,gradn CCTK_REAL, allocatable, dimension(:,:) :: txx,tyy,tzz,txy,txz,tyz +c Reduction related things + INTEGER reduction_handle + save xmn,ymn,zmn,xmx,ymx,zmx save dtheta,dphi,dtp,idtheta,idphi save npt,npp,l_ntheta,l_nphi,theta0,phi0 @@ -140,7 +143,14 @@ c@@*/ ! ! da Area element. - + + +c Get the reduction handel for the sum operation + call CCTK_ReductionArrayHandle(reduction_handle,"sum") + if (reduction_handle.lt.0) then + call CCTK_WARN(1,"Cannot get reduction handle for SUM operation.") + endif + ! ************************** ! *** DEFINE NUMBERS *** @@ -559,17 +569,30 @@ c@@*/ end if ! Reduce the errors across processors. - -#ifdef MPI - - call mpi_allreduce(interror1,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ interror1, i, CCTK_VARIABLE_INT) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction of norm failed!"); + endif interror1 = i - call mpi_allreduce(interror2,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ interror2, i, CCTK_VARIABLE_INT) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction of norm failed!"); + endif interror2 = i -#endif + +c#ifdef MPI +c +c call mpi_allreduce(interror1,i,1,MPI_INTEGER, +c . MPI_SUM,MPI_COMM_WORLD,ierror) +c interror1 = i +c call mpi_allreduce(interror2,i,1,MPI_INTEGER, +c . MPI_SUM,MPI_COMM_WORLD,ierror) +c interror2 = i +c +c#endif ! If there was an error on any processor then assign ! large values to the integrals and return from the @@ -923,31 +946,54 @@ c@@*/ ! Reduce the integrals across processors. -#ifdef MPI +! Area and expansion. + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intarea, aux, CCTK_VARIABLE_REAL) + intarea=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexp, aux, CCTK_VARIABLE_REAL) + intexp=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexp2, aux, CCTK_VARIABLE_REAL) + intexp2=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexpdel2, aux, CCTK_VARIABLE_REAL) + intexpdel2=aux + +! negative expansion elements on surfac + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ inside_min_neg_count, aux, CCTK_VARIABLE_REAL) + inside_min_neg_count=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ inside_min_neg_count, aux, CCTK_VARIABLE_REAL) + inside_min_count=aux + + +c#ifdef MPI ! Area and expansion. - call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intarea = aux - call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexp = aux - call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexp2 = aux - call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexpdel2 = aux +CCC# call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intarea = aux +CCC# call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexp = aux +CCC# call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexp2 = aux +CCC# call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexpdel2 = aux ! negative expansion elements on surface - call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - inside_min_count = aux - call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - inside_min_neg_count = aux +CCC# call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# inside_min_count = aux +CCC# call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# inside_min_neg_count = aux ! Spectral components for flow. @@ -958,19 +1004,31 @@ c@@*/ auxi = lmax+1 if (hw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflow0, tempv, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(hflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_SUM,MPI_COMM_WORLD,ierror) hflow0 = tempv end if - if (cw.ne.zero) then + if (cw.ne.zero) then +c write(*,*) "AHFinder_int1 cflow0:", cflow0 +c write(*,*) "AHFinder_int1 tempv:", tempv +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflow0, tempv, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(cflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_SUM,MPI_COMM_WORLD,ierror) +c write(*,*) "AHFinder_int2 cflow0:", cflow0 +c write(*,*) "AHFinder_int2 tempv:", tempv cflow0 = tempv +c write(*,*) "AHFinder_int3 cflow0:", cflow0 +c write(*,*) "AHFinder_int3 tempv:", tempv end if if (nw.ne.zero) then - call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) - nflow0 = tempv +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflow0, tempv, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION, + . MPI_SUM,MPI_COMM_WORLD,ierror) + nflow0 = tempv end if ! Non-axisymmetric. @@ -980,36 +1038,48 @@ c@@*/ auxi = lmax*lmax if (nw.ne.zero) then - call mpi_allreduce(hflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - hflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(hflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + hflowc = tempm end if if (cw.ne.zero) then - call mpi_allreduce(cflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - cflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(cflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + cflowc = tempm end if if (nw.ne.zero) then - call mpi_allreduce(nflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - nflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(nflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + nflowc = tempm end if if (.not.refy) then if (hw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(hflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) hflows = tempm end if if (cw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(cflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) cflows = tempm end if if (nw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(nflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) nflows = tempm end if end if @@ -1018,7 +1088,7 @@ c@@*/ end if -#endif +c#endif ! For cartoon multiply the integrals with 2*pi. -- cgit v1.2.3