diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/AHFinder.F | 2 | ||||
| -rw-r--r-- | src/AHFinder_gau.F | 45 | ||||
| -rw-r--r-- | src/AHFinder_int.F | 164 |
3 files changed, 154 insertions, 57 deletions
diff --git a/src/AHFinder.F b/src/AHFinder.F index 53cbd3b..f45ad29 100644 --- a/src/AHFinder.F +++ b/src/AHFinder.F @@ -241,7 +241,7 @@ c Note thadmcevol_onpunc_0.250_128_neu2_drop/at including cactus.h will also inc ! write(*,*) 'filestr =', filestr ! write(*,*) 'nfile =',nfile ! - write(*,*) 'jetzt bin ich da, wo ich nicht sein sollte' + write(*,*) 'jetzt bin ich da, wo ich nicht sein sollte ( ?? sounds like Jim Kopf und die Wilde 13 - gl)' if (find3) then if (mfind.eq.0) then diff --git a/src/AHFinder_gau.F b/src/AHFinder_gau.F index 3261bc7..a18e195 100644 --- a/src/AHFinder_gau.F +++ b/src/AHFinder_gau.F @@ -28,9 +28,9 @@ c Note that including cactus.h will also include AHFinder.h DECLARE_CCTK_FARGUMENTS DECLARE_CCTK_PARAMETERS -#ifdef MPI - include 'mpif.h' -#endif +c#ifdef MPI +c include 'mpif.h' +c#endif logical firstgau logical firstcal(4) @@ -71,6 +71,9 @@ c Note that including cactus.h will also include AHFinder.h character*200 gaussf +c Reduction related things + INTEGER reduction_handle + ! Declarations for macros. #include "CactusEinstein/Einstein/src/Einstein.h" @@ -200,6 +203,14 @@ c Note that including cactus.h will also include AHFinder.h pi = 3.141592654D0 +! ********************************************** +! * Get the reduction handle for sum operation * +! ********************************************** + call CCTK_ReductionArrayHandle(reduction_handle,"sum"); + if (reduction_handle.lt.0) then + call CCTK_WARN(1,"Cannot get reduction handle.") + endif + ! ************************************** ! *** ALLOCATE MEMORY FOR ARRAYS *** @@ -589,15 +600,31 @@ c Note that including cactus.h will also include AHFinder.h ! Reduce the errors across processors (all processors must ! know about this since all will participate on the interpolation ! below). + + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ error1, i, CCTK_VARIABLE_REAL) -#ifdef MPI - call mpi_allreduce(error1,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction failed!"); + endif error1 = i - call mpi_allreduce(error2,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ error2, i, CCTK_VARIABLE_REAL) + + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction failed!"); + endif error2 = i -#endif + +cc#ifdef MPI +cc call mpi_allreduce(error1,i,1,MPI_INTEGER, +cc . MPI_SUM,MPI_COMM_WORLD,ierror) +cc error1 = i +cc call mpi_allreduce(error2,i,1,MPI_INTEGER, +cc . MPI_SUM,MPI_COMM_WORLD,ierror) +cc error2 = i +cc#endif ! If there was an error then return from the subroutine ! (but remember to deallocate the arrays first!). diff --git a/src/AHFinder_int.F b/src/AHFinder_int.F index 83e73cb..b93f4eb 100644 --- a/src/AHFinder_int.F +++ b/src/AHFinder_int.F @@ -62,6 +62,9 @@ c@@*/ CCTK_REAL, allocatable, dimension(:,:) :: rr,xa,ya,za,da,exp,gradn CCTK_REAL, allocatable, dimension(:,:) :: txx,tyy,tzz,txy,txz,tyz +c Reduction related things + INTEGER reduction_handle + save xmn,ymn,zmn,xmx,ymx,zmx save dtheta,dphi,dtp,idtheta,idphi save npt,npp,l_ntheta,l_nphi,theta0,phi0 @@ -140,7 +143,14 @@ c@@*/ ! ! da Area element. - + + +c Get the reduction handel for the sum operation + call CCTK_ReductionArrayHandle(reduction_handle,"sum") + if (reduction_handle.lt.0) then + call CCTK_WARN(1,"Cannot get reduction handle for SUM operation.") + endif + ! ************************** ! *** DEFINE NUMBERS *** @@ -559,17 +569,30 @@ c@@*/ end if ! Reduce the errors across processors. - -#ifdef MPI - - call mpi_allreduce(interror1,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ interror1, i, CCTK_VARIABLE_INT) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction of norm failed!"); + endif interror1 = i - call mpi_allreduce(interror2,i,1,MPI_INTEGER, - . MPI_SUM,MPI_COMM_WORLD,ierror) + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ interror2, i, CCTK_VARIABLE_INT) + if (ierror.ne.0) then + call CCTK_WARN(1,"Reduction of norm failed!"); + endif interror2 = i -#endif + +c#ifdef MPI +c +c call mpi_allreduce(interror1,i,1,MPI_INTEGER, +c . MPI_SUM,MPI_COMM_WORLD,ierror) +c interror1 = i +c call mpi_allreduce(interror2,i,1,MPI_INTEGER, +c . MPI_SUM,MPI_COMM_WORLD,ierror) +c interror2 = i +c +c#endif ! If there was an error on any processor then assign ! large values to the integrals and return from the @@ -923,31 +946,54 @@ c@@*/ ! Reduce the integrals across processors. -#ifdef MPI +! Area and expansion. + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intarea, aux, CCTK_VARIABLE_REAL) + intarea=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexp, aux, CCTK_VARIABLE_REAL) + intexp=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexp2, aux, CCTK_VARIABLE_REAL) + intexp2=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ intexpdel2, aux, CCTK_VARIABLE_REAL) + intexpdel2=aux + +! negative expansion elements on surfac + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ inside_min_neg_count, aux, CCTK_VARIABLE_REAL) + inside_min_neg_count=aux + call CCTK_ReduceLocalScalar(ierror, cctkGH, -1, reduction_handle, + $ inside_min_neg_count, aux, CCTK_VARIABLE_REAL) + inside_min_count=aux + + +c#ifdef MPI ! Area and expansion. - call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intarea = aux - call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexp = aux - call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexp2 = aux - call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - intexpdel2 = aux +CCC# call mpi_allreduce(intarea,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intarea = aux +CCC# call mpi_allreduce(intexp,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexp = aux +CCC# call mpi_allreduce(intexp2,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexp2 = aux +CCC# call mpi_allreduce(intexpdel2,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# intexpdel2 = aux ! negative expansion elements on surface - call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - inside_min_count = aux - call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION, - .MPI_SUM,MPI_COMM_WORLD,ierror) - inside_min_neg_count = aux +CCC# call mpi_allreduce(inside_min_count,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# inside_min_count = aux +CCC# call mpi_allreduce(inside_min_neg_count,aux,1,MPI_DOUBLE_PRECISION, +CCC# .MPI_SUM,MPI_COMM_WORLD,ierror) +CCC# inside_min_neg_count = aux ! Spectral components for flow. @@ -958,19 +1004,31 @@ c@@*/ auxi = lmax+1 if (hw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflow0, tempv, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(hflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_SUM,MPI_COMM_WORLD,ierror) hflow0 = tempv end if - if (cw.ne.zero) then + if (cw.ne.zero) then +c write(*,*) "AHFinder_int1 cflow0:", cflow0 +c write(*,*) "AHFinder_int1 tempv:", tempv +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflow0, tempv, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(cflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_SUM,MPI_COMM_WORLD,ierror) +c write(*,*) "AHFinder_int2 cflow0:", cflow0 +c write(*,*) "AHFinder_int2 tempv:", tempv cflow0 = tempv +c write(*,*) "AHFinder_int3 cflow0:", cflow0 +c write(*,*) "AHFinder_int3 tempv:", tempv end if if (nw.ne.zero) then - call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION, - . MPI_SUM,MPI_COMM_WORLD,ierror) - nflow0 = tempv +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflow0, tempv, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(nflow0,tempv,auxi,MPI_DOUBLE_PRECISION, + . MPI_SUM,MPI_COMM_WORLD,ierror) + nflow0 = tempv end if ! Non-axisymmetric. @@ -980,36 +1038,48 @@ c@@*/ auxi = lmax*lmax if (nw.ne.zero) then - call mpi_allreduce(hflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - hflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(hflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + hflowc = tempm end if if (cw.ne.zero) then - call mpi_allreduce(cflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - cflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(cflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + cflowc = tempm end if if (nw.ne.zero) then - call mpi_allreduce(nflowc,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) - nflowc = tempm +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflowc, tempm, auxi, CCTK_VARIABLE_REAL) + call mpi_allreduce(nflowc,tempm,auxi, + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + nflowc = tempm end if if (.not.refy) then if (hw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ hflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(hflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) hflows = tempm end if if (cw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ cflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(cflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) cflows = tempm end if if (nw.ne.zero) then +c call CCTK_ReduceLocalArray1D(ierror, cctkGH, -1, reduction_handle, +c $ nflows, tempm, auxi, CCTK_VARIABLE_REAL) call mpi_allreduce(nflows,tempm,auxi, - . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) + . MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,ierror) nflows = tempm end if end if @@ -1018,7 +1088,7 @@ c@@*/ end if -#endif +c#endif ! For cartoon multiply the integrals with 2*pi. |