* major changes to driver routines to find multiple horizons in parallel

  across multiple processors -- see src/driver/README.parallel for details
* drop convergence checks on ||Delta_h|| in param.ccl because they don't
  fit well with parallelization changes

==> With this changes, AHFinderDirect is now (I think)
    multiprocessor-ready!!


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinderDirect/trunk@946 f88db872-0e4f-0410-b76b-b9085cfa78c5

4 files changed, 240 insertions, 0 deletions

diff --git a/run/test-ahfinderdirect/Kerr/Kerr-eval.par b/run/test-ahfinderdirect/Kerr/Kerr-eval.par
new file mode 100644
index 0000000..013d148
--- /dev/null
+++ b/run/test-ahfinderdirect/Kerr/Kerr-eval.par
@@ -0,0 +1,56 @@
+# This parameter file sets up Kerr/Kerr-Schild initial data, then
+# evalutes Theta for the "exact" initial guess.  The local coordinate
+# origin is deliberately de-centered with respect to the black hole,
+# to make this a non-trivial test for the apparent horizon finder.
+
+# flesh
+cactus::cctk_itlast = 0
+
+ActiveThorns = "PUGH"
+driver::ghost_size = 3
+driver::global_nx = 31
+driver::global_ny = 31
+driver::global_nz = 19
+
+ActiveThorns = "CartGrid3D"
+grid::domain = "bitant"
+grid::avoid_origin = "false"
+grid::type = "byspacing"
+grid::dxyz = 0.2
+
+ActiveThorns = "ADMBase ADMCoupling StaticConformal CoordGauge Exact"
+ADMBase::initial_lapse = "exact"
+ADMBase::initial_shift = "exact"
+ADMBase::initial_data = "exact"
+ADMBase::lapse_evolution_method = "static"
+ADMBase::shift_evolution_method = "static"
+ADMBase::metric_type = "physical"
+Exact::exact_model = "Kerr/Kerr-Schild"
+Exact::Kerr_KerrSchild__mass = 1.0
+Exact::Kerr_KerrSchild__spin = 0.6
+
+########################################
+
+ActiveThorns = "LocalInterp PUGHInterp PUGHReduce AHFinderDirect"
+
+AHFinderDirect::find_AHs_at_postinitial = "true"
+AHFinderDirect::find_AHs_at_poststep    = "false"
+AHFinderDIrect::method = "evaluate expansions"
+AHFinderDirect::print_timing_stats = "true"
+AHFinderDirect::verbose_level = "algorithm details"
+
+AHFinderDirect::how_often_to_output_Theta = 1
+AHFinderDirect::h_base_file_name     = "Kerr-eval.h"
+AHFinderDirect::Theta_base_file_name = "Kerr-eval.Theta"
+
+AHFinderDirect::N_horizons = 1
+AHFinderDirect::origin_x[1] = 0.5
+AHFinderDirect::origin_y[1] = 0.7
+AHFinderDirect::origin_z[1] = 0.0
+
+AHFinderDirect::initial_guess_method[1] = "Kerr/Kerr-Schild"
+AHFinderDirect::initial_guess__Kerr_Kerr__mass[1] = 1.0
+AHFinderDirect::initial_guess__Kerr_Kerr__spin[1] = 0.6
+AHFinderDirect::initial_guess__Kerr_Kerr__x_posn[1] = 0.0
+AHFinderDirect::initial_guess__Kerr_Kerr__y_posn[1] = 0.0
+AHFinderDirect::initial_guess__Kerr_Kerr__z_posn[1] = 0.0
diff --git a/run/test-ahfinderdirect/Kerr/Kerr-find.par b/run/test-ahfinderdirect/Kerr/Kerr-find.par
new file mode 100644
index 0000000..50b9720
--- /dev/null
+++ b/run/test-ahfinderdirect/Kerr/Kerr-find.par
@@ -0,0 +1,55 @@
+# This parameter file sets up Kerr/Kerr-Schild initial data, then
+# finds the apparent horizon in it.  The local coordinate system origin
+# and the initial guess are both deliberately de-centered with respect
+# to the black hole, to make this a non-trivial test for the apparent
+# horizon finder.
+
+# flesh
+cactus::cctk_itlast = 0
+
+ActiveThorns = "PUGH"
+driver::ghost_size = 3
+driver::global_nx = 31
+driver::global_ny = 31
+driver::global_nz = 19
+
+ActiveThorns = "CartGrid3D"
+grid::domain = "bitant"
+grid::avoid_origin = "false"
+grid::type = "byspacing"
+grid::dxyz = 0.2
+
+ActiveThorns = "ADMBase ADMCoupling StaticConformal CoordGauge Exact"
+ADMBase::initial_lapse = "exact"
+ADMBase::initial_shift = "exact"
+ADMBase::initial_data = "exact"
+ADMBase::lapse_evolution_method = "static"
+ADMBase::shift_evolution_method = "static"
+ADMBase::metric_type = "physical"
+Exact::exact_model = "Kerr/Kerr-Schild"
+Exact::Kerr_KerrSchild__mass = 1.0
+Exact::Kerr_KerrSchild__spin = 0.6
+
+########################################
+
+ActiveThorns = "LocalInterp PUGHInterp PUGHReduce AHFinderDirect"
+
+AHFinderDirect::find_AHs_at_postinitial = "true"
+AHFinderDirect::find_AHs_at_poststep    = "false"
+AHFinderDirect::print_timing_stats = "true"
+AHFinderDirect::verbose_level = "algorithm details"
+
+AHFinderDirect::how_often_to_output_Theta = 1
+AHFinderDirect::h_base_file_name     = "Kerr-find.h"
+AHFinderDirect::Theta_base_file_name = "Kerr-find.Theta"
+
+AHFinderDirect::N_horizons = 1
+AHFinderDirect::origin_x[1] = 0.5
+AHFinderDirect::origin_y[1] = 0.7
+AHFinderDirect::origin_z[1] = 0.0
+
+AHFinderDirect::initial_guess_method[1] = "coordinate sphere"
+AHFinderDirect::initial_guess__coord_sphere__x_center[1] = -0.2
+AHFinderDirect::initial_guess__coord_sphere__y_center[1] =  0.3
+AHFinderDirect::initial_guess__coord_sphere__z_center[1] =  0.0
+AHFinderDirect::initial_guess__coord_sphere__radius[1] = 2.0
diff --git a/run/test-ahfinderdirect/Kerr/Kerr.par b/run/test-ahfinderdirect/Kerr/Kerr.par
new file mode 100644
index 0000000..1d9a09a
--- /dev/null
+++ b/run/test-ahfinderdirect/Kerr/Kerr.par
@@ -0,0 +1,55 @@
+# This parameter file sets up Kerr/Kerr-Schild initial data, then
+# finds the apparent horizon in it.  The local coordinate system origin
+# and the initial guess are both deliberately de-centered with respect
+# to the black hole, to make this a non-trivial test for the apparent
+# horizon finder.
+
+# flesh
+cactus::cctk_itlast = 0
+
+ActiveThorns = "PUGH"
+driver::ghost_size = 2
+driver::global_nx = 31
+driver::global_ny = 31
+driver::global_nz = 17
+
+ActiveThorns = "CartGrid3D"
+grid::domain = "bitant"
+grid::avoid_origin = "false"
+grid::type = "byspacing"
+grid::dxyz = 0.2
+
+ActiveThorns = "ADMBase ADMCoupling StaticConformal CoordGauge Exact"
+ADMBase::initial_lapse = "exact"
+ADMBase::initial_shift = "exact"
+ADMBase::initial_data = "exact"
+ADMBase::lapse_evolution_method = "static"
+ADMBase::shift_evolution_method = "static"
+ADMBase::metric_type = "physical"
+Exact::exact_model = "Kerr/Kerr-Schild"
+Exact::Kerr_KerrSchild__mass = 1.0
+Exact::Kerr_KerrSchild__spin = 0.6
+
+########################################
+
+ActiveThorns = "LocalInterp PUGHInterp PUGHReduce AHFinderDirect"
+
+AHFinderDirect::find_AHs_at_postinitial = "true"
+AHFinderDirect::find_AHs_at_poststep    = "false"
+AHFinderDirect::print_timing_stats = "true"
+AHFinderDirect::verbose_level = "algorithm details"
+
+AHFinderDirect::how_often_to_output_Theta = 1
+AHFinderDirect::h_base_file_name     = "Kerr.h"
+AHFinderDirect::Theta_base_file_name = "Kerr.Theta"
+
+AHFinderDirect::N_horizons = 1
+AHFinderDirect::origin_x[1] = 0.5
+AHFinderDirect::origin_y[1] = 0.7
+AHFinderDirect::origin_z[1] = 0.0
+
+AHFinderDirect::initial_guess_method[1] = "coordinate sphere"
+AHFinderDirect::initial_guess__coord_sphere__x_center[1] = -0.2
+AHFinderDirect::initial_guess__coord_sphere__y_center[1] =  0.3
+AHFinderDirect::initial_guess__coord_sphere__z_center[1] =  0.0
+AHFinderDirect::initial_guess__coord_sphere__radius[1] = 2.0
diff --git a/run/test-ahfinderdirect/Kerr/Kerr3-find.par b/run/test-ahfinderdirect/Kerr/Kerr3-find.par
new file mode 100644
index 0000000..3885c21
--- /dev/null
+++ b/run/test-ahfinderdirect/Kerr/Kerr3-find.par
@@ -0,0 +1,74 @@
+# This parameter file sets up Kerr/Kerr-Schild initial data, then
+# finds the apparent horizon in it.  The local coordinate system origin
+# and the initial guess are both deliberately de-centered with respect
+# to the black hole, to make this a non-trivial test for the apparent
+# horizon finder.
+
+# flesh
+cactus::cctk_itlast = 0
+
+ActiveThorns = "PUGH"
+driver::ghost_size = 3
+driver::global_nx = 31
+driver::global_ny = 31
+driver::global_nz = 19
+
+ActiveThorns = "CartGrid3D"
+grid::domain = "bitant"
+grid::avoid_origin = "false"
+grid::type = "byspacing"
+grid::dxyz = 0.2
+
+ActiveThorns = "ADMBase ADMCoupling StaticConformal CoordGauge Exact"
+ADMBase::initial_lapse = "exact"
+ADMBase::initial_shift = "exact"
+ADMBase::initial_data = "exact"
+ADMBase::lapse_evolution_method = "static"
+ADMBase::shift_evolution_method = "static"
+ADMBase::metric_type = "physical"
+Exact::exact_model = "Kerr/Kerr-Schild"
+Exact::Kerr_KerrSchild__mass = 1.0
+Exact::Kerr_KerrSchild__spin = 0.6
+
+########################################
+
+ActiveThorns = "LocalInterp PUGHInterp PUGHReduce AHFinderDirect"
+
+AHFinderDirect::find_AHs_at_postinitial = "true"
+AHFinderDirect::find_AHs_at_poststep    = "false"
+AHFinderDirect::print_timing_stats = "true"
+AHFinderDirect::verbose_level = "algorithm details"
+
+AHFinderDirect::how_often_to_output_Theta = 1
+AHFinderDirect::h_base_file_name     = "Kerr3-find.h"
+AHFinderDirect::Theta_base_file_name = "Kerr3-find.Theta"
+
+AHFinderDirect::N_horizons = 3
+
+AHFinderDirect::origin_x[1] = 0.5
+AHFinderDirect::origin_y[1] = 0.7
+AHFinderDirect::origin_z[1] = 0.0
+AHFinderDirect::initial_guess_method[1] = "coordinate sphere"
+AHFinderDirect::initial_guess__coord_sphere__x_center[1] = -0.2
+AHFinderDirect::initial_guess__coord_sphere__y_center[1] =  0.3
+AHFinderDirect::initial_guess__coord_sphere__z_center[1] =  0.0
+AHFinderDirect::initial_guess__coord_sphere__radius[1] = 2.0
+
+AHFinderDirect::origin_x[2] = 0.9
+AHFinderDirect::origin_y[2] = 1.1
+AHFinderDirect::origin_z[2] = 0.0
+AHFinderDirect::initial_guess_method[1] = "coordinate sphere"
+AHFinderDirect::initial_guess__coord_sphere__x_center[2] = 0.1
+AHFinderDirect::initial_guess__coord_sphere__y_center[2] = 0.2
+AHFinderDirect::initial_guess__coord_sphere__z_center[2] = 0.0
+AHFinderDirect::initial_guess__coord_sphere__radius[2] = 1.7
+
+AHFinderDirect::origin_x[3] = 0.9
+AHFinderDirect::origin_y[3] = 1.1
+AHFinderDirect::origin_z[3] = 0.0
+AHFinderDirect::initial_guess_method[1] = "Kerr/Kerr-Schild"
+AHFinderDirect::initial_guess__Kerr_KerrSchild__x_posn[3] = 0.0
+AHFinderDirect::initial_guess__Kerr_KerrSchild__y_posn[3] = 0.0
+AHFinderDirect::initial_guess__Kerr_KerrSchild__z_posn[3] = 0.0
+AHFinderDirect::initial_guess__Kerr_KerrSchild__mass[3] = 1.0
+AHFinderDirect::initial_guess__Kerr_KerrSchild__spin[3] = 0.6