* major changes to driver routines to find multiple horizons in parallel

  across multiple processors -- see src/driver/README.parallel for details
* drop convergence checks on ||Delta_h|| in param.ccl because they don't
  fit well with parallelization changes

==> With this changes, AHFinderDirect is now (I think)
    multiprocessor-ready!!


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinderDirect/trunk@946 f88db872-0e4f-0410-b76b-b9085cfa78c5

1 files changed, 64 insertions, 34 deletions

diff --git a/src/driver/driver.hh b/src/driver/driver.hh
index bbee5aa..e5ad9aa 100644
--- a/src/driver/driver.hh
+++ b/src/driver/driver.hh
@@ -14,9 +14,9 @@
 //
 enum	method
 	{
-	method__evaluate_expansion,
-	method__test_expansion_Jacobian,
-	method__find_horizon // no comma
+	method__evaluate_expansions,
+	method__test_expansion_Jacobians,
+	method__find_horizons // no comma
 	};
 
 //
@@ -88,10 +88,8 @@ struct	solver_info
 	bool debugging_output_at_each_Newton_iteration;
 	int max_Newton_iterations__initial,
 	    max_Newton_iterations__subsequent;
-	fp max_Delta_h_over_h;
+	fp max_allowable_Delta_h_over_h;
 	fp Theta_norm_for_convergence;
-	fp Delta_h_norm_for_convergence;
-	bool final_Theta_update_if_Delta_h_converged;
 	};
 
 //
@@ -172,6 +170,9 @@ struct	BH_diagnostics
 	fp circumference_xy, circumference_xz, circumference_yz;
 	fp area;
 	fp m_irreducible;
+
+	// constructor initializes all diagnostics to 0.0
+	BH_diagnostics();
 	};
 
 //
@@ -183,38 +184,59 @@ struct	AH_info
 	patch_system* ps_ptr;
 	Jacobian* Jac_ptr;
 
+	// are we finding this horizon "from scratch"?
+	// ... true if this is the first time we've tried to find it,
+	//          or if we've tried before but failed to find it the
+	//          last time
+	//     false if we've tried before and succeeded the last time
+	//           (so we have that position as a very good initial guess)
+	bool initial_find_flag;
+
 	struct initial_guess_info initial_guess_info;
 
-	bool AH_found;
+	bool found_flag;	// did we find this horizon (successfully)
 	struct BH_diagnostics BH_diagnostics;
 	FILE *BH_diagnostics_fileptr;
 	};
 
 //
-// (A single copy of) this struct holds all of our state that's
-// persistent across Cactus scheduler calls.  This copy lives in "state.cc".
+// (A single copy of) this struct holds all of our state on this processor
+// that's persistent across Cactus scheduler calls.  This copy lives in
+// "state.cc".
 //
 struct	state
 	{
 	enum method method;
 	struct verbose_info verbose_info;
 	int timer_handle;
-	int my_proc;			// our processor number (0 to N_procs-1)
 	int N_procs;			// total number of processors
+	int my_proc;			// processor number of this processor
+					// (0 to N_procs-1)
+
+	int N_horizons;			// total number of genuine horizons
+					// being searched for
+	int N_active_procs;		// total number of active processors
+					// (the active processors are processor
+					//  numbers 0 to N_active_procs-1)
 
-	struct IO_info IO_info;
-	struct Jacobian_info Jac_info;
-	struct solver_info solver_info;
 	struct cactus_grid_info cgi;
 	struct geometry_info gi;
+	struct Jacobian_info Jac_info;
+	struct solver_info solver_info;
+	struct IO_info IO_info;
 
 	struct BH_diagnostics_info BH_diagnostics_info;
 
-	int N_horizons;
+	horizon_sequence *my_hs;	// --> new-allocated object describing
+					//     the sequence of horizons
+					//     assigned to this processor
 
-	// --> array of size N_horizons+1,
-	// indexed with "horizon number" hn (should be in range [1,N_horizons]
-	AH_info* AH_info_array;
+	// horizon numbers ("hn") run from 1 to N_horizons inclusive
+	struct AH_info** my_AH_info;	// --> new[]-allocated array of size
+					//     N_horizons+1, subscripted by hn,
+					//     of --> info or NULL for horizons
+					//            not assigned to this
+					//            processor
 	};
 
 //******************************************************************************
@@ -244,16 +266,31 @@ enum initial_guess_method
 
 // Newton.cc
 // returns true for success, false for failure to converge
-bool Newton_solve(patch_system& ps,
-		  Jacobian& Jac,
-		  const struct cactus_grid_info& cgi,
-		  const struct geometry_info& gi,
-		  const struct Jacobian_info& Jacobian_info,
-		  const struct solver_info& solver_info,
-		  bool initial_find_flag,
-		  const struct IO_info& IO_info,
-		  bool active_flag,
-		  int hn, const struct verbose_info& verbose_info);
+void Newton(cGH* GH,
+	    int N_procs, int N_active_procs, int my_proc,
+	    horizon_sequence& hs, struct AH_info* const my_AH_info[],
+	    const struct cactus_grid_info& cgi,
+	    const struct geometry_info& gi,
+	    const struct Jacobian_info& Jacobian_info,
+	    const struct solver_info& solver_info,
+	    const struct IO_info& IO_info,
+	    const struct BH_diagnostics_info& BH_diagnostics_info,
+	    const struct verbose_info& verbose_info);
+
+// BH_diagnostics.cc
+void compute_BH_diagnostics
+	(const patch_system& ps,
+	 const struct BH_diagnostics_info& BH_diagnostics_info,
+	 const struct verbose_info& verbose_info,
+	 struct BH_diagnostics& BH_diagnostics);
+void print_BH_diagnostics(const struct BH_diagnostics& BH_diagnostics,
+			  int N_horizons, int hn,
+			  const struct verbose_info& verbose_info);
+FILE* setup_BH_diagnostics_output_file(const struct IO_info& IO_info,
+				       int N_horizons, int hn);
+void output_BH_diagnostics_fn(const struct BH_diagnostics& BH_diagnostics,
+			      const struct IO_info& IO_info,
+			      int hn, FILE* fileptr);
 
 // io.cc
 void input_gridfn(patch_system& ps, int unknown_gfn,
@@ -267,10 +304,3 @@ void output_Jacobians(const patch_system& ps,
 		      const Jacobian* Jac_SD_FDdr_ptr,
 		      const struct IO_info& IO_info, const char base_file_name[],
 		      int hn, bool print_msg_flag, int AHF_iteration = 0);
-FILE* setup_BH_diagnostics_output_file(const struct IO_info& IO_info,
-				       int hn, int N_horizons);
-void output_BH_diagnostics_fn(const struct BH_diagnostics& BH_diagnostics,
-			      const struct IO_info& IO_info,
-			      int hn, FILE* fileptr);
-enum horizon_file_format
-  decode_horizon_file_format(const char horizon_file_format_string[]);