major rewrite of multiprocessor synchronization logic to fix assorted bugs

biggest change is that we now group all the per-Newton-iteration
interprocessor communication logic into a single call per iteration;
this eats more bandwidth, but saves on latency


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinderDirect/trunk@971 f88db872-0e4f-0410-b76b-b9085cfa78c5

1 files changed, 71 insertions, 9 deletions

diff --git a/src/driver/driver.hh b/src/driver/driver.hh
index 0995f50..3640102 100644
--- a/src/driver/driver.hh
+++ b/src/driver/driver.hh
@@ -155,14 +155,67 @@ struct	verbose_info
 	bool print_algorithm_debug;
 	};
 
+//
+// This struct holds buffers for broadcasting status information from
+// each active processor to all processors.
+//
+struct	iteration_status_buffers
+	{
+	// --> high-level buffers for broadcast-level semantics in Newton()
+	int* hn_buffer;
+	int* iteration_buffer;
+	enum expansion_status* expansion_status_buffer;
+	fp* rms_norm_buffer;
+	fp* infinity_norm_buffer;
+	bool* found_horizon_buffer;
+
+	// --> low-level buffers for CCTK_Reduce()
+	jtutil::array2d<CCTK_REAL> *send_buffer_ptr;
+	jtutil::array2d<CCTK_REAL> *receive_buffer_ptr;
+
+	iteration_status_buffers()
+		: hn_buffer(NULL), iteration_buffer(NULL),
+		  expansion_status_buffer(NULL),
+		  rms_norm_buffer(NULL), infinity_norm_buffer(NULL),
+		  found_horizon_buffer(NULL),
+		  send_buffer_ptr(NULL), receive_buffer_ptr(NULL)
+		{ }
+	};
+
+//
+// This struct holds interprocessor-communication buffers for
+// broadcasting the BH diagnostics and horizon shape from the
+// processor which finds a given horizon, to all processors.
+//
+struct	horizon_buffers
+	{
+	int N_buffer;		// number of CCTK_REAL values in each buffer
+	CCTK_REAL* send_buffer;
+	CCTK_REAL* receive_buffer;
+
+	horizon_buffers()
+		: N_buffer(0),
+		  send_buffer(NULL),
+		  receive_buffer(NULL)
+		{ }
+	};
+
 //******************************************************************************
 
 //
 // (A single copy of) this struct holds all of our information about
 // a single apparent horizon.
 //
-struct	AH_info
+struct	AH_data
 	{
+	//
+	// Any given horizon is allocated to a single processor.
+	// On that processor (where we actually find the horizon)
+	// we keep a "full-fledged" patch system and a Jacobian.
+	// On other processors (where we only use the horizon shape
+	// to (optionally) set the BH excision mask), we keep only a
+	// "skeletal" patch system, and no Jacobian.
+	//
 	patch_system* ps_ptr;
 	Jacobian* Jac_ptr;
 
@@ -172,13 +225,19 @@ struct	AH_info
 	//          last time
 	//     false if we've tried before and succeeded the last time
 	//           (so we have that position as a very good initial guess)
+	// ... also false if we're not finding this horizon on this processor
 	bool initial_find_flag;
 
+	// used only if we're finding this horizon on this processor
 	struct initial_guess_info initial_guess_info;
 
 	bool found_flag;	// did we find this horizon (successfully)
 	struct BH_diagnostics BH_diagnostics;
 	FILE *BH_diagnostics_fileptr;
+
+	// interprocessor-communication buffers
+	// for this horizon's BH diagnostics and (optionally) horizon shape
+	struct horizon_buffers horizon_buffers;
 	};
 
 //
@@ -210,16 +269,18 @@ struct	state
 
 	struct BH_diagnostics_info BH_diagnostics_info;
 
+	// interprocessor-communication buffers for broadcasting
+	// Newton-iteration status from active processors to all processors
+	struct iteration_status_buffers isb;
+
 	horizon_sequence *my_hs;	// --> new-allocated object describing
-					//     the sequence of horizons
+					//     the sequence of genuine horizons
 					//     assigned to this processor
 
 	// horizon numbers ("hn") run from 1 to N_horizons inclusive
-	struct AH_info** my_AH_info;	// --> new[]-allocated array of size
+	struct AH_data** my_AH_data;	// --> new[]-allocated array of size
 					//     N_horizons+1, subscripted by hn,
-					//     of --> info or NULL for horizons
-					//            not assigned to this
-					//            processor
+					//     of --> info
 	};
 
 //******************************************************************************
@@ -249,9 +310,9 @@ enum initial_guess_method
 
 // Newton.cc
 // returns true for success, false for failure to converge
-void Newton(cGH* GH,
+void Newton(const cGH* GH,
 	    int N_procs, int N_active_procs, int my_proc,
-	    horizon_sequence& hs, struct AH_info* const my_AH_info[],
+	    horizon_sequence& hs, struct AH_data* const my_AH_data[],
 	    const struct cactus_grid_info& cgi,
 	    const struct geometry_info& gi,
 	    const struct Jacobian_info& Jacobian_info,
@@ -259,7 +320,8 @@ void Newton(cGH* GH,
 	    const struct IO_info& IO_info,
 	    const struct BH_diagnostics_info& BH_diagnostics_info,
 	    const struct error_info& error_info,
-	    const struct verbose_info& verbose_info);
+	    const struct verbose_info& verbose_info,
+	    struct iteration_status_buffers& isb);
 
 // io.cc
 void input_gridfn(patch_system& ps, int unknown_gfn,