Fix a problem found by Erik Schnetter & Thomas Radke involving

writing data to the SphericalSurface variables in a multiprocessor run.

In detail, the code writes data to the SphericalSurface variables
on each processor.  For everything except the horizon shape this
is fine.  However, the code formerly didn't broadcast the horizon shape
to all processors, so the values written to the SphericalSurface variables
were all 0 on processors other than the one that actually found that horizon.

The fix is to broadcast the horizon shape if any horizon info is
being written to the SphericalSurface variables.  (In addition to
broadcasting it whenever an excision mask is being set, which the
code already did.)


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinderDirect/trunk@1392 f88db872-0e4f-0410-b76b-b9085cfa78c5

4 files changed, 23 insertions, 1 deletions

diff --git a/src/driver/README.parallel b/src/driver/README.parallel
index 136c026..67e3394 100644
--- a/src/driver/README.parallel
+++ b/src/driver/README.parallel
@@ -195,3 +195,11 @@ received from the broadcast, and for each flag which is true, all
 processors then know to participate in the broadcast of the BH diagnostics
 and (optionally) the horizon shape from the just-found-it processor
 to all processors.
+
+We only broadcast the horizon shapes if they're needed, i.e. if either
+or both of the following conditions hold:
+* we're setting an excision mask
+* we're storing horizon information in the SphericalSurface variables
+  for at least one horizon
+In either case, each processor needs to know the shapes of *all* horizons,
+so we do the broadcast.
diff --git a/src/driver/driver.hh b/src/driver/driver.hh
index 1233b50..1b19a7a 100644
--- a/src/driver/driver.hh
+++ b/src/driver/driver.hh
@@ -367,6 +367,11 @@ struct	state
 	bool announce_centroid_flag;	// should we announce horizon centroid?
 	int which_horizon_to_announce_centroid;	// if so, which horizon?
 
+	// this is the inclusive-or of  AH_data.store_info_in_SS_vars
+	// over all horizons, i.e. it says if we will store horizon
+	// information in the SphericalSurface variables for *any* horizon
+	bool store_info_in_SS_vars_for_any_horizon;
+
 	// interprocessor-communication buffers for broadcasting
 	// Newton-iteration status from active processors to all processors
 	struct iteration_status_buffers isb;
diff --git a/src/driver/find_horizons.cc b/src/driver/find_horizons.cc
index 24caa8c..24b5b7c 100644
--- a/src/driver/find_horizons.cc
+++ b/src/driver/find_horizons.cc
@@ -104,7 +104,9 @@ const int my_proc = state.my_proc;
 const int N_horizons = state.N_horizons;
 horizon_sequence& hs = *state.my_hs;
 const bool active_flag = hs.has_genuine_horizons();
-const bool broadcast_horizon_shape = state.mask_info.set_mask_for_any_horizon;
+const bool broadcast_horizon_shape
+	= state.mask_info.set_mask_for_any_horizon
+	  || state.store_info_in_SS_vars_for_any_horizon;
 
       struct cactus_grid_info&          cgi = state.cgi;
 const struct    geometry_info&           gi = state.gi;
diff --git a/src/driver/setup.cc b/src/driver/setup.cc
index 13a404f..9a76e9a 100644
--- a/src/driver/setup.cc
+++ b/src/driver/setup.cc
@@ -379,6 +379,10 @@ if (state.announce_centroid_flag)
 	}
    else state.which_horizon_to_announce_centroid = 0;	// dummy value; unused
 
+// initial value, will get each horizon's  store_info_in_SS_vars
+// flag inclusive-ored into it below
+state.store_info_in_SS_vars_for_any_horizon = false;
+
 
 //
 // (genuine) horizon sequence for this processor
@@ -589,6 +593,9 @@ if (strlen(surface_interpolator_name) > 0)
 								/*NOTREACHED*/
 		}
 	   else AH_data.SS_surface_number = 0;	// dummy value
+
+	state.store_info_in_SS_vars_for_any_horizon
+		|= AH_data.store_info_in_SS_vars;
 	}
 	  }
 }