switch mask processing from being done at the tail of

AHFinderDirect_find_horizons() (which is scheduled with options:GLOBAL)
into a separately scheduled routine, so we can schedle this without
options:GLOBAL -- we want to set the mask on each refined subgrid!


git-svn-id: http://svn.einsteintoolkit.org/cactus/EinsteinAnalysis/AHFinderDirect/trunk@1149 f88db872-0e4f-0410-b76b-b9085cfa78c5

4 files changed, 64 insertions, 25 deletions

diff --git a/src/CODESTYLE b/src/CODESTYLE
index 4e7439b..dbbc9c3 100644
--- a/src/CODESTYLE
+++ b/src/CODESTYLE
@@ -1,6 +1,6 @@
 AHFinderDirect Code Style
 =========================
-$Header: /usr/local/svn/cvs-repositories/numrelcvs/AEIThorns/AHFinderDirect/src/CODESTYLE,v 1.4 2003-06-23 19:38:34 jthorn Exp $
+$Header: /usr/local/svn/cvs-repositories/numrelcvs/AEIThorns/AHFinderDirect/src/CODESTYLE,v 1.5 2003-07-27 14:02:53 jthorn Exp $
 
 This file documents some general programming conventions used in this
 thorn.
@@ -240,3 +240,10 @@ names, contain the substring "__" (two consecutive underscores).
 According to the C++ standard, such names are reserved to the [compiler]
 implementation, so this code isn't strictly legal.  In practice, so
 far I haven't had any problems...
+
+Each .cc source file should begin with a "table of comments" block
+comment listing all the functions defined in the file.  Functions with
+a "///" comment (or "**" for C) are local to the file; functions with
+a "//" comment (or "*" for C) are visible to the linker.
+//
+// 
diff --git a/src/driver/driver.hh b/src/driver/driver.hh
index 48d26c5..fbe081a 100644
--- a/src/driver/driver.hh
+++ b/src/driver/driver.hh
@@ -353,6 +353,11 @@ extern "C"
 extern "C"
   void AHFinderDirect_find_horizons(CCTK_ARGUMENTS);
 
+// mask.cc
+// ... called from Cactus Scheduler
+extern "C"
+  void AHFinderDirect_do_masks(CCTK_ARGUMENTS);
+
 // initial_guess.cc
 void setup_initial_guess(patch_system& ps,
 			 const struct initial_guess_info& igi,
@@ -397,15 +402,6 @@ void output_Jacobians(const patch_system& ps,
 		      const struct IO_info& IO_info, const char base_file_name[],
 		      int hn, bool print_msg_flag, int AHF_iteration = 0);
 
-// mask.cc
-void setup_mask_dataptrs_and_bitfields(const cGH *GH,
-				       struct mask_info& mask_info);
-void set_mask_gridfn(int N_horizons,
-		     const struct AH_data* const AH_data_array[],
-		     const struct cactus_grid_info& cgi,
-		     const struct mask_info& mask_info,
-		     const struct verbose_info& verbose_info);
-
 // misc-driver.cc
 int Cactus_gridfn_varindex(const char gridfn_name[]);
 template <typename T>
diff --git a/src/driver/find_horizons.cc b/src/driver/find_horizons.cc
index 3464177..e8336db 100644
--- a/src/driver/find_horizons.cc
+++ b/src/driver/find_horizons.cc
@@ -83,8 +83,7 @@ void do_test_expansion_Jacobians(int my_proc, int N_horizons,
 
 //
 // This function is called by the Cactus scheduler to find the apparent
-// horizon or horizons in the current slice, and optionally set a mask
-// grid function (or functions) based on each horizon shape.
+// horizon or horizons in the current slice.
 //
 extern "C"
   void AHFinderDirect_find_horizons(CCTK_ARGUMENTS)
@@ -98,12 +97,12 @@ if (state.timer_handle >= 0)
 const int my_proc = state.my_proc;
 horizon_sequence& hs = *state.my_hs;
 const bool active_flag = hs.has_genuine_horizons();
+const bool broadcast_horizon_shape = state.mask_info.set_mask;
 
       struct cactus_grid_info&          cgi = state.cgi;
 const struct    geometry_info&           gi = state.gi;
       struct    Jacobian_info&     Jac_info = state.Jac_info;
       struct          IO_info&      IO_info = state.IO_info;
-      struct        mask_info&    mask_info = state.mask_info;
 const struct       error_info&   error_info = state.error_info;
 const struct     verbose_info& verbose_info = state.verbose_info;
 
@@ -176,7 +175,6 @@ case method__find_horizons:
 	  {
 	if (state.timer_handle >= 0)
 	   then CCTK_TimerStartI(state.timer_handle);
-	const bool broadcast_horizon_shape = mask_info.set_mask;
 	Newton(cctkGH,
 	       state.N_procs, state.N_active_procs, my_proc,
 	       *state.my_hs, state.AH_data_array,
@@ -204,15 +202,6 @@ if (state.timer_handle >= 0)
 		   "timer stats for computation:");
 	CCTK_TimerPrintDataI(state.timer_handle, -1);
 	}
-
-// optionally set the mask gridfn based on each horizon's shape
-if (mask_info.set_mask)
-   then {
-	setup_mask_dataptrs_and_bitfields(cctkGH, mask_info);
-	set_mask_gridfn(N_horizons, state.AH_data_array,
-			cgi, mask_info,
-			verbose_info);
-	}
 }
 
 //******************************************************************************
diff --git a/src/driver/mask.cc b/src/driver/mask.cc
index 600f9ba..04a0b57 100644
--- a/src/driver/mask.cc
+++ b/src/driver/mask.cc
@@ -1,9 +1,11 @@
 // mask.cc -- set a mask gridfn based on each horizon shape
 // $Header$
 //
+// <<<access to persistent data>>>
 // <<<prototypes for functions local to this file>>>
-// setup_mask_dataptrs_and_bitfields - map gridfn/bitfield names to ptr/bitmasks
-// set_mask_gridfn - set mask gridfn(s) based on each horizon's shape
+// AHFinderDirect_do_masks - top-level driver for all mask stuff
+/// setup_mask_dataptrs_and_bitfields - map gridfn/bitfield names to ptr/bitmask
+/// set_mask_gridfn - set mask gridfn(s) based on each horizon's shape
 /// set_mask_gridfn_to_outside_value - ... "outside" value
 /// set_mask_gridfn_to_inside_and_buffer_values - "inside"/"buffer" values
 //
@@ -48,10 +50,25 @@ using jtutil::error_exit;
 //******************************************************************************
 
 //
+// ***** access to persistent data *****
+//
+extern struct state state;
+
+//******************************************************************************
+
+//
 // prototypes for functions local to this file
 //
 
 namespace {
+void setup_mask_dataptrs_and_bitfields(const cGH *GH,
+				       struct mask_info& mask_info);
+void set_mask_gridfn(int N_horizons,
+		     const struct AH_data* const AH_data_array[],
+		     const struct cactus_grid_info& cgi,
+		     const struct mask_info& mask_info,
+		     const struct verbose_info& verbose_info);
+
 void set_mask_gridfn_to_outside_value(const struct cactus_grid_info& cgi,
 				      const struct mask_info& mask_info,
 				      const struct verbose_info& verbose_info);
@@ -68,6 +85,32 @@ void set_mask_gridfn_to_inside_and_buffer_values
 //******************************************************************************
 
 //
+// This function is called by the Cactus scheduler after we find any
+// apparent horizons, to do all this thorn's mask processing.
+//
+extern "C"
+  void AHFinderDirect_do_masks(CCTK_ARGUMENTS)
+{
+DECLARE_CCTK_ARGUMENTS
+DECLARE_CCTK_PARAMETERS
+
+const struct cactus_grid_info&          cgi = state.cgi;
+const struct     verbose_info& verbose_info = state.verbose_info;
+      struct        mask_info&    mask_info = state.mask_info;
+
+// optionally set the mask gridfn based on each horizon's shape
+if (mask_info.set_mask)
+   then {
+	setup_mask_dataptrs_and_bitfields(cctkGH, mask_info);
+	set_mask_gridfn(N_horizons, state.AH_data_array,
+			cgi, mask_info,
+			verbose_info);
+	}
+}
+
+//******************************************************************************
+
+//
 // This function maps the character-string names of the mask gridfn(s)
 // and/or bitfield(s) into internal data pointers and/or bit masks/values:
 //
@@ -80,6 +123,7 @@ void set_mask_gridfn_to_inside_and_buffer_values
 //		 .buffer_value    --> .buffer_bitvalue
 //		 .outside_value   --> .outside_bitvalue
 //
+namespace {
 void setup_mask_dataptrs_and_bitfields(const cGH *GH,
 				       struct mask_info& mask_info)
 {
@@ -136,6 +180,7 @@ if (mask_info.set_new_style_mask)
 			   nsmi.outside_value);			/*NOTREACHED*/
 	}
 }
+	  }
 
 //******************************************************************************
 
@@ -143,6 +188,7 @@ if (mask_info.set_new_style_mask)
 // This function sets the mask grid function specified by  mask_info
 // based on each horizon's shape.
 //
+namespace {
 void set_mask_gridfn(int N_horizons,
 		     const struct AH_data* const AH_data_array[],
 		     const struct cactus_grid_info& cgi,
@@ -291,6 +337,7 @@ set_mask_gridfn_to_outside_value(cgi,
 						    verbose_info);
 	}
 }
+	  }
 
 //******************************************************************************