Support accelerator data transfer

This patch provides a function interface for accelerator (GPU) devices
to be called at certain points when Carpet traverses the schedule
tree.  This can be used to copy data between the host (CPU) and the
device (GPU) as it is needed.
---
 Carpet/Carpet/interface.ccl       |   28 ++++++++++++++++++++++++++++
 Carpet/Carpet/param.ccl           |    6 ++++++
 Carpet/Carpet/src/CallFunction.cc |   12 ++++++++++++
 Carpet/Carpet/src/Comm.cc         |    8 ++++++++
 Carpet/Carpet/src/Cycle.cc        |    4 ++++
 Carpet/Carpet/src/Evolve.cc       |    2 ++
 Carpet/Carpet/src/Initialise.cc   |    2 ++
 Carpet/Carpet/src/Requirements.cc |   34 ++++++++++++++++++++++------------
 Carpet/Carpet/src/variables.cc    |    3 +++
 Carpet/Carpet/src/variables.hh    |    3 +++
 10 files changed, 90 insertions(+), 12 deletions(-)

10 files changed, 90 insertions, 12 deletions

diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl
index f56e3ba75..0415465de 100644
--- a/Carpet/Carpet/interface.ccl
+++ b/Carpet/Carpet/interface.ccl
@@ -193,6 +193,34 @@ USES FUNCTION MultiPatch_ConvertFromPhysicalBoundary
 
 
 
+CCTK_INT FUNCTION Accelerator_Cycle             \
+  (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION Accelerator_Cycle
+
+CCTK_INT FUNCTION Accelerator_PreCallFunction   \
+  (CCTK_POINTER_TO_CONST IN cctkGH,             \
+   CCTK_POINTER_TO_CONST IN attribute)
+USES FUNCTION Accelerator_PreCallFunction
+
+CCTK_INT FUNCTION Accelerator_PostCallFunction  \
+  (CCTK_POINTER_TO_CONST IN cctkGH,             \
+   CCTK_POINTER_TO_CONST IN attribute)
+USES FUNCTION Accelerator_PostCallFunction
+
+CCTK_INT FUNCTION Accelerator_PreSync           \
+  (CCTK_POINTER_TO_CONST IN cctkGH,             \
+   CCTK_INT ARRAY        IN groups,             \
+   CCTK_INT              IN ngroups)
+USES FUNCTION Accelerator_PreSync
+
+CCTK_INT FUNCTION Accelerator_PostSync          \
+  (CCTK_POINTER_TO_CONST IN cctkGH,             \
+   CCTK_INT ARRAY        IN groups,             \
+   CCTK_INT              IN ngroups)
+USES FUNCTION Accelerator_PostSync
+
+
+
 # Access coordinate information (on the coarse level)
 CCTK_INT FUNCTION GetCoordRange        \
     (CCTK_POINTER_TO_CONST IN  cctkGH, \
diff --git a/Carpet/Carpet/param.ccl b/Carpet/Carpet/param.ccl
index e8b0e193b..68faa78ad 100644
--- a/Carpet/Carpet/param.ccl
+++ b/Carpet/Carpet/param.ccl
@@ -558,3 +558,9 @@ INT output_timer_tree_every "Output timing information in tree form to standard
   0   :: "don't report"
   1:* :: "report every so many iterations"
 } 0
+
+
+
+BOOLEAN requirement_inconsistencies_are_fatal "Abort when encountering inconsistencies in requirements" STEERABLE=recover
+{
+} "no"
diff --git a/Carpet/Carpet/src/CallFunction.cc b/Carpet/Carpet/src/CallFunction.cc
index 936c8525d..e0550907b 100644
--- a/Carpet/Carpet/src/CallFunction.cc
+++ b/Carpet/Carpet/src/CallFunction.cc
@@ -332,8 +332,20 @@ namespace Carpet {
       
       user_timer.start();
       timer.start();
+      if (CCTK_IsFunctionAliased("Accelerator_PreCallFunction")) {
+        Timer pre_timer("PreCall");
+        pre_timer.start();
+        Accelerator_PreCallFunction(cctkGH, attribute);
+        pre_timer.stop();
+      }
       int const res = CCTK_CallFunction (function, attribute, data);
       assert (res==0);
+      if (CCTK_IsFunctionAliased("Accelerator_PostCallFunction")) {
+        Timer post_timer("PostCall");
+        post_timer.start();
+        Accelerator_PostCallFunction(cctkGH, attribute);
+        post_timer.stop();
+      }
       timer.stop();
       user_timer.stop();
       
diff --git a/Carpet/Carpet/src/Comm.cc b/Carpet/Carpet/src/Comm.cc
index 9fa2893f3..ab6106c2b 100644
--- a/Carpet/Carpet/src/Comm.cc
+++ b/Carpet/Carpet/src/Comm.cc
@@ -214,6 +214,10 @@ namespace Carpet {
     Checkpoint ("SyncGroups");
 
     assert (groups.size() > 0);
+    
+    if (CCTK_IsFunctionAliased("Accelerator_PreSync")) {
+      Accelerator_PreSync(cctkGH, &groups.front(), groups.size());
+    }
 
     for (comm_state state; not state.done(); state.step()) {
       for (int group = 0; group < (int)groups.size(); ++group) {
@@ -232,6 +236,10 @@ namespace Carpet {
         }
       }
     }
+
+    if (CCTK_IsFunctionAliased("Accelerator_PostSync")) {
+      Accelerator_PostSync(cctkGH, &groups.front(), groups.size());
+    }
   }
 
 
diff --git a/Carpet/Carpet/src/Cycle.cc b/Carpet/Carpet/src/Cycle.cc
index 89be14882..bd0acac0a 100644
--- a/Carpet/Carpet/src/Cycle.cc
+++ b/Carpet/Carpet/src/Cycle.cc
@@ -99,6 +99,10 @@ namespace Carpet {
       } // if storage
     } // for group
     
+    if (CCTK_IsFunctionAliased("Accelerator_Cycle")) {
+      Accelerator_Cycle(cctkGH);
+    }
+    
     if (errors > 0) {
       CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Errors in %d groups detected; aborting", errors);
diff --git a/Carpet/Carpet/src/Evolve.cc b/Carpet/Carpet/src/Evolve.cc
index 21647a4e3..8a11fb4ed 100644
--- a/Carpet/Carpet/src/Evolve.cc
+++ b/Carpet/Carpet/src/Evolve.cc
@@ -602,7 +602,9 @@ namespace Carpet {
               ScheduleTraverse (where, "CCTK_CHECKPOINT", cctkGH);
               
               // Analysis
+              in_analysis_bin = true;
               ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH);
+              in_analysis_bin = false;
               
               if (do_late_global_mode) {
                 // Timing statistics
diff --git a/Carpet/Carpet/src/Initialise.cc b/Carpet/Carpet/src/Initialise.cc
index 976601ad9..559d58cfe 100644
--- a/Carpet/Carpet/src/Initialise.cc
+++ b/Carpet/Carpet/src/Initialise.cc
@@ -602,7 +602,9 @@ namespace Carpet {
             ScheduleTraverse (where, "CCTK_CPINITIAL", cctkGH);
             
             // Analysis
+            in_analysis_bin = true;
             ScheduleTraverse (where, "CCTK_ANALYSIS", cctkGH);
+            in_analysis_bin = false;
             
             if (do_late_global_mode) {
               // Timing statistics
diff --git a/Carpet/Carpet/src/Requirements.cc b/Carpet/Carpet/src/Requirements.cc
index da0beaf6b..630d1057d 100644
--- a/Carpet/Carpet/src/Requirements.cc
+++ b/Carpet/Carpet/src/Requirements.cc
@@ -95,11 +95,16 @@ namespace Carpet {
     CheckEntry (void * const attribute,
                 void * const data)
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       if (not attribute) {
         // Nothing to check
         return 1;
       }
       
+      int (*const warn) (char const *thorn, char const *format, ...) =
+        requirement_inconsistencies_are_fatal ? CCTK_VParamWarn : CCTK_VInfo;
+      
       // Convert argument types
       cFunctionData & function_data =
         (static_cast <t_attribute *> (attribute))->FunctionData;
@@ -125,12 +130,12 @@ namespace Carpet {
              ri != required_but_not_provided.end(); ++ ri)
         {
           string const req = * ri;
-          CCTK_VParamWarn (CCTK_THORNSTRING,
-                           "Requirement inconsistency:\n"
-                           "   Group %s, function %s::%s requires \"%s\" which has not been provided",
-                           function_data.where,
-                           function_data.thorn, function_data.routine,
-                           req.c_str());
+          warn (CCTK_THORNSTRING,
+                "Requirement inconsistency:\n"
+                "   Group %s, function %s::%s requires \"%s\" which has not been provided",
+                function_data.where,
+                function_data.thorn, function_data.routine,
+                req.c_str());
         }
       }
       
@@ -144,11 +149,16 @@ namespace Carpet {
     CheckExit (void * const attribute,
                void * const data)
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       if (not attribute) {
         // Nothing to check
         return 1;
       }
       
+      int (*const warn) (char const *thorn, char const *format, ...) =
+        requirement_inconsistencies_are_fatal ? CCTK_VParamWarn : CCTK_VInfo;
+      
       // Convert argument types
       cFunctionData & function_data =
         (static_cast <t_attribute *> (attribute))->FunctionData;
@@ -187,12 +197,12 @@ namespace Carpet {
              pi != provided_too_often.end(); ++ pi)
         {
           string const prov = * pi;
-          CCTK_VParamWarn (CCTK_THORNSTRING,
-                           "Requirement inconsistency:\n"
-                           "   Group %s, function %s::%s provides (and does not require) \"%s\" which has already been provided",
-                           function_data.where,
-                           function_data.thorn, function_data.routine,
-                           prov.c_str());
+          warn (CCTK_THORNSTRING,
+                "Requirement inconsistency:\n"
+                "   Group %s, function %s::%s provides (and does not require) \"%s\" which has already been provided",
+                function_data.where,
+                function_data.thorn, function_data.routine,
+                prov.c_str());
         }
       }
       
diff --git a/Carpet/Carpet/src/variables.cc b/Carpet/Carpet/src/variables.cc
index 1de61db65..1e1b31b06 100644
--- a/Carpet/Carpet/src/variables.cc
+++ b/Carpet/Carpet/src/variables.cc
@@ -106,6 +106,9 @@ namespace Carpet {
   // Should we warn about groups with insufficiently many time levels?
   bool do_warn_about_storage;
   
+  // Are we in the analysis bin?
+  bool in_analysis_bin;
+  
   
   
   // Data for grid functions
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index 778f6f516..707e6f1ae 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -147,6 +147,9 @@ namespace Carpet {
   // Should we warn about groups with insufficiently many time levels?
   extern bool do_warn_about_storage;
   
+  // Are we in the analysis bin?
+  extern bool in_analysis_bin;
+  
   
   
   // Data for grid functions