Hyperslab mappings can now be defined to include periodic boundary points.

This is necessary for PUGH-based checkpointing/recovery.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGHSlab/trunk@116 10716dce-81a3-4424-a2c8-48026a0d3035

1 files changed, 55 insertions, 15 deletions

diff --git a/src/Mapping.c b/src/Mapping.c
index bd6efdf..4cd0c94 100644
--- a/src/Mapping.c
+++ b/src/Mapping.c
@@ -13,6 +13,7 @@
 #include <string.h>
 
 #include "cctk.h"
+#include "util_Table.h"
 
 #include "CactusPUGH/PUGH/src/include/pugh.h"
 
@@ -122,14 +123,14 @@ static CCTK_INT DefineMapping (const cGH      *GH,
                                const CCTK_INT *origin,        /* vdim */
                                const CCTK_INT *extent,        /* hdim */
                                const CCTK_INT *downsample,    /* hdim */
-                               CCTK_INT        table_handle,
+                               CCTK_INT        table,
                                t_hslabConversionFn function,
                                int             is_global_hyperslab,
                                CCTK_INT       *hsize_local,   /* hdim */
                                CCTK_INT       *hsize_global,  /* hdim */
                                CCTK_INT       *hoffset_global /* hdim */)
 {
-  int vdim, dim, num_dirs;
+  int vdim, dim, num_dirs, iterator;
   int retval;
   int myproc, nprocs;
   int i, j, k, npoints;
@@ -141,6 +142,8 @@ static CCTK_INT DefineMapping (const cGH      *GH,
   const pGA *GA;                  /* the variable's GA structure from PUGH */
   const pGExtras *extras;
   cGroup vinfo;
+  char key[128];
+  CCTK_INT type, nelems, with_ghostzones;
 
 
   /* prevent compiler warnings */
@@ -151,13 +154,47 @@ static CCTK_INT DefineMapping (const cGH      *GH,
   myproc = CCTK_MyProc (GH);
   nprocs = CCTK_nProcs (GH);
 
-  /* PUGHSlab doesn't use table information */
-  if (table_handle >= 0)
+  /* parse the table information if there is any */
+  with_ghostzones = 0;
+  if (table >= 0)
   {
-    CCTK_WARN (1, "PUGHlab_DefineGlobalMappingByIndex: table information is "
-                  "ignored");
+    /* warn about other options */
+    iterator = Util_TableItCreate (table);
+    for (iterator = Util_TableItCreate (table);
+         Util_TableItQueryIsNonNull (iterator) > 0 &&
+         Util_TableItQueryKeyValueInfo (iterator, sizeof (key),
+                                        key, &type, &nelems) > 0;
+         Util_TableItAdvance (iterator))
+    {
+      if (CCTK_Equals (key, "with_ghostzones"))
+      {
+        if (type == CCTK_VARIABLE_INT && nelems == 1)
+        {
+          Util_TableGetInt (table, &with_ghostzones, "with_ghostzones");
+        }
+        else
+        {
+          CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                      "Invalid value for key 'with_ghostzones' in table "
+                      "(must be an integer). Key/value pair will be ignored.");
+        }
+      }
+      else
+      {
+        CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                    "Found unrecognized key '%s' in table. Key/value pair will "
+                    "be ignored.", key);
+      }
+    }
+    Util_TableItDestroy (iterator);
   }
 
+#ifdef DEBUG
+  fprintf (stderr, "proc %d: mapping will %sinclude ghostzones\n",
+           CCTK_MyProc (GH), with_ghostzones ? "" : "not ");
+  CCTK_Barrier (GH);
+#endif
+
   /* check parameter consistency */
   retval = 0;
   error_msg = NULL;
@@ -366,7 +403,7 @@ static CCTK_INT DefineMapping (const cGH      *GH,
         (extent[dim] + mapping->downsample[vdim]-1) / mapping->downsample[vdim];
 
       /* subtract ghostzones for periodic BC */
-      if (GA->connectivity->perme[vdim])
+      if (! with_ghostzones && GA->connectivity->perme[vdim])
       {
         mapping->global_hsize[dim] -= 2 * extras->nghostzones[vdim];
       }
@@ -378,7 +415,7 @@ static CCTK_INT DefineMapping (const cGH      *GH,
         (extent[0] + mapping->downsample[0]-1) / mapping->downsample[0];
 
       /* subtract ghostzones for periodic BC */
-      if (GA->connectivity->perme[vdim])
+      if (! with_ghostzones && GA->connectivity->perme[vdim])
       {
         mapping->totals -= 2 * extras->nghostzones[vdim];
       }
@@ -442,8 +479,10 @@ static CCTK_INT DefineMapping (const cGH      *GH,
 
       my_local_startpoint = extras->ownership[i][0][vdim];
       my_local_endpoint   = extras->ownership[i][1][vdim];
-      /* local hyperslabs include ghostzones, global ones don't */
-      if (! is_global_hyperslab)
+
+      /* include ghostzones if explicitely requested,
+         otherwise only for local hyperslabs */
+      if (with_ghostzones || ! is_global_hyperslab)
       {
         my_local_startpoint -= extras->nghostzones[vdim];
         my_local_endpoint   += extras->nghostzones[vdim];
@@ -471,10 +510,11 @@ static CCTK_INT DefineMapping (const cGH      *GH,
     {
       i = CCTK_StaggerDirIndex (vdim, vinfo.stagtype);
 
-      /* non-periodic local hyperslabs also include ghostzones */
+      /* only non-periodic local hyperslabs include ghostzones by default */
       my_local_startpoint = extras->ownership[i][0][vdim];
       my_local_endpoint   = extras->ownership[i][1][vdim];
-      if (! is_global_hyperslab && ! GA->connectivity->perme[vdim])
+      if (with_ghostzones ||
+          ! (is_global_hyperslab || GA->connectivity->perme[vdim]))
       {
         my_local_startpoint -= extras->nghostzones[vdim];
         my_local_endpoint   += extras->nghostzones[vdim];
@@ -501,8 +541,8 @@ static CCTK_INT DefineMapping (const cGH      *GH,
       }
 
 #ifdef DEBUG
-      fprintf (stderr, "direction %d: global ranges [%d, %d), local ranges[%d, "
-               "%d)\n", vdim,
+      fprintf (stderr, "direction %d: global ranges [%d, %d), "
+               "local ranges [%d, %d)\n", vdim,
                mapping->global_startpoint[vdim], mapping->global_endpoint[vdim],
                mapping->local_startpoint[vdim], mapping->local_endpoint[vdim]);
 #endif
@@ -545,7 +585,7 @@ static CCTK_INT DefineMapping (const cGH      *GH,
         {
           mapping->global_hoffset[dim] = (mapping->global_startpoint[vdim] -
                                           origin[vdim]) / mapping->downsample[vdim];
-          if (GA->connectivity->perme[vdim])
+          if (! with_ghostzones && GA->connectivity->perme[vdim])
           {
             if (mapping->global_hoffset[dim] < extras->nghostzones[vdim])
             {