Get the MPI communicator more intelligently: use aliased function

GetMPICommWorld if it is provided, otherwise fall back to using
Carpet- and PUGH- specific methods, otherwise use MPI_COMM_WORLD.

Don't disable (i.e., re-enable) some assert checks even if NDEBUG is
defined.  These are cheap checks that check the arguments that are
passed in.  Otherwise errors would go undetected.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusNumerical/Slab/trunk@43 2e825fa2-fb71-486d-8b7f-a5ff3f0f6cb8

2 files changed, 85 insertions, 55 deletions

diff --git a/interface.ccl b/interface.ccl
index 171bc4a..100ed3a 100644
--- a/interface.ccl
+++ b/interface.ccl
@@ -5,3 +5,6 @@ IMPLEMENTS: Slab
 
 INCLUDES HEADER: slab.h IN Slab.h
 INCLUDES HEADER: slab.inc IN Slab.inc
+
+CCTK_INT FUNCTION GetMPICommWorld (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION GetMPICommWorld
diff --git a/src/slab.c b/src/slab.c
index b61f48b..3fec618 100644
--- a/src/slab.c
+++ b/src/slab.c
@@ -34,6 +34,8 @@
 
 #include "cctk.h"
 #include "cctk_DefineThorn.h"
+#include "util_ErrorCodes.h"
+#include "util_Table.h"
 
 #ifdef CCTK_MPI
 #  include <mpi.h>
@@ -61,6 +63,12 @@ CCTK_FILEVERSION(TAT_Slab_slab_c);
 #  define ifcheck while (0)
 #endif
 
+#ifdef NDEBUG
+#  define check(x) ((x) ? 0 : CCTK_WARN (0, "internal error"))
+#else
+#  define check(x) assert (x)
+#endif
+
 
 
 static int timer_init;
@@ -88,7 +96,7 @@ void Slab_PrintTimers (void)
 
 /* Find out which driver to use */
 #ifdef CCTK_MPI
-#  if defined CARPET_CARPET || defined CARPETORIG_CARPET
+#  if defined CARPET_CARPET
 #    include "Carpet/Carpet/src/carpet_public.h"
 #  endif
 #  if defined CACTUSPUGH_PUGH
@@ -293,7 +301,10 @@ static MPI_Comm
 get_mpi_comm (const cGH * restrict const cctkGH)
 {
 #ifdef CCTK_MPI
-#  if defined CARPET_CARPET || defined CARPETORIG_CARPET
+  if (CCTK_IsFunctionAliased ("GetMPICommWorld")) {
+    return GetMPICommWorld (cctkGH);
+  }
+#  if defined CARPET_CARPET
   {
     static int Carpet_active = -1;
     if (Carpet_active == -1) Carpet_active = CCTK_IsThornActive ("Carpet");
@@ -659,43 +670,45 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   int d;
   
   /* Check arguments */
-  assert (cctkGH);
-  assert (dim >= 0);
-  assert (xferinfo);
-  assert (nvars >= 0);
-  assert (nvars==0 || srctypes);
-  for (var=0; var<nvars; ++var) assert (srctypes[var] >= 0);
-  assert (nvars==0 || srcptrs);
-/*   for (var=0; var<nvars; ++var) assert (srcptrs[var]); */
-  assert (nvars==0 || dsttypes);
-  for (var=0; var<nvars; ++var) assert (dsttypes[var] >= 0);
-  assert (nvars==0 || dstptrs);
-/*   for (var=0; var<nvars; ++var) assert (dstptrs[var]); */
+  check (cctkGH);
+  check (dim >= 0);
+  check (xferinfo);
+  check (nvars >= 0);
+  check (nvars==0 || srctypes);
+  for (var=0; var<nvars; ++var) check (srctypes[var] >= 0);
+  check (nvars==0 || srcptrs);
+/*   for (var=0; var<nvars; ++var) check (srcptrs[var]); */
+  check (nvars==0 || dsttypes);
+  for (var=0; var<nvars; ++var) check (dsttypes[var] >= 0);
+  check (nvars==0 || dstptrs);
+/*   for (var=0; var<nvars; ++var) check (dstptrs[var]); */
+  
+  if (nvars==0) return 0;
   
   CCTK_TimerStartI (timer_init);
   
-  assert (dim <= SLAB_MAXDIM);
+  check (dim <= SLAB_MAXDIM);
   info = malloc (SLAB_MAXDIM * sizeof *info);
-  assert (info);
+  check (info);
   for (d=0; d<dim; ++d) {
     global2bbox (&xferinfo[d].src, &info[d].src.global);
     local2bbox  (&xferinfo[d].src, &info[d].src.local);
     active2bbox (&xferinfo[d].src, &info[d].src.active, 0);
     slab2bbox   (&xferinfo[d].src, &info[d].src.slab);
-    assert (bbox_iscontained (&info[d].src.active, &info[d].src.local));
-    assert (bbox_iscontained (&info[d].src.local, &info[d].src.global));
+    check (bbox_iscontained (&info[d].src.active, &info[d].src.local));
+    check (bbox_iscontained (&info[d].src.local, &info[d].src.global));
     
     global2bbox (&xferinfo[d].dst, &info[d].dst.global);
     local2bbox  (&xferinfo[d].dst, &info[d].dst.local);
     active2bbox (&xferinfo[d].dst, &info[d].dst.active, 1);
     slab2bbox   (&xferinfo[d].dst, &info[d].dst.slab);
-    assert (bbox_iscontained (&info[d].dst.active, &info[d].dst.local));
-    assert (bbox_iscontained (&info[d].dst.local, &info[d].dst.global));
+    check (bbox_iscontained (&info[d].dst.active, &info[d].dst.local));
+    check (bbox_iscontained (&info[d].dst.local, &info[d].dst.global));
     
     info[d].xpose = xferinfo[d].xpose;
-    assert (info[d].xpose >= 0 && info[d].xpose < dim);
+    check (info[d].xpose >= 0 && info[d].xpose < dim);
     info[d].flip = xferinfo[d].flip;
-    assert (info[d].flip == 0 || info[d].flip == 1);
+    check (info[d].flip == 0 || info[d].flip == 1);
   }
   for (d=dim; d<SLAB_MAXDIM; ++d) {
     static struct bbox const fake_bbox = { 0, 1, 1 };
@@ -705,17 +718,17 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
     bbox_check (&fake_bbox);
     
     info[d].src = fake_arrays;
-    assert (bbox_iscontained (&info[d].src.active, &info[d].src.local));
-    assert (bbox_iscontained (&info[d].src.local, &info[d].src.global));
+    check (bbox_iscontained (&info[d].src.active, &info[d].src.local));
+    check (bbox_iscontained (&info[d].src.local, &info[d].src.global));
     
     info[d].dst = fake_arrays;
-    assert (bbox_iscontained (&info[d].dst.active, &info[d].dst.local));
-    assert (bbox_iscontained (&info[d].dst.local, &info[d].dst.global));
+    check (bbox_iscontained (&info[d].dst.active, &info[d].dst.local));
+    check (bbox_iscontained (&info[d].dst.local, &info[d].dst.global));
     
     info[d].xpose = d;
-    assert (info[d].xpose >= 0 && info[d].xpose < SLAB_MAXDIM);
+    check (info[d].xpose >= 0 && info[d].xpose < SLAB_MAXDIM);
     info[d].flip = 0;
-    assert (info[d].flip == 0 || info[d].flip == 1);
+    check (info[d].flip == 0 || info[d].flip == 1);
   }
   
   ifcheck {
@@ -786,7 +799,21 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   
   
   
-  comm = get_mpi_comm (cctkGH);
+  {
+    CCTK_INT tmp;
+    int const iret = Util_TableGetInt (options, &tmp, "comm");
+    if (iret == 1) {
+      /* There was an entry, use it */
+      comm = tmp;
+    } else if (iret == UTIL_ERROR_BAD_HANDLE
+               || iret == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
+      /* There was no entry, use a default */
+      comm = get_mpi_comm (cctkGH);
+    } else {
+      /* Something went wrong, abort */
+      check (0);
+    }
+  }
   
   ifcheck {
     ifdebug fflush (stdout);
@@ -807,27 +834,27 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
     ++ count;
   }
   
-  assert (nvars >= 1);
+  check (nvars >= 1);
   srctype = srctypes[0];
   dsttype = dsttypes[0];
   for (var=0; var<nvars; ++var) {
-    assert (srctypes[var] == srctype);
-    assert (dsttypes[var] == dsttype);
+    check (srctypes[var] == srctype);
+    check (dsttypes[var] == dsttype);
   }
-  assert (srctype == dsttype);
+  check (srctype == dsttype);
   srctypesize = CCTK_VarTypeSize (srctype);
-  assert (srctypesize > 0);
+  check (srctypesize > 0);
   dsttypesize = CCTK_VarTypeSize (dsttype);
-  assert (dsttypesize > 0);
+  check (dsttypesize > 0);
   srcdatatype = mpi_type (srctype);
-  assert (srcdatatype >= 0);
+  check (srcdatatype >= 0);
   dstdatatype = mpi_type (dsttype);
-  assert (dstdatatype >= 0);
+  check (dstdatatype >= 0);
   
   
   
   allinfo = malloc (size * SLAB_MAXDIM * sizeof *allinfo);
-  assert (allinfo);
+  check (allinfo);
   {
     int const info_nints = sizeof(struct info) / sizeof(int);
     ifdebug fflush (stdout);
@@ -872,7 +899,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   
   
   srcdetail = malloc (size * SLAB_MAXDIM * sizeof *srcdetail);
-  assert (srcdetail);
+  check (srcdetail);
   for (n = 0; n < size; ++n) {
     ifdebug printf ("srcdetail n=%d:\n", n);
     for (d=0; d<SLAB_MAXDIM; ++d) {
@@ -912,11 +939,11 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   }
   
   srcelems = malloc (size * sizeof *srcelems);
-  assert (srcelems);
+  check (srcelems);
   srccount = malloc (size * sizeof *srccount);
-  assert (srccount);
+  check (srccount);
   srcoffset = malloc ((size + 1) * sizeof *srcoffset);
-  assert (srcoffset);
+  check (srcoffset);
   srcoffset[0] = 0;
   for (n = 0; n < size; ++n) {
     srcelems[n] = 1;
@@ -929,7 +956,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
     srcoffset[n+1] = srcoffset[n] + srccount[n];
   }
   srcdata = malloc (srcoffset[size] * srctypesize);
-  assert (nvars==0 || srcdata);
+  check (nvars==0 || srcdata);
   ifcheck {
     if (srctype == CCTK_VARIABLE_REAL) {
       CCTK_REAL * restrict const srcdataptr = srcdata;
@@ -942,7 +969,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   }
   
   dstdetail = malloc (size * SLAB_MAXDIM * sizeof *dstdetail);
-  assert (dstdetail);
+  check (dstdetail);
   for (n = 0; n < size; ++n) {
     ifdebug printf ("dstdetail n=%d:\n", n);
     for (d=0; d<SLAB_MAXDIM; ++d) {
@@ -984,11 +1011,11 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   }
   
   dstelems = malloc (size * sizeof *dstelems);
-  assert (dstelems);
+  check (dstelems);
   dstcount = malloc (size * sizeof *dstcount);
-  assert (dstcount);
+  check (dstcount);
   dstoffset = malloc ((size + 1) * sizeof *dstoffset);
-  assert (dstoffset);
+  check (dstoffset);
   dstoffset[0] = 0;
   for (n = 0; n < size; ++n) {
     dstelems[n] = 1;
@@ -1001,7 +1028,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
     dstoffset[n+1] = dstoffset[n] + dstcount[n];
   }
   dstdata = malloc (dstoffset[size] * dsttypesize);
-  assert (nvars==0 || dstdata);
+  check (nvars==0 || dstdata);
   ifcheck {
     if (dsttype == CCTK_VARIABLE_REAL) {
       CCTK_REAL * restrict const dstdataptr = dstdata;
@@ -1013,21 +1040,21 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
     }
   }
   
-  assert (srccount[rank] == dstcount[rank]);
+  check (srccount[rank] == dstcount[rank]);
   
   ifcheck {
     int * restrict src2count;
     int * restrict dst2count;
     src2count = malloc (size * sizeof *src2count);
-    assert (src2count);
+    check (src2count);
     dst2count = malloc (size * sizeof *dst2count);
-    assert (dst2count);
+    check (dst2count);
     ifdebug fflush (stdout);
     MPI_Alltoall (srccount, 1, MPI_INT, src2count, 1, MPI_INT, comm);
     MPI_Alltoall (dstcount, 1, MPI_INT, dst2count, 1, MPI_INT, comm);
     for (n = 0; n < size; ++n) {
-      assert (src2count[n] == dstcount[n]);
-      assert (dst2count[n] == srccount[n]);
+      check (src2count[n] == dstcount[n]);
+      check (dst2count[n] == srccount[n]);
     }
     free (src2count);
     free (dst2count);
@@ -1040,7 +1067,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   CCTK_TimerStartI (timer_copy_in);
   
   for (n = 0; n < size; ++n) {
-    assert (SLAB_MAXDIM == 3);
+    check (SLAB_MAXDIM == 3);
     
     if (info[0].xpose==0 && info[1].xpose==1 && info[2].xpose==2
         && srcdetail[n*SLAB_MAXDIM  ].str==1 && srcdetail[n*SLAB_MAXDIM+1].str==1 && srcdetail[n*SLAB_MAXDIM+2].str==1
@@ -1254,7 +1281,7 @@ int Slab_MultiTransfer (cGH             const         * const cctkGH,
   
   CCTK_TimerStartI (timer_copy_back);
   for (n = 0; n < size; ++n) {
-    assert (SLAB_MAXDIM == 3);
+    check (SLAB_MAXDIM == 3);
     
     if (info[0].flip==0 && info[1].flip==0 && info[2].flip==0
         && dstdetail[n*SLAB_MAXDIM  ].str==1 && dstdetail[n*SLAB_MAXDIM+1].str==1 && dstdetail[n*SLAB_MAXDIM+2].str==1