Changes from Dylan Stark to allow registration of topology

routines and introducing a physical to logical processor
mapping.

Now it is possible to register custom topology routines if
you don't like the two ones we have at the moment (the
traditional one and the new one from Frank Loeffler).

You can also provide routines to optimise the placement of
neighbouring processes on the processors by giving a custom
mapping from the numbers PUGH uses for the processors to
the actual MPI processor.

Please treat this registration as semi-volatile for the moment
and coordinate with Dylan if you fancy using it.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@475 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 20 insertions, 298 deletions

diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 93a2d56..149eaa0 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -63,9 +63,6 @@ static int PUGH_SetupPGExtrasSizes(int is_gf,
                                    pGExtras *this);
 static int PUGH_ComposeIJK(int dim, int *nprocs, int *pos);
 static int PUGH_DecomposeIJK(int dim, int ijk, int *nprocs, int *pos);
-static int PUGH_IntegerRoot(int number, int invpower);
-
-static int IntSort(const void *a, const void *b);
 
 /********************************************************************
  ********************* Other Routine Prototypes *********************
@@ -241,15 +238,9 @@ pConnectivity *PUGH_SetupConnectivity(int dim,
       this->neighbours[i] = this->neighbours[0]+(2*dim*i);
     }
 
-    if (CCTK_EQUALS(processor_topology, "automatic"))
-    {
-      PUGH_GenerateAlternativeTopology(dim, total_procs, nsize, nghostzones,
-                                       this->nprocs);
-    }
-    else
-    {
-      PUGH_GenerateTopology(dim, total_procs, this->nprocs);
-    }
+    PUGHi_GenerateTopology(dim, total_procs, nsize, nghostzones, 
+                             this->nprocs);
+
     PUGH_GenerateNeighbours(dim, total_procs, this->nprocs, this->neighbours,
                             this->perme);
   }
@@ -281,211 +272,6 @@ void PUGH_DestroyConnectivity(pConnectivity **conn)
 }
 
  /*@@
-   @routine    PUGH_GenerateTopology
-   @date       Fri Nov  5 11:31:21 1999
-   @author     Tom Goodale
-   @desc
-   Generate the appropriate processor topology for this processor
-   decomposition.
-   @enddesc
-   @history
-   @hdate Tue Jan 30 17:04:50 2001 @hauthor Tom Goodale
-   @hdesc Added call to integer root function and qsort to
-          avoid problems with real to integer conversions and
-          demonstrable failure of the algorithm when dealing
-          with large prime factors.
-   @endhistory
-@@*/
-int PUGH_GenerateTopology(int dim, int total_procs, int *nprocs)
-{
-  int i;
-  int used_procs;
-  int free_procs;
-  int retval;
-  int free_dims;
-
-  used_procs = 0;
-  free_procs = total_procs;
-
-  retval = 0;
-
-  free_dims = dim;
-
-  for(i=0; i < dim; i++)
-  {
-    if((nprocs[i])>0)
-    {
-      free_dims--;
-      if(used_procs)
-      {
-        used_procs *= nprocs[i];
-      }
-      else
-      {
-        used_procs = nprocs[i];
-      }
-      if (total_procs%used_procs)
-      {
-        CCTK_WARN(0, "Inconsistent PUGH topology");
-        retval = 1;
-      }
-      else
-      {
-        free_procs = total_procs/used_procs;
-      }
-    }
-  }
-
-  /* Ok calculate topology if necessary */
-  if(free_dims && ! retval)
-  {
-    /* This algorithm gives the most number of processors
-     * in the highest dimension.
-     */
-
-    int *working;
-    int root;
-    int place;
-
-    root = free_dims;
-    working = calloc(free_dims,sizeof(int));
-#ifdef DEBUG_PUGH
-    printf("Processor topology for dimension %d\n",dim);
-#endif
-
-
-    for(i = 0; i < free_dims  ; i++)
-    {
-      working[i] = PUGH_IntegerRoot(free_procs, root);
-
-      while(free_procs % working[i]) working[i]--;
-
-#ifdef DEBUG_PUGH
-      printf(" working[%d] = %d\n",i,working[i]);
-#endif
-      free_procs /= working[i];
-      root--;
-    }
-
-
-    /* The above doesn't necessarily sort them properly
-     * e.g. if one of the factors is a prime then the
-     * above will sort the 1 before the prime.
-     */
-    qsort(working,free_dims,sizeof(int),IntSort);
-
-    for(i = 0,place=0; i < dim ; i++)
-    {
-      if(nprocs[i] <= 0)
-      {
-        nprocs[i] = working[place];
-        place++;
-      }
-
-#ifdef DEBUG_PUGH
-      printf(" nprocs[%d] = %d\n",i,nprocs[i]);
-#endif
-    }
-
-    free(working);
-  }
-
-  return retval;
-}
-
-
-
- /*@@
-   @routine    PUGH_GenerateAlternativeTopology
-   @date       Thu Feb 02 17:39:21 2005
-   @author     Frank Loeffler
-   @desc
-   Generate the appropriate processor topology for this processor
-   decomposition.
-   This routine tries to decompose using the information about the actual
-   grid size. It can fail in certain situations in which it falls back
-   gracefully to the traditional decomposition giving a warning.
-   It also does not support manually set topologies and falls back in this
-   case.
-   @enddesc
-   @history
-   @endhistory
-@@*/
-int PUGH_GenerateAlternativeTopology(int dim,
-                                     int total_procs, const int *nsize,
-                                     const int *nghostzones, int *nprocs)
-{
-  int i;
-  int max_dir, max_length;
-  int free_procs = total_procs-1;
-  int used_procs = 1;
-  
-  
-  /* Nothing to decompose here */
-  if (dim == 0)
-  {
-    return PUGH_GenerateTopology(dim, total_procs, nprocs);
-  }
-  /* If there are numbers already set up or we have funny grid sizes,
-     fall back */
-  for (i = 0; i < dim; i++)
-  {
-    if (nprocs[i] || (nsize[i]<1))
-    {
-      return PUGH_GenerateTopology(dim, total_procs, nprocs);
-    }
-  }
-  /* start with a single domain */
-  for (i = 0; i < dim; i++)
-  {
-    nprocs[i] = 1;
-  }
-  /* divide as long as there are processors left */
-  while (free_procs)
-  {
-    used_procs = total_procs - free_procs;
-    /* find the direction with the longest length, which is allowed */
-    max_dir = -1;
-    max_length = 0;
-    for (i = 0; i < dim; i++)
-    {
-      /* is one part larger than the max? (then it might be a new max) */
-      if ((nsize[i] / nprocs[i] > max_length) &&
-          /* would there be at least one real point if we divide? */
-          (nsize[i] > 2*nghostzones[i]+nprocs[i]) &&
-          /* do we have enough processors left to divide in this direction? */
-          (used_procs/nprocs[i] <= free_procs))
-      {
-        max_length = nsize[i] / nprocs[i];
-        max_dir = i;
-      }
-    }
-    /* if no such direction is found: fall back giving a warning */
-    if (max_dir == -1)
-    {
-      for (i = 0; i < dim; i++)
-      {
-        nprocs[i] = 0;
-      }
-      CCTK_WARN(CCTK_WARN_COMPLAIN,
-                "Falling back to the old PUGH topology method, overwriting "
-                "eventually set manual topologies");
-      return PUGH_GenerateTopology(dim, total_procs, nprocs);
-    }
-    /* count the new direction and go on */
-    /* note: this is garanteed to decrement at least by one, since the number
-       of used processors is naturally always >= the number of processors used
-       for one dimension */
-    free_procs -= used_procs/nprocs[max_dir];
-    nprocs[max_dir]++;
-  }
-  /* success */
-  return 0;
-}
-
-
-
- /*@@
    @routine    PUGH_GenerateNeighbours
    @date       Mon Nov  8 08:15:08 1999
    @author     Tom Goodale
@@ -507,13 +293,18 @@ int PUGH_GenerateNeighbours(int dim,
   int *pos;
   int temp;
 
+  void *table;
+
+  /* Create P2L translation table. */
+  table = (void *)PUGHi_CreateP2LTranslation(dim, nprocs, total_procs);
+
   pos = malloc(dim*sizeof(int));
 
   if(dim == 0 || pos)
   {
     for(i = 0; i < total_procs; i++)
     {
-      PUGH_DecomposeIJK(dim,i,nprocs, pos);
+      PUGH_DecomposeIJK(dim, PUGHi_PhysicalToLogical(table, i), nprocs, pos);
 
       for(idim = 0; idim < dim ; idim++)
       {
@@ -522,13 +313,15 @@ int PUGH_GenerateNeighbours(int dim,
 
         if(pos[idim] > -1)
         {
-          neighbours[i][idim*2] = PUGH_ComposeIJK(dim, nprocs, pos);
+          neighbours[i][idim*2] = 
+           PUGHi_LogicalToPhysical(table, PUGH_ComposeIJK(dim, nprocs, pos));
         }
         else if(perme[idim])
         {
           temp = pos[idim];
           pos[idim] = nprocs[idim]-1;
-          neighbours[i][idim*2] = PUGH_ComposeIJK(dim, nprocs, pos);
+          neighbours[i][idim*2] = 
+            PUGHi_LogicalToPhysical(table, PUGH_ComposeIJK(dim, nprocs, pos));
           pos[idim] = temp;
         }
         else
@@ -543,13 +336,15 @@ int PUGH_GenerateNeighbours(int dim,
 
         if(pos[idim] < nprocs[idim])
         {
-          neighbours[i][idim*2+1] = PUGH_ComposeIJK(dim, nprocs, pos);
+          neighbours[i][idim*2+1] = 
+            PUGHi_LogicalToPhysical(table, PUGH_ComposeIJK(dim, nprocs, pos));
         }
         else if(perme[idim])
         {
           temp = pos[idim];
           pos[idim] = 0;
-          neighbours[i][idim*2+1] = PUGH_ComposeIJK(dim, nprocs, pos);
+          neighbours[i][idim*2+1] = 
+            PUGHi_LogicalToPhysical(table, PUGH_ComposeIJK(dim, nprocs, pos));
           pos[idim] = temp;
         }
         else
@@ -568,6 +363,9 @@ int PUGH_GenerateNeighbours(int dim,
     retval = 1;
   }
 
+  /* Destory P2L translation table. */
+  PUGHi_DestroyP2LTranslation(table);
+  
   free(pos);
 
 #ifdef DEBUG_PUGH
@@ -1593,79 +1391,3 @@ void PUGH_DestroyGArray(pGA **GA)
 /********************************************************************
  *********************     Local Routines   *************************
  ********************************************************************/
-
- /*@@
-   @routine    PUGH_IntegerRoot
-   @date       Tue Jan 30 17:06:21 2001
-   @author     Tom Goodale
-   @desc
-   Generate the highest integer below a given integer root of an integer.
-   @enddesc
-
-   @var     number
-   @vdesc   The number to take the root of
-   @vtype   int
-   @vio     in
-   @endvar
-   @var     invpower
-   @vdesc   The root to take
-   @vtype   int
-   @vio     in
-   @endvar
-
-   @returntype int
-   @returndesc
-   The highest integer below the desired root.
-   @endreturndesc
-@@*/
-static int PUGH_IntegerRoot(int number, int invpower)
-{
-  int i;
-  int tmp;
-  int root;
-
-  for(root = 1; root <= number; root++)
-  {
-    for(i=1, tmp=root; i < invpower; i++, tmp*=root);
-
-    if(tmp > number)
-    {
-      root--;
-      break;
-    }
-  }
-
-  return root;
-}
-
-
- /*@@
-   @routine    IntSort
-   @date       Tue Jan 30 17:08:47 2001
-   @author     Tom Goodale
-   @desc
-               Sorts two integers for the qsort routine.
-   @enddesc
-
-   @var        a
-   @vdesc      Pointer to first integer to compare
-   @vtype      const void *
-   @vio        in
-   @endvar
-   @var        b
-   @vdesc      Pointer to second integer to compare
-   @vtype      const void *
-   @vio        in
-   @endvar
-
-   @returntype int
-   @returndesc
-               -ve if b is greater than a.<BR>
-               +ve if a is greater than b.<BR>
-               0   if a is equal to b.
-   @endreturndesc
-@@*/
-static int IntSort (const void *a, const void *b)
-{
-  return (*(const int *) a - *(const int *) b);
-}