Load balancing grid decomposition for PUGH.

Contributed by Matei Ripeneau


Here are his notes (hopefully there will soon be PUGH docs to add them to):


1. Purpose

Allow manual (user specified) grid partitioning


2. Parameters

I've added the following parameters to ./PUGH/param.ccl

KEYWORD partition "Is the partition manual/automatic"
{
 "automatic"    :: "even implicit partition"
 "manual"       :: "specified by partition_#d_XYZ .."
} "automatic"

STRING partition_1d_X "Tells how to partition on direction X"  ...
STRING partition_2d_X "Tells how to partition on direction X"  ...
STRING partition_2d_Y "Tells how to partition on direction Y"  ...
STRING partition_3d_X "Tells how to partition on direction X"  ...
STRING partition_3d_Y "Tells how to partition on direction Y"  ...
STRING partition_3d_Z "Tells how to partition on direction Z"  ...

(all these strings have default "")


How to specify a manual partition?

Simpler to explain using an example: partitioning a grid space with 30 x 30 x
30 points for a configuration with 8 processors can be specified as:

PUGH::partition = "manual"
PUGH::partition_1d_X = ""           // uses default
PUGH::partition_2d_X = "16:14"
PUGH::partition_2d_Y = "7:9:5:9"
PUGH::partition_3d_X = "16:14"
PUGH::partition_3d_Y = "12:18"
PUGH::partition_3d_Z = "17:13"

Even if PUGH::partition = "manual" an empty string like PUGH::partition_1d_X =
"" will lead to default (even) partition on that direction


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@174 b61c5cb5-eaca-4651-9a7a-d64986f99364

1 files changed, 46 insertions, 30 deletions

diff --git a/src/SetupPGV.c b/src/SetupPGV.c
index 33114fb..b0642ea 100644
--- a/src/SetupPGV.c
+++ b/src/SetupPGV.c
@@ -8,7 +8,6 @@
    @version $Header$
  @@*/
 
-
 /*#define DEBUG_PUGH*/
 
 #include <stdio.h>
@@ -1100,58 +1099,72 @@ int pugh_SetupBoundingBox(int dim,
                           int *nprocs,
                           pGExtras *this)
 {
-  int pnum,dir;
+  int pnum,dir, i;
 
-  int *step;
+  int **bounds;
   int *pos;
 
-  step = (int *)malloc(dim*sizeof(int));
+  bounds = (int **)malloc(dim*sizeof(int*));
   pos  = (int *)malloc(dim*sizeof(int));
 
-  if(step && pos)
+  if(bounds && pos)
   {
-    /* Work out the step in each direction */
-    for (dir = 0 ; dir < dim; dir++)
+    /* Work out the bounds in each direction - either from parameters 
+       file or default*/
+    pugh_GetBounds(dim, bounds, nprocs, this->nsize);
+
+    /*    for (dir = 0 ; dir < dim; dir++)
     {
       step[dir] = (this->nsize[dir]-1) / nprocs[dir];
     }
+    */
 
     for(pnum = 0; pnum < total_procs; pnum++)
     {
 
-      pugh_DecomposeIJK(dim, pnum,nprocs, pos);
+      pugh_DecomposeIJK(dim, pnum, nprocs, pos);
       
       for(dir = 0 ; dir < dim; dir++)
       {
-        if (pos[dir] == 0)
-        {
-          this->lb[pnum][dir] = 0;
-        }
-        else            
-        {
-          this->lb[pnum][dir] = pos[dir]*step[dir] +1 - nghosts[dir];
-          if(stagger == PUGH_STAGGER)
-          {
-            this->lb[pnum][dir] --;
-          }
-        }
-
-        if (pos[dir] == nprocs[dir]-1) 
-        {
-          this->ub[pnum][dir] = this->nsize[dir]-1;
-        }
-        else
-        {
-          this->ub[pnum][dir] = (pos[dir]+1)*step[dir] + this->nghostzones[dir];
-        }
+	if (pos[dir] == 0)
+	{
+	  this->lb[pnum][dir] = 0;
+	}
+	else		
+	{
+	  this->lb[pnum][dir] = bounds[dir][pos[dir]] +1 - nghosts[dir];
+	  if(stagger == PUGH_STAGGER)
+	  {
+	    this->lb[pnum][dir] --;
+	  }
+	}
+
+	if (pos[dir] == nprocs[dir]-1) 
+	{
+	  this->ub[pnum][dir] = this->nsize[dir]-1;
+	}
+	else
+	{
+	  this->ub[pnum][dir] = bounds[dir][pos[dir]+1] + this->nghostzones[dir];
+	}
       }
     }
 
   }
 
-  free(step);
+  free(bounds);
   free(pos);
 
+#ifdef DEBUG_PUGH
+  for(i=0; i<total_procs; i++) {
+    printf(" setup_Bounding_Box (%d):", i);
+    for (dir = 0 ; dir < dim; dir++)
+      printf(" (%d,%d)",
+             this->lb[i][dir], this->ub[i][dir]);               
+    printf(" \n");
+  }
+#endif
+
   return 0;
 }
 
@@ -1568,3 +1581,6 @@ void pugh_GAComm(pGA *GA, int docomm)
     
 }
 
+
+
+