Now creates all variable types except arrays.

Tom


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@6 b61c5cb5-eaca-4651-9a7a-d64986f99364

8 files changed, 532 insertions, 14 deletions

diff --git a/param.ccl b/param.ccl
index 1575be5..07e9a60 100644
--- a/param.ccl
+++ b/param.ccl
@@ -49,3 +49,29 @@ INTEGER proc_top_nz "No of Procs in Z direction"
 {
   0:*  :: "See proc_topology"
 } 0
+
+# padding stuff
+
+LOGICAL padding_active "Padd 3D arrays so they line up on cache lines?"
+{
+} yes
+
+
+INTEGER padding_cacheline_bits "Number of bits which have to be unique to padd properly for cache lines"
+{
+  1:*  :: "Any positive number"
+} 12
+
+INTEGER padding_size "How many Doubles to pad by; we have to noodle around with the starting address, so if there isn't enough space we may not be able to pad"
+{
+  0:*  ::  "Any positive number"
+} 4112
+
+INTEGER padding_address_spacing "Number of Doubles to space starting addresses in lowest padding_cacheline_bits bits"
+{
+  0:*  ::  "Any positive number"
+} 24
+
+LOGICAL zero_memory "Zero memory for GF's at allocation time ?"
+{
+} yes
diff --git a/src/GHExtension.c b/src/GHExtension.c
index 5005514..ce3c953 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -9,6 +9,8 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+
+#include "cctk.h"
 #include "flesh.h"
 #include "Groups.h"
 
@@ -27,7 +29,7 @@ void *pugh_SetupGH(tFleshConfig *config,
   DECLARE_PARAMETERS
   pGH *newGH;
   int group;
-  int gtype, vtype, n_variables;
+  int gtype, vtype, dim, n_variables;
 
   newGH = SetupPGH(global_nx, global_ny, global_nz, 
 		   ghost_size, PUGH_NO_STAGGER);
@@ -40,9 +42,9 @@ void *pugh_SetupGH(tFleshConfig *config,
 
   for(group = 0; group < CCTK_GetNumGroups(); group++)
   {
-    CCTK_GetGroupData(group, &gtype, &vtype, &n_variables);
+    CCTK_GetGroupData(group, &gtype, &vtype, &dim, &n_variables);
     
-    pugh_SetupGroup(newGH, gtype, vtype, n_variables);
+    pugh_SetupGroup(newGH, gtype, vtype, dim, n_variables);
     
   }
 
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index e1678d3..4290f5b 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -7,6 +7,7 @@
    @enddesc 
  @@*/
 
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "cctk.h"
@@ -16,16 +17,16 @@
 
 #include "pugh.h"
 
-/*#include "pGH.h"*/
-
 static char *rcisd = "$Header$";
 
+pGF *SetupPGF(pGH *GH, const char *name, int dim);
+
  /*@@
    @routine    pugh_SetupGroup
    @date       Mon Feb  8 19:37:55 1999
    @author     Tom Goodale
    @desc 
-   Sets up a griup on a pGH
+   Sets up a group on a pGH
    @enddesc 
    @calls     
    @calledby   
@@ -34,7 +35,7 @@ static char *rcisd = "$Header$";
    @endhistory 
 
 @@*/
-int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int n_variables)
+int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables)
 {
   int returncode;
   switch(gtype)
@@ -44,11 +45,13 @@ int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int n_variables)
 							   n_variables); 
                         break;
     case GROUP_ARRAY  : returncode = pugh_SetupScalarGroup(newGH, 
-							   vtype, 
+							   vtype,
+							   dim,
 							   n_variables); 
                         break;
     case GROUP_GF     : returncode = pugh_SetupScalarGroup(newGH, 
-							   vtype, 
+							   vtype,
+							   dim,
 							   n_variables); 
                         break;
     default : fprintf(stderr, "Unknown group type in pugh_SetupGroup\n");
@@ -110,12 +113,44 @@ int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables)
   return 0;
 }
 
-int pugh_SetupArrayGroup(pGH *newGH, int vtype, int n_variables)
+int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables)
 {
   return 0;
 }
 
-int pugh_SetupGFGroup(pGH *newGH, int vtype, int n_variables)
+int pugh_SetupGFGroup(pGH *newGH, int vtype, int dim, int n_variables)
 {
+
+  int variable;
+
+  int var_size;
+
+  void **temp;
+
+  int returncode;
+
+  switch(vtype)
+  {
+    case VARIABLE_CHAR    : var_size = sizeof(char); break;
+    case VARIABLE_INTEGER : var_size = sizeof(int) ; break;
+    case VARIABLE_REAL    : var_size = sizeof(Double); break;
+    case VARIABLE_COMPLEX : var_size = sizeof(Complex); break;
+    default : fprintf(stderr, 
+		      "Unknown variable type in pugh_SetupScalarGroup\n");
+               var_size = 1;
+  }
+
+  temp = (void **)realloc(newGH->variables, (newGH->nvariables+n_variables)*sizeof(void *));
+
+  if(temp)
+  {
+    newGH->variables = temp;
+
+    for(variable = 0; variable < n_variables; variable++)
+    {
+      newGH->variables[newGH->nvariables] = SetupPGF(newGH, CCTK_GetVarName(n_variables), dim);
+    }
+  }
+
   return 0;
 }
diff --git a/src/SetupPGF.c b/src/SetupPGF.c
new file mode 100644
index 0000000..8b92fb8
--- /dev/null
+++ b/src/SetupPGF.c
@@ -0,0 +1,443 @@
+ /*@@
+   @file      SetupPGF.c
+   @date      Fri Feb 21 11:04:20 1997
+   @author    
+   @desc 
+   Sets up (eg allocates) the pugh Grid Function. This is
+   crucial to understaiding the grid function struct.
+   @enddesc 
+   @version $Id$
+ @@*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "cctk.h"
+#include "declare_parameters.h"
+
+#include "pugh.h"
+
+static char *rcsid = "$Id$";
+
+void SetGFComm(pGH *GH, pGF *res, int docomm);
+
+ /*@@
+   @routine    SetupPGF
+   @date       Fri Feb 21 11:05:04 1997
+   @author     
+   @desc 
+   Allocates the Grid Func structure (I wanted to say object ...)
+   There are several steps to this
+   <ul>
+     <li> Set up the name
+     <li> Set up the docomm flag and IO flags
+     <li> Allocate output file pointer arrays
+     <li> Allocate the data buffer
+     <li> Allocate the send buffer
+     <li> Add myself to the Grid Hierarchy
+   </ul>
+   But this is pretty straightforward code nonetheless.
+   <p>
+   Note that the input variable "storage" determines whether to
+   allocate the 3-D data or not. You can toggle this later
+   with @seeroutine DisableGFDataStorage and @seeroutine
+   EnableGFDataStorage
+   @enddesc 
+   @calls DisableGFDataStorage, EnableGFDataStorage
+@@*/
+
+
+pGF *SetupPGF(pGH *GH, const char *name, int dim) {
+  pGF *res;			/* The result */
+  pGF **rlist;			/* A temporary to add to the GH */
+  int i, dir, dirp1, dirp2, sz;	/* Counter thingies */
+
+  /* Fudge for CCTK. */
+  int stagger = PUGH_VERTEXCTR;
+  int storage = PUGH_NOSTORAGE;
+  int docomm = PUGH_NOCOMM;
+
+  /* Space for the struct */
+  res = (pGF *)malloc(sizeof(pGF));
+
+  /* Set my parent GH */
+  res->parentGH = GH;
+
+  /* Set up the name */
+  res->name = (char *)malloc((1+strlen(name))*sizeof(char));
+  strcpy(res->name,name);
+
+  res->stagger = stagger;
+  if( (GH->stagger == PUGH_NO_STAGGER) &&
+      (res->stagger != PUGH_VERTEXCTR) )
+  {  printf ("FATAL ERROR! Cannot have staggered grids inside a GH with \n");
+     printf ("             designation PUGH_NO_STAGGER. \n");
+     STOP;
+  }
+
+#ifdef 0
+  /* Setup IO stuff to default no IO with space for xgraph files.*/
+  res->do_3dio = 0;
+  res->do_2dio = 0;
+  res->do_1dio = 0;
+  res->do_0dio = 0;
+  res->do_conv = 0;
+  
+  res->convfile = NULL;
+  
+  res->xgfile = (FILE **)malloc(5*sizeof(FILE *));
+  for (i=0;i<5;i++)
+    res->xgfile[i] = NULL;
+
+  res->zerodfile = (FILE **)malloc(4*sizeof(FILE *));
+  for (i=0;i<4;i++)
+    res->zerodfile[i] = NULL;
+
+  res->IEEEfile = NULL;
+
+  for (i=0;i<3;i++) 
+    res->IEEEfile_2d[i] = NULL;
+  
+  res->show_maxval = 0;
+
+  res->lastio_it[0] = -1; res->lastio_it[1] = -1; 
+  res->lastio_it[2] = -1; res->lastio_it[3] = -1;
+  
+   /* Finally add this to the list of grid funcs in the gh */
+  GH->ngridFuncs ++;
+  rlist = (pGF **)malloc(GH->ngridFuncs * sizeof(pGF *));
+  if (GH->ngridFuncs > 1) {
+    for (i=0;i<GH->ngridFuncs-1;i++)
+      rlist[i] = GH->gridFuncs[i];
+    free(GH->gridFuncs);
+  }
+  res->gfno = GH->ngridFuncs - 1;
+  rlist[GH->ngridFuncs-1] = res;
+  GH->gridFuncs = rlist;
+
+#endif
+
+ SetGFComm(GH, res, docomm);
+
+  /* Set up the comm layer */
+  res->padddata = NULL;
+  res->data = NULL;
+  res->send_buffer = NULL;
+  res->recv_buffer = NULL;
+
+  if (storage == PUGH_NOSTORAGE) {
+    res->storage = PUGH_STORAGE; /* Fake it out... */
+    DisableGFDataStorage(GH, res);
+  } else if (storage == PUGH_STORAGE) {
+    res->storage = PUGH_NOSTORAGE; /* Fake it out... */
+    EnableGFDataStorage(GH, res);
+  } else {
+    printf ("WARNING: Storage keyword set incorrectly for %s [%d]\n",res->name,storage);
+    printf ("         Perhaps misspelled in GridFuncList? Defaulting to storage active.\n");
+    EnableGFDataStorage(GH, res);
+  }
+
+#ifdef MPI
+  /* Null my send and recieve requests */
+  for (i=0;i<6;i++) {
+    res->sreq[i] = MPI_REQUEST_NULL;
+    res->rreq[i] = MPI_REQUEST_NULL;
+  }
+#endif
+
+  /* And return myself... */
+  return res;
+}
+
+#ifdef 0
+ /*@@
+   @routine    DestroyPGF
+   @date       Thu Aug 21 11:44:22 1997
+   @author     Paul Walker
+   @desc 
+   Destroys a GF object.
+   @enddesc 
+@@*/
+
+void DestroyPGF(pGH *GH, pGF **GFin) {
+  pGF *GF;
+  GF = *GFin;
+
+  if (GF->storage)
+    DisableGFDataStorage(GH, GF);
+  
+  free(GF->name);
+  free(GF->padddata);
+
+  free(GF->xgfile);
+  free(GF->zerodfile);
+
+  GF->data = NULL;
+
+  free(GF);
+  *GFin = NULL;
+
+}
+
+#endif
+
+ /*@@
+   @routine    EnableGFDataStorage
+   @date       Thu Apr  3 12:44:38 1997
+   @author     Paul Walker
+   @desc 
+   This routine toggles the data storage to the "on"
+   position. That means that data points to a 3D
+   array, rather than a single Double, and the
+   storage flag is set to one. This is used quite
+   a lot by thorns which need to toggle memory. for
+   instance, see the trK Driver in the util thron.
+   @enddesc 
+@@*/
+
+void EnableGFDataStorage(pGH *GH, pGF *GF) {
+  int i, dir, dirp1, dirp2, sz;
+  int special_pad, cache_size, start;
+  DECLARE_PARAMETERS;
+
+  if (zero_memory) {
+    printf ("Zeroing memory for allocated GFs at alloc time\n");
+  }
+  
+  if (padding_active) {
+    printf ("PUGH Memory padding active. Stats:\n");
+    printf ("  cacheline_bits   : %d\n",
+	    padding_cacheline_bits);
+    printf ("  size             : %d\n",
+	    padding_size);
+    printf ("  spacing          : %d\n",
+	    padding_address_spacing);
+  }
+
+  if (GF->storage) {
+    printf ("WARNING: Tried to enable %s when already enabled\n",
+	    GF->name);
+    return;
+  }
+
+  /* Set up the send buffers. Note we only do this if
+     we are not using the derived types comm style.
+     */
+  assert(!GF->send_buffer);
+  GF->send_buffer = (Double **)malloc(6*sizeof(Double *));
+
+  assert(!GF->recv_buffer);
+  GF->recv_buffer = (Double **)malloc(6*sizeof(Double *));
+  
+  for (i=0;i<6;i++) {
+    dir = i/2;
+    dirp1 = (i/2+1)%3;
+    dirp2 = (i/2+2)%3;
+    
+    if (dir == 0) {
+      sz = GH->stencil_width * GH->lny * GH->lnz;
+    } else if (dir == 1) {
+      sz = GH->stencil_width * GH->lnx * GH->lnz;
+    } else {
+      sz = GH->stencil_width * GH->lnx * GH->lny;
+    }
+    
+    if (GH->neighbors[GH->myproc][i] >= 0) {
+      GF->buffer_sz[i] = sz;
+      GF->send_buffer[i] = (Double *)malloc(sz*sizeof(Double));
+      GF->recv_buffer[i] = (Double *)malloc(sz*sizeof(Double));
+      assert(GF->recv_buffer[i]);
+      assert(GF->send_buffer[i]);
+    } else {
+      GF->buffer_sz[i] = 0;
+      GF->send_buffer[i] = NULL;
+      GF->recv_buffer[i] = NULL;
+    }
+  }
+
+  /* Set up the storage */
+  if (GF->padddata) free (GF->padddata);
+  if (!padding_active) {
+    GF->padddata = (Double *)malloc((GH->npoints)*sizeof(Double));
+    GF->data     = GF->padddata;	/* No padding case */
+    if (zero_memory) {
+      for (i=0;i<GH->npoints;i++)
+        GF->padddata[i] = 0.0;
+    }
+  } else {
+    GF->padddata  = (Double *)malloc((GH->npoints + padding_size)
+				    *sizeof(Double));
+   
+    /* Align the actual starting address on a cache line.
+       More specifically align on a unique cache line for
+       each field. 
+       */
+    cache_size = 2 << (padding_cacheline_bits - 1);
+    start = ((long) (GF->padddata) / sizeof(Double))%cache_size;
+
+    special_pad = ((GF->gfno) * padding_address_spacing + cache_size - start)%cache_size;
+    GF->data = (GF->padddata) + special_pad;
+
+    if (special_pad > padding_size){
+      printf("FATAL ERROR: padding size not large enough for cache type specified %d %d %d \n",special_pad,padding_size,cache_size);
+      STOP;
+    }
+    if (zero_memory) {
+      for (i=0;i<GH->npoints+padding_size;i++)
+        GF->padddata[i] = 0.0;
+    }
+  }
+
+  if (!GF->padddata) {
+    printf ("FATAL ERROR: Cannot allocate data for %s [%d]\n",
+	    GF->name, GF->gfno);
+    STOP;
+  }
+  GF->storage = 1;
+
+
+}
+
+ /*@@
+   @routine    DisableGFDataStorage
+   @date       Thu Apr  3 12:45:34 1997
+   @author     Paul Walker
+   @desc 
+   This routine disables the grid function storage.
+   That is, it un-allocates the 3D array and in its
+   place allocates a single Double with the value
+   0.0. This allows us to still ahve something to
+   pass around (an array of size (1,1,1) in fortran
+   speak) but also to not need all our 3D arrays 
+   "on" all the time.
+   @enddesc 
+@@*/
+
+
+void DisableGFDataStorage(pGH *GH, pGF *GF) {
+  int i;
+
+  if (!GF->storage) {
+    printf ("Warning: Tried to disable %s when already disabled\n",
+	    GF->name);
+    return;
+  }
+
+  if (GF->padddata) {
+    free(GF->padddata);
+    GF->padddata = NULL;
+    GF->data = NULL;
+  }
+
+  if (GF->send_buffer) {
+    for (i=0;i<6;i++)
+      if (GF->send_buffer[i]) {
+#ifdef MPI
+	if(GF->sreq[i] != MPI_REQUEST_NULL){
+	  CACTUS_MPI_ERROR(MPI_Request_free(&(GF->sreq[i])));
+	}
+#endif
+    	free(GF->send_buffer[i]);
+	GF->send_buffer[i] = NULL;
+      }
+    free(GF->send_buffer);
+    GF->send_buffer = NULL;
+  }
+
+
+  if (GF->recv_buffer) {
+    for (i=0;i<6;i++)
+      if (GF->recv_buffer[i]) {
+	free(GF->recv_buffer[i]);
+	GF->recv_buffer[i] = NULL;
+      }
+    free(GF->recv_buffer);
+    GF->recv_buffer = NULL;
+  }
+
+  GF->padddata = (Double *)malloc(sizeof(Double));
+  GF->data = GF->padddata;
+  GF->data[0] = 0.0;		/* Very important! */
+  GF->storage = 0;
+}
+
+
+ /*@@
+   @routine    SetGFComm
+   @date       Thu Apr  3 12:46:23 1997
+   @author     Paul Walker
+   @desc 
+   This sets the docomm[3] array of the GF based
+   on the setting of the comm flag. Note the
+   comm flag constants are defined in
+   @seeheader pughDriver.h. As well as SetupPGF
+   this is called by thorns (eg, elliptic) which
+   need to toggle this state.
+   @enddesc 
+   @calledby   SetupPGF
+@@*/
+
+
+void SetGFComm(pGH *GH, pGF *res, int docomm) {
+  /* Copy docomm */
+  int i;
+  res->commflag = docomm;
+
+  for (i=0;i<6;i++)
+    res->docomm[i] = 0;
+
+  if (docomm == PUGH_NOCOMM) {
+    for (i=0;i<6;i++)
+      res->docomm[i] = 0;
+  } else if (docomm == PUGH_ALLCOMM) {
+    for (i=0;i<6;i++)
+      res->docomm[i] = 1;
+  } else if (docomm == PUGH_XCOMM) {
+    res->docomm[0] = 1;
+    res->docomm[1] = 1;
+  } else if (docomm == PUGH_YCOMM) {
+    res->docomm[2] = 1;
+    res->docomm[3] = 1;
+  } else if (docomm == PUGH_ZCOMM) {
+    res->docomm[4] = 1;
+    res->docomm[5] = 1;
+  } else if (docomm == PUGH_PLUSFACESCOMM) {
+    res->docomm[1] = 1;
+    res->docomm[3] = 1;
+    res->docomm[5] = 1;
+  } else if (docomm == PUGH_MINUSFACESCOMM) {
+    res->docomm[0] = 1;
+    res->docomm[2] = 1;
+    res->docomm[4] = 1;
+  } else {
+    printf("WARNING: Unsupported comm. in grid function %s .\n",res->name);
+    printf("         Perhaps misspelled in GridFuncList? Defaulting to allcomm.\n");
+    for (i=0;i<6;i++)
+      res->docomm[i] = 1;
+  }
+
+  /* Handle nx = 1 type cases. This is only important for one
+     processor MPI periodic boundaries.
+   */
+  /* This is a hack to get a GH. Unfortunately I decided not
+     to pass a GH to this routine, and I should go back and
+     change it, but that is a lot of changes so for now I'll
+     do the following.
+    */
+  if (GH->nx == 1) {
+    res->docomm[0] = 0;
+    res->docomm[1] = 0;
+  }
+
+  if (GH->ny == 1) {
+    res->docomm[2] = 0;
+    res->docomm[3] = 0;
+  }
+
+  if (GH->nz == 1) {
+    res->docomm[4] = 0;
+    res->docomm[5] = 0;
+  }
+}
+
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index d137131..df6f869 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -12,8 +12,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "pugh.h"
 #include "cctk.h"
+#include "pugh.h"
 #include "declare_parameters.h"
 
 /* Old Id: SetupPGH.c,v 1.29 1999/01/12 08:23:35 bruegman Exp */
diff --git a/src/include/pGF.h b/src/include/pGF.h
index d8dd2fa..7d3698c 100644
--- a/src/include/pGF.h
+++ b/src/include/pGF.h
@@ -55,7 +55,7 @@ typedef struct PGF {
 				   need the full name for the lookahead
 				   structure reference thingy.
 				 */
-
+#ifdef 0
   /* IO Related things */
   FILE     *convfile;		/* File for convergence information */
   FILE    **xgfile;		/* x,y,z and dline file pointers */
@@ -83,6 +83,9 @@ typedef struct PGF {
 
   Double    maxval, minval;	/* To store in case we toggle storage */
   Double    norm1, norm2;
+
+#endif
+
 #ifdef MPI
   MPI_Request sreq[6], rreq[6];	/* Comm requests and statuses. */
   MPI_Status ms;
diff --git a/src/include/pugh.h b/src/include/pugh.h
index 673eea0..9675da9 100644
--- a/src/include/pugh.h
+++ b/src/include/pugh.h
@@ -11,17 +11,24 @@
   @version $Id$
 @@*/
 
+/*
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <math.h>
 #include <malloc.h>
 #include <assert.h>
+*/
 
 /* 3 ways up shutting down: */
+/*
 #define STOP  cshutdown(0);
 #define ABORT cabort(0);
 #define CORE  ccore();
+*/
+#define STOP  exit(0);
+#define ABORT exit(1);
+#define CORE  exit(2);
 
 /* Two ways to shut down with a non-zero exit code. */
 #define ERROR_STOP(xerrcode)  cshutdown(xerrcode);
@@ -60,6 +67,7 @@ void cactus_free(void *);
 #include "TCP++/CommandProcessor.h"
 #endif
 
+#ifdef 0
 /* Handle precision */
 #ifdef SINGLE_PRECISION
 #define PUGH_MPI_TYPE MPI_FLOAT
@@ -68,6 +76,7 @@ typedef float Double;
 #define PUGH_MPI_TYPE MPI_DOUBLE
 typedef double Double;
 #endif
+#endif
 
 #include "pugh_constants.h"
 
diff --git a/src/make.code.defn b/src/make.code.defn
index 4628886..8f5f713 100644
--- a/src/make.code.defn
+++ b/src/make.code.defn
@@ -1,7 +1,7 @@
 # Main make.code.defn file for thorn pugh
 # : /usr/users/cactus/CCTK/lib/make/new_thorn.pl,v 1.1 1999/02/03 17:00:50 goodale Exp n
 # Source files in this directory
-SRCS = Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c
+SRCS = Startup.c GHExtension.c Comm.c SetupPGH.c SetupGroup.c SetupPGF.c
 
 # Subdirectories containing source files
 SUBDIRS =