staggering with pugh

git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGH/PUGH/trunk@143 b61c5cb5-eaca-4651-9a7a-d64986f99364

3 files changed, 97 insertions, 18 deletions

diff --git a/src/GHExtension.c b/src/GHExtension.c
index 300bd7d..83f9fb8 100644
--- a/src/GHExtension.c
+++ b/src/GHExtension.c
@@ -27,13 +27,13 @@ int pugh_SetupGroup(pGH *newGH,
 		    int vtype, 
 		    int dim, 
 		    int n_variables,
+		    int staggertype,
 		    int n_timelevels);
 
 pGH *SetupPGH(void *callerid, 
 	      int dim, 
 	      int *nsize, 
-	      int *ghostsize, 
-	      int stagger);
+	      int *ghostsize);
 
 void *pugh_SetupGH(tFleshConfig *config, 
 		   int convergence_level, 
@@ -43,7 +43,7 @@ void *pugh_SetupGH(tFleshConfig *config,
   DECLARE_CCTK_PARAMETERS
 
   cGroup *pgroup;
-  pGH *newGH;
+  pGH    *newGH;
   int group;
   int dim;
   int *nsize;
@@ -133,7 +133,7 @@ void *pugh_SetupGH(tFleshConfig *config,
     }
   }
     
-  newGH = SetupPGH(GH,dim,nsize, ghostsize, PUGH_NO_STAGGER);
+  newGH = SetupPGH(GH, dim, nsize, ghostsize);
 
   if(!newGH)
   {
@@ -150,14 +150,13 @@ void *pugh_SetupGH(tFleshConfig *config,
 #endif 
 
     pgroup=CCTK_GroupData(group);
-
     pugh_SetupGroup(newGH, 
 		    pgroup->grouptype, 
 		    pgroup->variabletype, 
 		    pgroup->dim, 
 		    pgroup->numvariables, 
+		    pgroup->staggertype,
 		    pgroup->numtimelevels);
-    
   }
 
   free(nsize);
diff --git a/src/SetupGroup.c b/src/SetupGroup.c
index 9cf6679..e6ad426 100644
--- a/src/SetupGroup.c
+++ b/src/SetupGroup.c
@@ -20,11 +20,13 @@ pGF *SetupPGF(pGH *GH,
 	      int dim, 
 	      int varsize, 
 	      int vtype, 
-	      int stagger);
+	      int staggercode);
 
 int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timelevels);
-int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables, int n_timelevels);
-int pugh_SetupGFGroup(pGH *newGH, int vtype, int dim, int n_variables, int n_timelevels);
+int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables, 
+			 int staggergroup, int n_timelevels);
+int pugh_SetupGFGroup(pGH *newGH, int vtype, int dim, int n_variables, 
+		      int staggercode, int n_timelevels);
 
  /*@@
    @routine    pugh_SetupGroup
@@ -40,7 +42,8 @@ int pugh_SetupGFGroup(pGH *newGH, int vtype, int dim, int n_variables, int n_tim
    @endhistory 
 
 @@*/
-int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables, int n_timelevels)
+int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables, 
+		    int staggercode, int n_timelevels)
 {
   int returncode;
   switch(gtype)
@@ -54,12 +57,14 @@ int pugh_SetupGroup(pGH *newGH, int gtype, int vtype, int dim, int n_variables,
 							  vtype,
 							  dim,
 							  n_variables,
+							  staggercode,
 							  n_timelevels); 
                         break;
     case GROUP_GF     : returncode = pugh_SetupGFGroup(newGH, 
 							   vtype,
 							   dim,
 							   n_variables,
+						           staggercode,
 							   n_timelevels); 
                         break;
     default : CCTK_WARN(0, "Unknown group type in pugh_SetupGroup");
@@ -177,22 +182,21 @@ int pugh_SetupScalarGroup(pGH *newGH, int vtype, int n_variables, int n_timeleve
       }
       else
       {
-	fprintf(stderr, "Memory error in SetupScalarGroup at line %d\n", 
-		__LINE__);
+	CCTK_WARN(1," Memory A error in SetupScalarGroup: ");
 	break;
       }
     }
   }
   else
   {
-	fprintf(stderr, "Memory error in SetupScalarGroup at line %d\n", 
-		__LINE__);
+    CCTK_WARN(1," Memory A error in SetupScalarGroup: ");
   }
 
   return 0;
 }
 
-int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables, int n_timelevels)
+int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables, 
+			 int staggercode, int n_timelevels)
 {
   int variable;
   char *msg;
@@ -260,7 +264,8 @@ int pugh_SetupArrayGroup(pGH *newGH, int vtype, int dim, int n_variables, int n_
 int pugh_SetupGFGroup(pGH *newGH, 
 		      int vtype, 
 		      int dim, 
-		      int n_variables, 
+		      int n_variables,
+		      int staggercode,
 		      int n_timelevels)
 {
 
@@ -274,6 +279,7 @@ int pugh_SetupGFGroup(pGH *newGH,
 
   int level;
 
+
   var_size = CCTK_VarTypeSize(vtype);
 
   temp = (void ***)realloc(newGH->variables, 
@@ -298,7 +304,7 @@ int pugh_SetupGFGroup(pGH *newGH,
 		     dim, 
 		     var_size, 
 		     vtype,
-		     PUGH_VERTEXCTR);
+		     staggercode);
 	}
 	newGH->nvariables++;
       }
diff --git a/src/SetupPGH.c b/src/SetupPGH.c
index 870efdf..a760b54 100644
--- a/src/SetupPGH.c
+++ b/src/SetupPGH.c
@@ -58,6 +58,7 @@ pGH *SetupPGH(void *callerid, int dim, int *nsize, int *nghostzones,int staggert
   void pGH_SetupBasics(pGH *, int, int *, int *, int);
   void pGH_SetupBounds(pGH *GH);
   void pGH_SetupOwnership(pGH *,int dim);
+  void pGH_SetupStaggering(pGH *GH, int dim);
 
   /* Allocate for myself */
   GH = (pGH*) malloc(sizeof(pGH));
@@ -68,6 +69,7 @@ pGH *SetupPGH(void *callerid, int dim, int *nsize, int *nghostzones,int staggert
   pGH_SetupBasics(GH,dim,nsize,nghostzones,staggertype);
   pGH_SetupBounds(GH);
   pGH_SetupOwnership(GH,dim);
+  pGH_SetupStaggering(GH,dim);
 
   return GH;
 
@@ -612,6 +614,78 @@ void pGH_SetupOwnership(pGH *GH,int dim) {
   free(iend);
 }
 
+void pGH_SetupStaggering(pGH *GH, int dim) {
+  int s,d,k,i,dir;
+  
+  int upperbnd[dim];
+  
+  for (d=0;d<dim;d++) 
+    upperbnd[d] = (GH->ub[GH->myproc][d] == GH->nsize[d]-1) ? 1 : 0;
+
+  /* copy stuff from PUGH_M */
+  for(s=1;s<PUGH_NSTAGGER;s++)
+    for(d=0;d<dim;d++)
+      for(k=0;k<2;k++) {
+	GH->ownership[s][k][d]    = GH->ownership[0][k][d];
+	for (dir=0;dir<2*dim;dir++) {
+	  GH->overlap[s][k][dir][d] = GH->overlap[0][k][dir][d];
+	  GH->ghosts [s][k][dir][d] = GH->ghosts [0][k][dir][d]; 
+	}
+      }
+  
+  /* correct ownership for staggered grids for the C,P cases */
+  for(s=1;s<PUGH_NSTAGGER;s++)
+  {
+    for (d=0;d<dim;d++)
+    {	
+      /* if we are at a upper physical bnd, decrease by one */
+      if(upperbnd[d]) 
+      {
+	GH->ownership[s][1][d] --;
+	printf(" upperbnd[%d] = %d\n",d,upperbnd[d]);
+	printf(" SetupPGH: OS[%d][1][%d] %d\n",s,d, GH->ownership[s][1][d]);
+      }
+    }
+  }
+
+  /* correct overlaps for staggered grids */
+  for(s=1; s<PUGH_NSTAGGER; s++) 
+  {
+    for(dir=0;dir<2*dim;dir++)
+    {
+      for (d=0;d<dim;d++)
+      {
+	if((upperbnd[d])&&((dir+1)%2==0)&&(dir==2*d+1))
+        {
+	  GH->overlap[s][0][dir][d]--;
+	  GH->overlap[s][1][dir][d]--;
+	  printf(" upperbnd[%d] = %d\n",d,upperbnd[d]);
+	  printf(" OL[%d][0/1][%d][%d]: %d %d -> %d %d \n",
+		  s,dir,d,
+		 GH->overlap[0][0][dir][d],GH->overlap[0][1][dir][d],
+		 GH->overlap[s][0][dir][d],GH->overlap[s][1][dir][d]);
+	}
+      }
+    }
+  }
+       
+  /* correct ghosts for staggered grids */
+  for(s=1; s<PUGH_NSTAGGER; s++)
+  { 
+    for(dir=0;dir<2*dim;dir++)
+    {
+      for (d=0;d<dim;d++)
+      {
+	if(upperbnd[d]&&((dir+1)%2==0)&&(dir==2*d+1))
+	{
+	  GH->ghosts[s][0][dir][d]--;
+	  GH->ghosts[s][1][dir][d]--;
+	}
+      }
+    }
+  }
+
+}
 
  /*@@
    @routine    DestroyPGH
@@ -624,7 +698,7 @@ void pGH_SetupOwnership(pGH *GH,int dim) {
 void DestroyPGH(pGH **GHin) 
 {
   /* First remove me from the list. */
-  pGH *GH;
+  pGH    *GH;
   cGroup *pgroup;
   int i;
   int didit;