Fix for evolving arrays on multi-processors with the generic RK

method. This fix should use minimal memory, and works even when there
is no storage for the array on the local processor.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusNumerical/MoL/trunk@115 578cdeb0-5ea1-4b81-8215-5a3b8777ee0b

3 files changed, 67 insertions, 28 deletions

diff --git a/src/GenericRK.c b/src/GenericRK.c
index 60648f1..58185ca 100644
--- a/src/GenericRK.c
+++ b/src/GenericRK.c
@@ -73,7 +73,7 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
     
   cGroupDynamicData arraydata;
   CCTK_INT groupindex, ierr;
-  CCTK_INT arraytotalsize, arraydim;
+  CCTK_INT arraytotalsize, arraydim, singlearraysize;
 
   /* FIXME */
 
@@ -346,6 +346,28 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
 
   arrayscratchlocation = 0;
 
+  arraytotalsize = MoL_Max_Evolved_Array_Size;
+  if (MoL_Num_Scratch_Levels)
+  {
+    groupindex = CCTK_GroupIndex("MOL::ARRAYSCRATCHSPACE");
+    ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
+                                 &arraydata);
+    if (ierr)
+    {
+      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, 
+                 "The driver does not return group information "
+                 "for group '%s' (%d).", 
+                 CCTK_GroupName(groupindex), ierr);
+    }
+    arraytotalsize = 1;
+    for (arraydim = 0; arraydim < arraydata.dim; arraydim++)
+    {
+      arraytotalsize *= arraydata.lsh[arraydim];
+    }
+  }
+  singlearraysize = arraytotalsize / MoLNumEvolvedArrayVariables;
+  
+
   for (var = 0; var < MoLNumEvolvedArrayVariables; var++)
   {
     
@@ -387,14 +409,19 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
 
       if (scratchstep) 
       {
-        ScratchVar = &ArrayScratchSpace[scratchindex*
-                                       (MoL_Max_Evolved_Array_Size+1) + 
+/*         ScratchVar = &ArrayScratchSpace[scratchindex* */
+/*                                        (MoL_Max_Evolved_Array_Size+1) +  */
+/*                                         arrayscratchlocation]; */
+        ScratchVar = &ArrayScratchSpace[scratchindex*singlearraysize + 
                                         arrayscratchlocation];
 #ifdef MOLDEBUG
         if (CCTK_EQUALS(verbose,"extreme"))
         {
+/*           printf("Reading from scratch space, initial address %ld index %d\n",  */
+/*                  ScratchVar, scratchindex*(MoL_Max_Evolved_Array_Size+1) +  */
+/*                  arrayscratchlocation); */
           printf("Reading from scratch space, initial address %ld index %d\n", 
-                 ScratchVar, scratchindex*(MoL_Max_Evolved_Array_Size+1) + 
+                 ScratchVar, scratchindex*singlearraysize + 
                  arrayscratchlocation);
         }
 #endif
@@ -438,7 +465,8 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
                                                 EvolvedArrayVariableIndex[var]);
       ScratchVar = &ArrayScratchSpace[(MoL_Intermediate_Steps -
                                        (*MoL_Intermediate_Step)) * 
-                                     (MoL_Max_Evolved_Array_Size+1) + 
+                                     singlearraysize +
+/*                                      (MoL_Max_Evolved_Array_Size+1) +  */
                                      arrayscratchlocation];
       groupindex = CCTK_GroupIndexFromVarI(EvolvedArrayVariableIndex[var]);
       ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
@@ -460,7 +488,8 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
       printf("Writing to scratch space, initial address %ld, index %d \n", 
              ScratchVar, (MoL_Intermediate_Steps -
                           (*MoL_Intermediate_Step)) * 
-             (MoL_Max_Evolved_Array_Size+1) + 
+             singlearraysize +
+/*              (MoL_Max_Evolved_Array_Size+1) +  */
              arrayscratchlocation);
 #endif
       for (index = 0; index < arraytotalsize; index++)
diff --git a/src/ICN.c b/src/ICN.c
index 18f2d6a..7e0acde 100644
--- a/src/ICN.c
+++ b/src/ICN.c
@@ -151,6 +151,10 @@ void MoL_ICNAdd(CCTK_ARGUMENTS)
       arraytotalsize *= arraydata.lsh[arraydim];
     }
 
+/*     CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,  */
+/*                  "This proc array total size is %d.",  */
+/*                  arraytotalsize); */
+
     for (index = 0; index < arraytotalsize; index++)
     {
       UpdateVar[index] = OldVar[index] + CCTK_DELTA_TIME * RHSVar[index];
diff --git a/src/InitialCopy.c b/src/InitialCopy.c
index f326ead..2b63697 100644
--- a/src/InitialCopy.c
+++ b/src/InitialCopy.c
@@ -167,18 +167,21 @@ void MoL_InitialCopy(CCTK_ARGUMENTS)
       arraytotalsize *= arraydata.lsh[arraydim];
     }
 
-    if (PreviousVar && CurrentVar)
-    {      
-      memcpy(CurrentVar, PreviousVar, arraytotalsize * sizeof(CCTK_REAL));
-    }
-    else
-    {
-      printf("The pointers are %p (prev) and %p (curr)\n.",
-             PreviousVar, CurrentVar);
-      CCTK_VWarn(0,__LINE__,__FILE__,CCTK_THORNSTRING,"Null pointer for variable %s", 
-                 CCTK_VarName(EvolvedArrayVariableIndex[var]));
+    if (arraytotalsize)
+    {  
+      if (PreviousVar && CurrentVar)
+      {      
+        memcpy(CurrentVar, PreviousVar, arraytotalsize * sizeof(CCTK_REAL));
+      }
+      else
+      {
+        printf("The pointers are %p (prev) and %p (curr)\n.",
+               PreviousVar, CurrentVar);
+        CCTK_VWarn(0,__LINE__,__FILE__,CCTK_THORNSTRING,"Null pointer for variable %s", 
+                   CCTK_VarName(EvolvedArrayVariableIndex[var]));
+      }
     }
-
+    
   }
 
   /* FIXME */
@@ -543,21 +546,24 @@ void MoL_InitRHS(CCTK_ARGUMENTS)
       arraytotalsize *= arraydata.lsh[arraydim];
     }
 
-    if (RHSVar)
+    if (arraytotalsize)
     {
-      for (index = 0; index < arraytotalsize; index++)
+      if (RHSVar)
       {
-        RHSVar[index] = 0;
+        for (index = 0; index < arraytotalsize; index++)
+        {
+          RHSVar[index] = 0;
+        }
+      }
+      else
+      {
+        printf("The pointer is %p (rhs)\n.",
+               RHSVar);
+        CCTK_VWarn(0,__LINE__,__FILE__,CCTK_THORNSTRING,"Null pointer for variable %s", 
+                   CCTK_VarName(RHSArrayVariableIndex[var]));
       }
     }
-    else
-    {
-      printf("The pointer is %p (rhs)\n.",
-             RHSVar);
-      CCTK_VWarn(0,__LINE__,__FILE__,CCTK_THORNSTRING,"Null pointer for variable %s", 
-                 CCTK_VarName(RHSArrayVariableIndex[var]));
-    }
-
+    
   }
 
   /* FIXME */