Modification for evolving array variables; change to local

arrays. This gets around the problems with the driver allocating the
scratch space differently on multiprocessors from the individual
arrays.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusNumerical/MoL/trunk@116 578cdeb0-5ea1-4b81-8215-5a3b8777ee0b

6 files changed, 95 insertions, 105 deletions

diff --git a/src/ExternalVariables.h b/src/ExternalVariables.h
index 0c6457b..54a5129 100644
--- a/src/ExternalVariables.h
+++ b/src/ExternalVariables.h
@@ -64,3 +64,7 @@ extern CCTK_INT MoLNumSandRComplexArrayVariables;
 
 extern CCTK_INT ScheduleStatus;
 
+
+extern CCTK_REAL *ArrayScratchSpace;
+extern CCTK_INT *ArrayScratchSizes;
+extern CCTK_INT CurrentArrayScratchSize;
diff --git a/src/GenericRK.c b/src/GenericRK.c
index 58185ca..f9622c7 100644
--- a/src/GenericRK.c
+++ b/src/GenericRK.c
@@ -15,6 +15,8 @@
 #include "cctk_Arguments.h"
 #include "cctk_Parameters.h"
 
+#include <stdio.h>
+
 #include "ExternalVariables.h"
 
 static const char *rcsid = "$Header$";
@@ -73,7 +75,7 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
     
   cGroupDynamicData arraydata;
   CCTK_INT groupindex, ierr;
-  CCTK_INT arraytotalsize, arraydim, singlearraysize;
+  CCTK_INT arraytotalsize, arraydim;
 
   /* FIXME */
 
@@ -344,29 +346,14 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
 
   /* Real arrays */
 
+/* #define MOLDEBUGARRAYS 1 */
+
   arrayscratchlocation = 0;
 
-  arraytotalsize = MoL_Max_Evolved_Array_Size;
-  if (MoL_Num_Scratch_Levels)
-  {
-    groupindex = CCTK_GroupIndex("MOL::ARRAYSCRATCHSPACE");
-    ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
-                                 &arraydata);
-    if (ierr)
-    {
-      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, 
-                 "The driver does not return group information "
-                 "for group '%s' (%d).", 
-                 CCTK_GroupName(groupindex), ierr);
-    }
-    arraytotalsize = 1;
-    for (arraydim = 0; arraydim < arraydata.dim; arraydim++)
-    {
-      arraytotalsize *= arraydata.lsh[arraydim];
-    }
-  }
-  singlearraysize = arraytotalsize / MoLNumEvolvedArrayVariables;
-  
+#ifdef MOLDEBUGARRAYS
+  CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING, 
+             "Array sizes are %d %d %d\n", MoL_Max_Evolved_Array_Size, arraytotalsize, singlearraysize);
+#endif  
 
   for (var = 0; var < MoLNumEvolvedArrayVariables; var++)
   {
@@ -375,22 +362,8 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
                                               EvolvedArrayVariableIndex[var]);
     RHSVar = (CCTK_REAL const *)CCTK_VarDataPtrI(cctkGH, 0, 
                                                  RHSArrayVariableIndex[var]);
-    
-    groupindex = CCTK_GroupIndexFromVarI(EvolvedArrayVariableIndex[var]);
-    ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
-                                 &arraydata);
-    if (ierr)
-    {
-      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, 
-                 "The driver does not return group information "
-                 "for group '%s'.", 
-                 CCTK_GroupName(groupindex));
-    }
-    arraytotalsize = 1;
-    for (arraydim = 0; arraydim < arraydata.dim; arraydim++)
-    {
-      arraytotalsize *= arraydata.lsh[arraydim];
-    }
+
+    arraytotalsize = ArrayScratchSizes[var];
 
     for (index = 0; index < arraytotalsize; index++)
     {
@@ -409,22 +382,8 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
 
       if (scratchstep) 
       {
-/*         ScratchVar = &ArrayScratchSpace[scratchindex* */
-/*                                        (MoL_Max_Evolved_Array_Size+1) +  */
-/*                                         arrayscratchlocation]; */
-        ScratchVar = &ArrayScratchSpace[scratchindex*singlearraysize + 
+        ScratchVar = &ArrayScratchSpace[scratchindex*CurrentArrayScratchSize + 
                                         arrayscratchlocation];
-#ifdef MOLDEBUG
-        if (CCTK_EQUALS(verbose,"extreme"))
-        {
-/*           printf("Reading from scratch space, initial address %ld index %d\n",  */
-/*                  ScratchVar, scratchindex*(MoL_Max_Evolved_Array_Size+1) +  */
-/*                  arrayscratchlocation); */
-          printf("Reading from scratch space, initial address %ld index %d\n", 
-                 ScratchVar, scratchindex*singlearraysize + 
-                 arrayscratchlocation);
-        }
-#endif
       }
       else
       {
@@ -437,13 +396,14 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
         for (index = 0; index < arraytotalsize; index++)
         {
           UpdateVar[index] += alpha * ScratchVar[index];
-#ifdef MOLDEBUG
+#ifdef MOLDEBUGARRAYS
           if (CCTK_EQUALS(verbose,"extreme"))
           {
-            printf("Variable: %d. Index: %d. step: %d. "
-                   "alpha: %f. Scratch: %f. q: %f.\n",
-                   var, index, (*MoL_Intermediate_Step), 
-                   alpha, ScratchVar[index], UpdateVar[index]);
+            CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING, 
+                       "Variable: %d. Index: %d. step: %d. "
+                       "alpha: %f. Scratch: %f. q: %f.\n",
+                       var, index, (*MoL_Intermediate_Step), 
+                       alpha, ScratchVar[index], UpdateVar[index]);
           }
 #endif
         }
@@ -465,41 +425,31 @@ void MoL_GenericRKAdd(CCTK_ARGUMENTS)
                                                 EvolvedArrayVariableIndex[var]);
       ScratchVar = &ArrayScratchSpace[(MoL_Intermediate_Steps -
                                        (*MoL_Intermediate_Step)) * 
-                                     singlearraysize +
+                                      CurrentArrayScratchSize +
+/*                                      singlearraysize + */
 /*                                      (MoL_Max_Evolved_Array_Size+1) +  */
                                      arrayscratchlocation];
-      groupindex = CCTK_GroupIndexFromVarI(EvolvedArrayVariableIndex[var]);
-      ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
-                                   &arraydata);
-      if (ierr)
-      {
-        CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, 
-                   "The driver does not return group information "
-                   "for group '%s'.", 
-                   CCTK_GroupName(groupindex));
-      }
-      arraytotalsize = 1;
-      for (arraydim = 0; arraydim < arraydata.dim; arraydim++)
-      {
-        arraytotalsize *= arraydata.lsh[arraydim];
-      }
 
-#ifdef MOLDEBUG
-      printf("Writing to scratch space, initial address %ld, index %d \n", 
-             ScratchVar, (MoL_Intermediate_Steps -
-                          (*MoL_Intermediate_Step)) * 
-             singlearraysize +
-/*              (MoL_Max_Evolved_Array_Size+1) +  */
-             arrayscratchlocation);
+      arraytotalsize = ArrayScratchSizes[var];
+
+
+#ifdef MOLDEBUGARRAYS
+      CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING, 
+                 "Writing to scratch space, initial address %ld, index %d \n", 
+                 ScratchVar, (MoL_Intermediate_Steps -
+                              (*MoL_Intermediate_Step)) * 
+                 CurrentArrayScratchSize +
+                 arrayscratchlocation);
 #endif
       for (index = 0; index < arraytotalsize; index++)
       {
         ScratchVar[index] = UpdateVar[index];
-#ifdef MOLDEBUG
+#ifdef MOLDEBUGARRAYS
         if (CCTK_EQUALS(verbose,"extreme"))
         {
-          printf("Variable: %d. Index: %d. step: %d. Scratch: %f.\n",
-                 var, index, (*MoL_Intermediate_Step), ScratchVar[index]);
+          CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING, 
+                     "Variable: %d. Index: %d. step: %d. Scratch: %f.\n",
+                     var, index, (*MoL_Intermediate_Step), ScratchVar[index]);
         }
 #endif        
       }
diff --git a/src/IndexArrays.c b/src/IndexArrays.c
index e30d169..3c7c1e0 100644
--- a/src/IndexArrays.c
+++ b/src/IndexArrays.c
@@ -465,6 +465,18 @@ void MoL_FreeIndexArrays(void)
     SandRComplexArrayVariableIndex = NULL;
   }
 
+  if (ArrayScratchSizes)
+  {
+    free(ArrayScratchSizes);
+    ArrayScratchSizes = NULL;
+  }
+  
+  if (ArrayScratchSpace)
+  {
+    free(ArrayScratchSpace);
+    ArrayScratchSpace = NULL;
+  }
+
   return;
   
 }
diff --git a/src/InitialCopy.c b/src/InitialCopy.c
index 2b63697..7948006 100644
--- a/src/InitialCopy.c
+++ b/src/InitialCopy.c
@@ -11,6 +11,7 @@
    @version   $Header$
  @@*/
 
+#include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -80,6 +81,7 @@ void MoL_InitialCopy(CCTK_ARGUMENTS)
   cGroupDynamicData arraydata;
   CCTK_INT groupindex, ierr;
   CCTK_INT arraytotalsize, arraydim;
+  CCTK_INT totalarrayscratchsize;
 
   CCTK_INT var;
   CCTK_INT totalsize;
@@ -144,6 +146,23 @@ void MoL_InitialCopy(CCTK_ARGUMENTS)
 
   }
     
+  /* Set up the array sizes */
+
+  if (!ArrayScratchSizes)
+  {
+    ArrayScratchSizes = (CCTK_INT *)malloc(MoLNumEvolvedArrayVariables * sizeof(CCTK_INT));
+    if (!ArrayScratchSizes)
+    {
+      CCTK_WARN(0, "Failed to allocate the array scratch sizes array.");
+    }
+    for (var = 0; var < MoLNumEvolvedArrayVariables; var++)
+    {
+      ArrayScratchSizes[var] = -1;
+    }
+  }
+
+  totalarrayscratchsize = 0;
+
   for (var = 0; var < MoLNumEvolvedArrayVariables; var++)
   {
     PreviousVar = (CCTK_REAL const*)CCTK_VarDataPtrI(cctkGH, 1,
@@ -167,6 +186,9 @@ void MoL_InitialCopy(CCTK_ARGUMENTS)
       arraytotalsize *= arraydata.lsh[arraydim];
     }
 
+    ArrayScratchSizes[var] = arraytotalsize;
+    totalarrayscratchsize += arraytotalsize;
+
     if (arraytotalsize)
     {  
       if (PreviousVar && CurrentVar)
@@ -184,6 +206,25 @@ void MoL_InitialCopy(CCTK_ARGUMENTS)
     
   }
 
+  if (totalarrayscratchsize > CurrentArrayScratchSize)
+  {
+    if (ArrayScratchSpace)
+    {
+      free(ArrayScratchSpace);
+      ArrayScratchSpace = NULL;
+    }
+    ArrayScratchSpace = 
+      (CCTK_REAL*)malloc(totalarrayscratchsize * 
+                         MoL_Num_Scratch_Levels * 
+                         sizeof(CCTK_REAL));
+    for (var = 0; var < totalarrayscratchsize * MoL_Num_Scratch_Levels; var++)
+    {
+      ArrayScratchSpace[var] = 0.0;
+    }
+    CurrentArrayScratchSize = totalarrayscratchsize;
+  }
+  
+
   /* FIXME */
 
 #ifdef MOLDOESCOMPLEX
diff --git a/src/RK4.c b/src/RK4.c
index 940402b..c96377a 100644
--- a/src/RK4.c
+++ b/src/RK4.c
@@ -212,27 +212,6 @@ CCTK_WARN(0, "not done");
 
   arrayscratchlocation = 0;
 
-  arraytotalsize = MoL_Max_Evolved_Array_Size;
-  if (MoL_Num_Scratch_Levels)
-  {
-    groupindex = CCTK_GroupIndex("MOL::ARRAYSCRATCHSPACE");
-    ierr = CCTK_GroupDynamicData(cctkGH, groupindex,
-                                 &arraydata);
-    if (ierr)
-    {
-      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING, 
-                 "The driver does not return group information "
-                 "for group '%s' (%d).", 
-                 CCTK_GroupName(groupindex), ierr);
-    }
-    arraytotalsize = 1;
-    for (arraydim = 0; arraydim < arraydata.dim; arraydim++)
-    {
-      arraytotalsize *= arraydata.lsh[arraydim];
-    }
-  }
-  singlearraysize = arraytotalsize / MoLNumEvolvedArrayVariables;
-
   for (var = 0; var < MoLNumEvolvedArrayVariables; var++)
   {
     
diff --git a/src/Startup.c b/src/Startup.c
index 9b15e6b..06ae44d 100644
--- a/src/Startup.c
+++ b/src/Startup.c
@@ -66,6 +66,10 @@ CCTK_INT MoLNumSandRComplexArrayVariables = 0;
 
 CCTK_INT ScheduleStatus = 0;
 
+CCTK_REAL *ArrayScratchSpace = NULL;
+CCTK_INT *ArrayScratchSizes = NULL;
+CCTK_INT CurrentArrayScratchSize = 0;
+
 /********************************************************************
  *********************     Local Data Types   ***********************
  ********************************************************************/