Carpet: Provide GroupStorage{In,De}crease instead of {En,Dis}ableGroupStorage

darcs-hash:20050128113401-891bb-51eae41cd181ee8fbce59f429906046c131ba1b0.gz

5 files changed, 201 insertions, 153 deletions

diff --git a/Carpet/Carpet/src/CarpetStartup.cc b/Carpet/Carpet/src/CarpetStartup.cc
index c7ae8df18..b228a2980 100644
--- a/Carpet/Carpet/src/CarpetStartup.cc
+++ b/Carpet/Carpet/src/CarpetStartup.cc
@@ -25,6 +25,8 @@ namespace Carpet {
     CCTK_OverloadSyncGroup (SyncGroup);
     CCTK_OverloadEnableGroupStorage (EnableGroupStorage);
     CCTK_OverloadDisableGroupStorage (DisableGroupStorage); 
+    CCTK_OverloadGroupStorageIncrease (GroupStorageIncrease);
+    CCTK_OverloadGroupStorageDecrease (GroupStorageDecrease); 
     CCTK_OverloadEnableGroupComm (EnableGroupComm);
     CCTK_OverloadDisableGroupComm (DisableGroupComm);
     CCTK_OverloadBarrier (Barrier);
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index 65c71745e..24cdb9997 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -1019,7 +1019,9 @@ namespace Carpet {
     groupdata.at(group).info.nghostzones = new int [dim];
       
     groupdata.at(group).transport_operator = GetTransportOperator (cgh, group);
-      
+    
+    groupdata.at(group).activetimelevels = 0;
+    
     // Initialise group variables
     for (int m=0; m<(int)arrdata.at(group).size(); ++m) {
       
diff --git a/Carpet/Carpet/src/Storage.cc b/Carpet/Carpet/src/Storage.cc
index 9a07dee7b..0e606d9bd 100644
--- a/Carpet/Carpet/src/Storage.cc
+++ b/Carpet/Carpet/src/Storage.cc
@@ -17,190 +17,229 @@ namespace Carpet {
   
   
   
-  int EnableGroupStorage (const cGH* cgh, const char* groupname)
+  static int
+  GroupStorageCrease (const cGH* cgh, int n_groups, const int* groups,
+                      const int* timelevels, int* status,
+                      const bool inc);
+  
+  int
+  GroupStorageCrease (const cGH* cgh, int n_groups, const int* groups,
+                      const int* timelevels, int* status,
+                      const bool inc)
   {
     DECLARE_CCTK_PARAMETERS;
     
-    Checkpoint ("EnableGroupStorage \"%s\"", groupname);
-    
-    // TODO: Enabling storage for one refinement level has to enable
-    // it for all other refinement levels as well.  Disabling must
-    // wait until all refinement levels have been disabled.
-    
-    const int group = CCTK_GroupIndex(groupname);
-    assert (group>=0 and group<CCTK_NumGroups());
-    
-    if (CCTK_NumVarsInGroupI(group)==0) return 0;
-    
-    const int grouptype = CCTK_GroupTypeI(group);
-    
-    // No storage change in local mode
-    if (grouptype == CCTK_GF) {
-      assert ((map == -1 or maps == 1)
-              and (component == -1
-                   or vhh.at(0)->local_components(reflevel) == 1));
-    }
-    
-    if (CCTK_QueryGroupStorageI(cgh, group)) {
-      // storage was enabled previously
-      const int n0 = CCTK_FirstVarIndexI(group);
-      assert (n0>=0);
-      const int num_tl = CCTK_NumTimeLevelsFromVarI(n0);
-      assert (num_tl>0);
-      return num_tl;
-    }
-    
-    // There is a difference between the Cactus time levels and the
-    // Carpet time levels.  If there are n time levels, then the
-    // Cactus time levels are numbered 0 ... n-1, with the current
-    // time level being 0.  In Carpet, the time levels are numbered
-    // -(n-1) ... 0, where the current time level is also 0.
-    const int n0 = CCTK_FirstVarIndexI(group);
-    assert (n0>=0);
-    const int num_tl = CCTK_NumTimeLevelsFromVarI(n0);
-    assert (num_tl>0);
-    const int tmin = 1 - num_tl;
-    const int tmax = 0;
-    
-    if (grouptype == CCTK_GF) {
-      if (max_refinement_levels > 1) {
-        if (groupdata.at(group).transport_operator != op_none) {
-          if (num_tl <= prolongation_order_time) {
-            CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
-                        "There are not enough time levels for the desired temporal prolongation order in the grid function group \"%s\".  With Carpet::prolongation_order_time=%d, you need at least %d time levels.",
-                        CCTK_GroupName(group),
-                        prolongation_order_time, prolongation_order_time+1);
-          }
-        }
+    assert (cgh);
+    assert (n_groups >= 0);
+    assert (groups);
+    assert (timelevels);
+    for (int n=0; n<n_groups; ++n) {
+      assert (groups[n] >= 0 and groups[n] < CCTK_NumGroups());
+#if 0
+      for (int nn=0; nn<n; ++nn) {
+        assert (groups[nn] != groups[n]);
       }
+#endif
+      assert (timelevels[n] >= 0);
     }
     
-    cGroup gp;
-    const int ierr = CCTK_GroupData (group, &gp);
-    assert (!ierr);
-    const int vectorlength = gp.vectorgroup ? gp.vectorlength : 1;
+    int total_num_timelevels = 0;
     
-    assert (arrdata.at(group).at(0).data.size()==0
-	    or arrdata.at(group).at(0).data.at(0) == 0);
-    for (int m=0; m<(int)arrdata.at(group).size(); ++m) {
-      for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) {
-        const int vectorindex = gp.vectorgroup ? var % vectorlength : 0;
-        ggf* vectorleader
-          = (gp.vectorgroup && vectorindex>0
-             ? arrdata.at(group).at(m).data.at(var - vectorindex)
-             : NULL);
-        const int n = n0 + var;
-        switch (CCTK_VarTypeI(n)) {
+    for (int n=0; n<n_groups; ++n) {
+      int const group = groups[n];
+      
+      cGroup gp;
+      const int ierr = CCTK_GroupData (group, &gp);
+      assert (! ierr);
+      
+      int const firstvarindex = CCTK_FirstVarIndexI (group);
+      assert (gp.numvars == 0
+              or (firstvarindex >= 0 and firstvarindex < CCTK_NumVars()));
+      
+      // Check an assumption
+      if (! gp.vectorgroup) assert (gp.vectorlength == 1);
+      
+      // No storage change in local mode
+      if (gp.grouptype == CCTK_GF) {
+        assert ((map == -1 or maps == 1)
+                and (component == -1
+                     or vhh.at(0)->local_components(reflevel) == 1));
+      }
+      
+      // Record previous number of allocated time levels
+      if (status) {
+        status[n] = groupdata.at(group).activetimelevels;
+      }
+      
+      // Only do something if the number of time levels actually needs
+      // to be changed -- do nothing otherwise
+      if ((inc and timelevels[n] > groupdata.at(group).activetimelevels)
+          or
+          (! inc and timelevels[n] < groupdata.at(group).activetimelevels))
+      {
+        
+        // Set the new number of active time levels
+        groupdata.at(group).activetimelevels = timelevels[n];
+        
+        // There is a difference between the Cactus time levels and
+        // the Carpet time levels.  If there are n time levels, then
+        // the Cactus time levels are numbered 0 ... n-1, with the
+        // current time level being 0.  In Carpet, the time levels are
+        // numbered -(n-1) ... 0, where the current time level is also
+        // 0.
+        const int tmin = - timelevels[n] + 1;
+        const int tmax = 0;
+        
+        // Allocate the time levels as well
+        for (int m=0; m<(int)arrdata.at(group).size(); ++m) {
+          for (int var=0; var<gp.numvars; ++var) {
+            const int vectorindex
+              = (gp.contiguous
+                 ? var
+                 : gp.vectorgroup ? var % gp.vectorlength : 0);
+            const int vectorlength
+              = (gp.contiguous
+                 ? gp.numvars
+                 : gp.vectorgroup ? gp.vectorlength : 1);
+            assert (vectorindex>=0 and vectorindex<gp.numvars);
+            assert (vectorlength>0 and vectorlength<=gp.numvars);
+            ggf* vectorleader
+              = (vectorindex>0
+                 ? arrdata.at(group).at(m).data.at(var - vectorindex)
+                 : NULL);
+            const int varindex = firstvarindex + var;
+            switch (gp.vartype) {
 #define TYPECASE(N,T)                                                   \
-        case N:                                                         \
-          arrdata.at(group).at(m).data.at(var) = new gf<T>              \
-            (n, groupdata.at(group).transport_operator,                 \
-             *arrdata.at(group).at(m).tt, *arrdata.at(group).at(m).dd,  \
-             tmin, tmax, prolongation_order_time,                       \
-             vectorlength, vectorindex, (gf<T>*)vectorleader);          \
-          break;
+              case N:                                                   \
+                arrdata.at(group).at(m).data.at(var) = new gf<T>        \
+                (varindex, groupdata.at(group).transport_operator,      \
+                 *arrdata.at(group).at(m).tt, *arrdata.at(group).at(m).dd, \
+                 tmin, tmax, prolongation_order_time,                   \
+                 vectorlength, vectorindex, (gf<T>*)vectorleader);      \
+              break;
 #include "typecase"
 #undef TYPECASE
-        default:
-          UnsupportedVarType(n);
-        } // switch
+            default:
+              UnsupportedVarType (varindex);
+            } // switch gp.vartype
+            
+            // Set the data pointers for grid arrays
+            if (gp.grouptype != CCTK_GF) {
+              assert (m == 0);
+              int const c = CCTK_MyProc(cgh);
+              for (int tl=0; tl<gp.numtimelevels; ++tl) {
+                cgh->data[varindex][tl]
+                  = (tl < groupdata.at(group).activetimelevels
+                     ? ((*arrdata.at(group).at(m).data.at(var))
+                        (-tl, 0, c, 0)->storage())
+                     : NULL);
+              }
+            }
+            
+          } // for var
+        } // for m
         
-        if (grouptype != CCTK_GF) {
-          for (int tl=0; tl<num_tl; ++tl) {
-            assert (m == 0);
-            int const c = CCTK_MyProc(cgh);
-            cgh->data[n][tl] = ((*arrdata.at(group).at(m).data.at(var))
-                                (-tl, 0, c, 0)->storage());
+      } // if really change the number of active time levels
+      
+#if 0
+      // Complain if there are not enough active time levels
+      if (gp.grouptype == CCTK_GF) {
+        if (max_refinement_levels > 1) {
+          if (groupdata.at(group).transport_operator != op_none) {
+            if (groupdata.at(group).activetimelevels != 0
+                and (groupdata.at(group).activetimelevels
+                     <= prolongation_order_time))
+            {
+              char * const groupname = CCTK_GroupName (group);
+              CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING,
+                          "There are not enough time levels for the desired temporal prolongation order in the grid function group \"%s\".  With Carpet::prolongation_order_time=%d, you need at least %d time levels.",
+                          groupname,
+                          prolongation_order_time, prolongation_order_time+1);
+              free (groupname);
+            }
           }
         }
-        
-      } // for
-    } // for
-    
-//     PoisonGroup (cgh, group, alltimes);
+      }
+#endif
+      
+      // Record current number of time levels
+      total_num_timelevels += groupdata.at(group).activetimelevels;
+      
+    } // for n
     
-    // storage was not enabled previously
-    return 0;
+    return total_num_timelevels;
   }
   
   
   
-  int DisableGroupStorage (const cGH* cgh, const char* groupname)
+  int
+  GroupStorageIncrease (const cGH* cgh, int n_groups, const int* groups,
+                        const int* timelevels, int* status)
+  {
+    Checkpoint ("GroupStorageIncrease");
+    return
+      GroupStorageCrease (cgh, n_groups, groups, timelevels, status, true);
+  }
+  
+  
+  
+  int
+  GroupStorageDecrease (const cGH* cgh, int n_groups, const int* groups,
+                        const int* timelevels, int* status)
+  {
+    Checkpoint ("GroupStorageDecrease");
+    return
+      GroupStorageCrease (cgh, n_groups, groups, timelevels, status, false);
+  }
+  
+  
+  
+  int
+  EnableGroupStorage (const cGH* cgh, const char* groupname)
   {
-    Checkpoint ("DisableGroupStorage \"%s\"", groupname);
-    
     const int group = CCTK_GroupIndex(groupname);
     assert (group>=0 and group<CCTK_NumGroups());
-    
-    // No storage change in local mode
-    assert (! (component!=-1 and CCTK_GroupTypeI(group)==CCTK_GF));
-    
-    if (! CCTK_QueryGroupStorageI(cgh, group)) {
-      // storage was disabled previously
-      return 0;
-    }
-    
-    // TODO
-    CCTK_WARN (2, "Cannot disable storage -- storage management is not yet consistent for FMR");
-    return 1;
-    
-    const int n0 = CCTK_FirstVarIndexI(group);
-    assert (n0>=0);
-    const int num_tl = CCTK_NumTimeLevelsFromVarI(n0);
-    assert (num_tl>0);
-    
-    for (int m=0; m<(int)arrdata.at(group).size(); ++m) {
-      for (int var=0; var<CCTK_NumVarsInGroupI(group); ++var) {
-        const int n = n0 + var;
-        switch (CCTK_VarTypeI(n)) {
-#define TYPECASE(N,T)                                                   \
-        case N:                                                         \
-          assert (arrdata.at(group).at(m).data.at(var));                \
-          delete (gf<T>*)arrdata.at(group).at(m).data.at(var);          \
-          arrdata.at(group).at(m).data.at(var) = NULL;                  \
-          break;
-#include "typecase"
-#undef TYPECASE
-        default:
-          UnsupportedVarType(n);
-        } // switch
-        arrdata.at(group).at(m).data.at(var) = NULL;
-        
-        if (CCTK_GroupTypeI(group) != CCTK_GF) {
-          for (int tl=0; tl<num_tl; ++tl) {
-            cgh->data[n][tl] = NULL;
-          }
-        }
-        
-      } // for
-    } // for
-    
-    // storage was enabled previously
-    return 1;
+    const int timelevels = CCTK_MaxTimeLevelsGI(group);
+    int status;
+    GroupStorageIncrease (cgh, 1, &group, &timelevels, &status);
+    // Return whether storage was allocated previously
+    return status;
   }
   
   
   
-  int QueryGroupStorageB (const cGH* cgh, int group, const char* groupname)
+  int
+  DisableGroupStorage (const cGH* cgh, const char* groupname)
+  {
+    const int group = CCTK_GroupIndex(groupname);
+    assert (group>=0 and group<CCTK_NumGroups());
+    const int timelevels = 0;
+    int status;
+    GroupStorageDecrease (cgh, 1, &group, &timelevels, &status);
+    // Return whether storage was allocated previously
+    return status;
+  }
+  
+  
+  
+  int
+  QueryGroupStorageB (const cGH* cgh, int group, const char* groupname)
   {
     if (groupname) {
       group = CCTK_GroupIndex(groupname);
     }
     assert (group>=0 and group<CCTK_NumGroups());
-    
-    if (CCTK_NumVarsInGroupI(group)==0) return 0;
-    
-    const int n = CCTK_FirstVarIndexI(group);
-    assert (n>=0 and n<CCTK_NumVars());
-    const int var = 0;
-    
-    return arrdata.at(group).at(0).data.at(var) != NULL;
+    const int timelevels = 0;
+    int status;
+    GroupStorageIncrease (cgh, 1, &group, &timelevels, &status);
+    // Return whether storage is allocated
+    return status;
   }
   
   
   
-  const int* ArrayGroupSizeB (const cGH* cgh, int dir, int group,
-			      const char* groupname)
+  const int*
+  ArrayGroupSizeB (const cGH* cgh, int dir, int group, const char* groupname)
   {
     static const int zero = 0;
     static const int error = 0;
diff --git a/Carpet/Carpet/src/functions.hh b/Carpet/Carpet/src/functions.hh
index be95997c9..e5fdf4ed2 100644
--- a/Carpet/Carpet/src/functions.hh
+++ b/Carpet/Carpet/src/functions.hh
@@ -19,10 +19,14 @@ namespace Carpet {
   using namespace std;
   
   int SyncGroup (const cGH* cgh, const char* groupname);
-  int EnableGroupStorage (const cGH* cgh, const char* groupname);
-  int DisableGroupStorage (const cGH* cgh, const char* groupname); 
   int EnableGroupComm (const cGH* cgh, const char* groupname);
   int DisableGroupComm (const cGH* cgh, const char* groupname);
+  int EnableGroupStorage (const cGH* cgh, const char* groupname);
+  int DisableGroupStorage (const cGH* cgh, const char* groupname); 
+  int GroupStorageIncrease (const cGH* cgh, int n_groups, const int* groups,
+                            const int* timelevels, int* status);
+  int GroupStorageDecrease (const cGH* cgh, int n_groups, const int* groups,
+                            const int* timelevels, int* status);
   int Barrier (const cGH* cgh);
   int Exit (cGH* cgh, int retval);
   int Abort (cGH* cgh, int retval);
diff --git a/Carpet/Carpet/src/variables.hh b/Carpet/Carpet/src/variables.hh
index ae5ade6df..821b77ae3 100644
--- a/Carpet/Carpet/src/variables.hh
+++ b/Carpet/Carpet/src/variables.hh
@@ -115,6 +115,7 @@ namespace Carpet {
   // Data for the groups
   struct groupdesc {
     cGroupDynamicData info;
+    int activetimelevels;
     operator_type transport_operator; // prolongation and restriction
   };
   extern vector<groupdesc> groupdata; // [group]