From 6a344ba05da78f55de86fc966817cfdba159bc82 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Mon, 5 Aug 2013 14:16:18 -0400
Subject: Carpet: Store amount of allocated memory in grid scalars

---
 Carpet/Carpet/interface.ccl | 13 +++++-
 Carpet/Carpet/schedule.ccl  |  1 +
 Carpet/Carpet/src/Timing.cc | 96 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl
index 9bf2ee08b..edd3e75a1 100644
--- a/Carpet/Carpet/interface.ccl
+++ b/Carpet/Carpet/interface.ccl
@@ -61,7 +61,7 @@ PROVIDES FUNCTION CCTK_MyHost       WITH Carpet_MyHost       LANGUAGE C
 PROVIDES FUNCTION CCTK_nHosts       WITH Carpet_nHosts       LANGUAGE C
 PROVIDES FUNCTION CCTK_nProcsOnHost WITH Carpet_nProcsOnHost LANGUAGE C
 PROVIDES FUNCTION CCTK_ProcsOnHost  WITH Carpet_ProcsOnHost  LANGUAGE C
-
+    
 
 
 # Return total amount of memory used by Carpet
@@ -256,6 +256,7 @@ CCTK_INT FUNCTION Accelerator_NotifyVariableWritten       \
 USES FUNCTION Accelerator_NotifyVariableWritten
 
 
+
 # Access coordinate information (on the coarse level)
 CCTK_INT FUNCTION GetCoordRange        \
     (CCTK_POINTER_TO_CONST IN  cctkGH, \
@@ -406,6 +407,8 @@ CCTK_INT point_classification TYPE=gf TAGS='checkpoint="no" prolongation="none"'
   #                n>1: used only while time stepping (e.g. buffer zones)
 } "Grid point classification"
 
+CCTK_INT carpet_unusedpoints_mask TYPE=GF tags='Prolongation="none"' "mask which is set !=0 for points which do not influence future evolution, assuming appropriate regridding"
+
 
 
 CCTK_REAL timing TAGS='checkpoint="no"'
@@ -448,4 +451,10 @@ CCTK_REAL timing_levels TYPE=array DIM=1 SIZE=max_refinement_levels DISTRIB=cons
   time_level time_level_count
 } "Per-level timing information"
 
-CCTK_INT carpet_unusedpoints_mask TYPE=GF tags='Prolongation="none"' "mask which is set !=0 for points which do not influence future evolution, assuming appropriate regridding"
+CCTK_REAL memory_procs TYPE=array DIM=1 SIZE=1 DISTRIB=CONSTANT TAGS='checkpoint="no"'
+{
+  metadata
+  grid_structure
+  grid_arrays
+  grid_functions
+} "Memory usage information"
diff --git a/Carpet/Carpet/schedule.ccl b/Carpet/Carpet/schedule.ccl
index d726b58aa..dd38100a8 100644
--- a/Carpet/Carpet/schedule.ccl
+++ b/Carpet/Carpet/schedule.ccl
@@ -3,6 +3,7 @@
 # This is disabled because its implementation is not complete
 #storage: point_classification
 storage: timing timing_procs timing_levels
+storage: memory_procs
 
 schedule CarpetMultiModelStartup at STARTUP as MultiModel_Startup before Driver_Startup
 {
diff --git a/Carpet/Carpet/src/Timing.cc b/Carpet/Carpet/src/Timing.cc
index 6f58b3a3b..470f6367d 100644
--- a/Carpet/Carpet/src/Timing.cc
+++ b/Carpet/Carpet/src/Timing.cc
@@ -461,6 +461,101 @@ namespace Carpet {
   
   
   
+  static
+  void
+  UpdateMemoryStats (cGH const * const cctkGH)
+  {
+    DECLARE_CCTK_ARGUMENTS;
+    
+    // Various metadata, excluding the small change
+    *metadata = 0.0 +
+      // memoryof(main_timer_tree) +
+      // memoryof(mode_timer_tree) +
+      memoryof(origin_space) +
+      memoryof(delta_space) +
+      // memoryof(domainspecs) +
+      memoryof(vhh) +
+      memoryof(vdd) +
+      memoryof(level_regridding_epochs);
+      // memoryof(groupdata) +
+      // memoryof(arrdata);
+    
+    // TODO: Add TimerNode.root(?), TimerSet.timers
+    
+    // Grid structure
+    *grid_structure = 0.0;
+    // Storage for grid arrays
+    *grid_arrays = 0.0;
+    // Storage for grid functions
+    *grid_functions = 0.0;
+    
+    for (size_t m=0; m<vhh.size(); ++m) {
+      *grid_structure += memoryof(*vhh.AT(m));
+    }
+    for (size_t m=0; m<vdd.size(); ++m) {
+      *grid_structure += memoryof(*vdd.AT(m));
+    }
+    *grid_structure += memoryof(*tt);
+    
+    const int myproc = CCTK_MyProc(0);
+    for (size_t gi=0; gi<arrdata.size(); ++gi) {
+      if (CCTK_GroupTypeI(gi) == CCTK_GF) {
+        for (size_t m=0; m<arrdata.AT(gi).size(); ++m) {
+          const gh& hh = *vhh.AT(m);
+          for (size_t vi=0; vi<arrdata.AT(gi).AT(m).data.size(); ++vi) {
+            const ggf* f = arrdata.AT(gi).AT(m).data.AT(vi);
+            if (f) {
+              *grid_structure += memoryof(*f);
+              for (int ml=0; ml<hh.mglevels(); ++ml) {
+                for (int rl=0; rl<hh.reflevels(); ++rl) {
+                  const int tls = f->timelevels(ml, rl);
+                  for (int lc=0; lc<hh.local_components(rl); ++lc) {
+                    for (int tl=0; tl<tls; ++tl) {
+                      const gdata* d = f->data_pointer(tl, rl, lc, ml);
+                      *grid_structure += memoryof(*d);
+                      // TODO: add memoryof(d->_memory);
+                      if (d->has_storage()) {
+                        assert(d->proc() == myproc);
+                        *grid_functions += d->size() * d->elementsize();
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      } else {                  // not CCTK_GF
+        const int m = 0;
+        *grid_structure += 0.0 +
+          memoryof(*arrdata.AT(gi).AT(m).hh) +
+          memoryof(*arrdata.AT(gi).AT(m).dd) +
+          memoryof(*arrdata.AT(gi).AT(m).tt);
+        for (size_t vi=0; vi<arrdata.AT(gi).AT(m).data.size(); ++vi) {
+          const ggf* f = arrdata.AT(gi).AT(m).data.AT(vi);
+          if (f) {
+            *grid_structure += memoryof(*f);
+            const int ml = 0;
+            const int rl = 0;
+            const int tls = f->timelevels(ml, rl);
+            const int lc = 0;
+            for (int tl=0; tl<tls; ++tl) {
+              const gdata* d = f->data_pointer(tl, rl, lc, ml);
+              *grid_arrays += memoryof(*d);
+              // TODO: add memoryof(d->_memory);
+              if (d->has_storage()) {
+                assert(d->proc() == myproc);
+                *grid_functions += d->size() * d->elementsize();
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  
+  
+  
   // Calculate timing statistics (to be called before output)
   void
   UpdateTimingStats (cGH const * const cctkGH)
@@ -470,6 +565,7 @@ namespace Carpet {
     UpdateIOStats (cctkGH);
     UpdateCommunicationStats (cctkGH);
     UpdatePhysicalTimePerHour (cctkGH);
+    UpdateMemoryStats (cctkGH);
   }
   
   
-- 
cgit v1.2.3