CarpetLib: Split mem.cc into two files mem.cc and memstat.cc

mem.cc contains the mem class, memstat.cc the CarpetLib memory
statistics.

4 files changed, 217 insertions, 185 deletions

diff --git a/Carpet/CarpetLib/src/make.code.defn b/Carpet/CarpetLib/src/make.code.defn
index 0e7153bef..ccf896ae5 100644
--- a/Carpet/CarpetLib/src/make.code.defn
+++ b/Carpet/CarpetLib/src/make.code.defn
@@ -15,6 +15,7 @@ SRCS =	bbox.cc					\
 	gh.cc					\
 	limits.cc				\
 	mem.cc					\
+	memstat.cc				\
 	mpi_string.cc				\
 	region.cc				\
 	startup_time.cc				\
diff --git a/Carpet/CarpetLib/src/mem.cc b/Carpet/CarpetLib/src/mem.cc
index e81e86aa9..bb66c3b8f 100644
--- a/Carpet/CarpetLib/src/mem.cc
+++ b/Carpet/CarpetLib/src/mem.cc
@@ -9,24 +9,9 @@
 #include <string>
 
 #include "cctk.h"
-#include "cctk_Arguments.h"
 #include "cctk_Parameters.h"
 
-#ifdef HAVE_MALLOC_H
-#  include <malloc.h>
-#endif
-
-#include <sys/resource.h>
-#include <sys/time.h>
-
 #include "defs.hh"
-#include "dist.hh"
-#include "dh.hh"
-#include "gdata.hh"
-#include "ggf.hh"
-#include "gh.hh"
-#include "th.hh"
-
 #include "mem.hh"
 
 
@@ -35,33 +20,15 @@ using namespace std;
 
 
 
-double const MEGA = 1024*1024;
-
-
-
-struct mstat {
-  // Carpet object statistics
-  double total_bytes;
-  double total_objects;
-  double max_bytes;
-  double max_objects;
-  // Carpet administrative data structure statistics
-  double total_admin_bytes;
-  // malloc statistics
-  double malloc_used_bytes;
-  double malloc_free_bytes;
-};
-int const mstat_entries = sizeof(mstat) / sizeof(double);
-
-
+double const gmem::MEGA = 1000*1000;
 
 // Total number of currently allocated bytes and objects
-static double total_allocated_bytes   = 0;
-static double total_allocated_objects = 0;
+double gmem::total_allocated_bytes   = 0;
+double gmem::total_allocated_objects = 0;
 
 // Maximum of the above (over time)
-static double max_allocated_bytes   = 0;
-static double max_allocated_objects = 0;
+double gmem::max_allocated_bytes   = 0;
+double gmem::max_allocated_objects = 0;
 
 
 
@@ -225,7 +192,7 @@ alloc (size_t nbytes)
     if (not freeptr) {
       CCTK_VWarn (CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
                   "Failed to allocate %.3f MB of memory",
-                  double(freesize/MEGA));
+                  double(freesize/gmem::MEGA));
     }
     // Remember the pointer so that it can be freed
     chunks.push (freeptr);
@@ -258,151 +225,6 @@ memory ()
 
 
 
-////////////////////////////////////////////////////////////////////////////////
-
-
-
-extern "C" void CarpetLib_printmemstats (CCTK_ARGUMENTS);
-
-void CarpetLib_printmemstats (CCTK_ARGUMENTS)
-{
-  DECLARE_CCTK_ARGUMENTS;
-  DECLARE_CCTK_PARAMETERS;
-  
-  int const ioproc = 0;
-  
-  if ((print_memstats_every == 0 and cctk_iteration == 0) or
-      (print_memstats_every > 0 and cctk_iteration % print_memstats_every == 0))
-  {
-    mstat mybuf;
-    mybuf.total_bytes   = total_allocated_bytes;
-    mybuf.total_objects = total_allocated_objects;
-    mybuf.max_bytes     = max_allocated_bytes;
-    mybuf.max_objects   = max_allocated_objects;
-    mybuf.total_admin_bytes =
-      gh::allmemory() + dh::allmemory() + th::allmemory() +
-      ggf::allmemory() + gdata::allmemory();
-#ifdef HAVE_MALLINFO
-    // NOTE: struct mallinfo returns byte-counts as int, which can
-    // overflow.  In this case, the information is incorrect.
-    struct mallinfo const minfo = mallinfo ();
-    mybuf.malloc_used_bytes = minfo.uordblks;
-    mybuf.malloc_free_bytes = minfo.fordblks;
-#else
-    mybuf.malloc_used_bytes = 0;
-    mybuf.malloc_free_bytes = 0;
-#endif
-    
-    cout << "Memory statistics from CarpetLib:" << eol
-         << "   Current number of objects: " << total_allocated_objects << eol
-         << "   Current allocated memory:  "
-         << setprecision(3) << total_allocated_bytes / MEGA << " MB" << eol
-         << "   Maximum number of objects: " << max_allocated_objects << eol
-         << "   Maximum allocated memory:  "
-         << setprecision(3) << max_allocated_bytes / MEGA << " MB" << eol
-         << "   Current administrative memory: "
-         << setprecision(3) << mybuf.total_admin_bytes / MEGA << " MB" << eol
-         << "   Total allocated used system memory: "
-         << setprecision(3) << mybuf.malloc_used_bytes / MEGA << " MB" << eol
-         << "   Total allocated free system memory: "
-         << setprecision(3) << mybuf.malloc_free_bytes / MEGA << " MB" << endl;
-    
-#warning "TODO"
-    cout << "   gh::allmemory:    " << gh   ::allmemory() << eol
-         << "   dh::allmemory:    " << dh   ::allmemory() << eol
-         << "   th::allmemory:    " << th   ::allmemory() << eol
-         << "   ggf::allmemory:   " << ggf  ::allmemory() << eol
-         << "   gdata::allmemory: " << gdata::allmemory() << endl;
-    
-    if (strcmp (memstat_file, "") != 0) {
-      vector<mstat> allbuf (dist::size());
-      MPI_Gather (& mybuf, mstat_entries, MPI_DOUBLE,
-                  & allbuf.front(), mstat_entries, MPI_DOUBLE,
-                  ioproc, dist::comm());
-      
-      if (dist::rank() == ioproc) {
-        
-        double max_total_bytes = 0;
-        double avg_total_bytes = 0;
-        double cnt_total_bytes = 0;
-        double max_max_bytes = 0;
-        double avg_max_bytes = 0;
-        double cnt_max_bytes = 0;
-        double max_admin_bytes = 0;
-        double avg_admin_bytes = 0;
-        double cnt_admin_bytes = 0;
-        double max_used_bytes = 0;
-        double avg_used_bytes = 0;
-        double cnt_used_bytes = 0;
-        double max_free_bytes = 0;
-        double avg_free_bytes = 0;
-        double cnt_free_bytes = 0;
-        for (size_t n=0; n<allbuf.size(); ++n) {
-          max_total_bytes = max (max_total_bytes, allbuf[n].total_bytes);
-          avg_total_bytes += allbuf[n].total_bytes;
-          ++ cnt_total_bytes;
-          max_max_bytes = max (max_max_bytes, allbuf[n].max_bytes);
-          avg_max_bytes += allbuf[n].max_bytes;
-          ++ cnt_max_bytes;
-          max_admin_bytes = max (max_admin_bytes, allbuf[n].total_admin_bytes);
-          avg_admin_bytes += allbuf[n].total_admin_bytes;
-          ++ cnt_admin_bytes;
-          max_used_bytes = max (max_used_bytes, allbuf[n].malloc_used_bytes);
-          avg_used_bytes += allbuf[n].malloc_used_bytes;
-          ++ cnt_used_bytes;
-          max_free_bytes = max (max_free_bytes, allbuf[n].malloc_free_bytes);
-          avg_free_bytes += allbuf[n].malloc_free_bytes;
-          ++ cnt_free_bytes;
-        }
-        avg_total_bytes /= cnt_total_bytes;
-        avg_max_bytes /= cnt_max_bytes;
-        avg_admin_bytes /= cnt_admin_bytes;
-        avg_used_bytes /= cnt_used_bytes;
-        avg_free_bytes /= cnt_free_bytes;
-        
-        ostringstream filenamebuf;
-        filenamebuf << out_dir << "/" << memstat_file;
-        string const filename = filenamebuf.str();
-        ofstream file;
-        static bool did_truncate = false;
-        if (not did_truncate) {
-          did_truncate = true;
-          file.open (filename.c_str(), ios::out | ios::trunc);
-          if (CCTK_IsFunctionAliased ("UniqueBuildID")) {
-            char const * const build_id
-              = static_cast<char const *> (UniqueBuildID (cctkGH));
-            file << "# Build ID: " << build_id << eol;
-          }
-          if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
-            char const * const job_id
-              = static_cast<char const *> (UniqueSimulationID (cctkGH));
-            file << "# Simulation ID: " << job_id << eol;
-          }
-          file << "# Running on " << dist::size() << " processors" << eol;
-          file << "#" << eol;
-          file << "# iteration   maxtotalbytes avgtotalbytes   maxmaxbytes avgm avgfreebytes" << eol;
-        } else {
-          file.open (filename.c_str(), ios::out | ios::app);
-        }
-        
-        file << cctk_iteration
-              << "\t "<< max_total_bytes << " " << avg_total_bytes
-              << "\t "<< max_max_bytes << " " << avg_max_bytes
-              << "\t "<< max_admin_bytes << " " << avg_admin_bytes
-              << "\t "<< max_used_bytes << " " << avg_used_bytes
-              << "\t "<< max_free_bytes << " " << avg_free_bytes
-             << eol;
-        
-        file.close ();
-        
-      } // if on root processor
-    } // if output to file
-    
-  }
-}
-
-
-
 #define TYPECASE(N,T)                           \
   template class mem<T>;
 
diff --git a/Carpet/CarpetLib/src/mem.hh b/Carpet/CarpetLib/src/mem.hh
index fe8694d32..5fdc70018 100644
--- a/Carpet/CarpetLib/src/mem.hh
+++ b/Carpet/CarpetLib/src/mem.hh
@@ -1,15 +1,39 @@
 #ifndef MEM_HH
 #define MEM_HH
 
+#include <cctk.h>
+
+#include <cassert>
 #include <cstdlib>
 #include <stack>
 #include <vector>
 
+#include "defs.hh"
+
 using namespace std;
 
+
+
 // A chunk of memory, possibly shared between some clients
+class gmem
+{
+public:
+  
+  static double const MEGA;
+  
+  // Total number of currently allocated bytes and objects
+  static double total_allocated_bytes;
+  static double total_allocated_objects;
+  
+  // Maximum of the above (over time)
+  static double max_allocated_bytes;
+  static double max_allocated_objects;
+};
+
+
+
 template<typename T>
-class mem
+class mem: gmem
 {
   T * storage_;
   size_t nelems_;
diff --git a/Carpet/CarpetLib/src/memstat.cc b/Carpet/CarpetLib/src/memstat.cc
new file mode 100644
index 000000000..96b0fbc52
--- /dev/null
+++ b/Carpet/CarpetLib/src/memstat.cc
@@ -0,0 +1,185 @@
+#include <cctk.h>
+#include <cctk_Arguments.h>
+#include <cctk_Parameters.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#ifdef HAVE_MALLOC_H
+#  include <malloc.h>
+#endif
+
+#include <sys/resource.h>
+#include <sys/time.h>
+
+#include "mem.hh"
+
+#include "dh.hh"
+#include "gdata.hh"
+#include "ggf.hh"
+#include "gh.hh"
+#include "th.hh"
+
+
+
+struct mstat {
+  // Carpet object statistics
+  double total_bytes;
+  double total_objects;
+  double max_bytes;
+  double max_objects;
+  // Carpet administrative data structure statistics
+  double total_admin_bytes;
+  // malloc statistics
+  double malloc_used_bytes;
+  double malloc_free_bytes;
+};
+int const mstat_entries = sizeof(mstat) / sizeof(double);
+
+
+
+extern "C" void CarpetLib_printmemstats (CCTK_ARGUMENTS);
+
+void CarpetLib_printmemstats (CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+  
+  int const ioproc = 0;
+  
+  if ((print_memstats_every == 0 and cctk_iteration == 0) or
+      (print_memstats_every > 0 and cctk_iteration % print_memstats_every == 0))
+  {
+    mstat mybuf;
+    mybuf.total_bytes   = gmem::total_allocated_bytes;
+    mybuf.total_objects = gmem::total_allocated_objects;
+    mybuf.max_bytes     = gmem::max_allocated_bytes;
+    mybuf.max_objects   = gmem::max_allocated_objects;
+    mybuf.total_admin_bytes =
+      gh::allmemory() + dh::allmemory() + th::allmemory() +
+      ggf::allmemory() + gdata::allmemory();
+#ifdef HAVE_MALLINFO
+    // NOTE: struct mallinfo returns byte-counts as int, which can
+    // overflow.  In this case, the information is incorrect.
+    struct mallinfo const minfo = mallinfo ();
+    mybuf.malloc_used_bytes = minfo.uordblks;
+    mybuf.malloc_free_bytes = minfo.fordblks;
+#else
+    mybuf.malloc_used_bytes = 0;
+    mybuf.malloc_free_bytes = 0;
+#endif
+    
+    cout << "Memory statistics from CarpetLib:" << eol
+         << "   Current number of objects: " << gmem::total_allocated_objects << eol
+         << "   Current allocated memory:  "
+         << setprecision(3) << gmem::total_allocated_bytes / gmem::MEGA << " MB" << eol
+         << "   Maximum number of objects: " << gmem::max_allocated_objects << eol
+         << "   Maximum allocated memory:  "
+         << setprecision(3) << gmem::max_allocated_bytes / gmem::MEGA << " MB" << eol
+         << "   Current administrative memory: "
+         << setprecision(3) << mybuf.total_admin_bytes / gmem::MEGA << " MB" << eol
+         << "   Total allocated used system memory: "
+         << setprecision(3) << mybuf.malloc_used_bytes / gmem::MEGA << " MB" << eol
+         << "   Total allocated free system memory: "
+         << setprecision(3) << mybuf.malloc_free_bytes / gmem::MEGA << " MB" << endl;
+    
+#warning "TODO"
+    cout << "   gh::allmemory:    " << gh   ::allmemory() << eol
+         << "   dh::allmemory:    " << dh   ::allmemory() << eol
+         << "   th::allmemory:    " << th   ::allmemory() << eol
+         << "   ggf::allmemory:   " << ggf  ::allmemory() << eol
+         << "   gdata::allmemory: " << gdata::allmemory() << endl;
+    
+    if (strcmp (memstat_file, "") != 0) {
+      vector<mstat> allbuf (dist::size());
+      MPI_Gather (& mybuf, mstat_entries, MPI_DOUBLE,
+                  & allbuf.front(), mstat_entries, MPI_DOUBLE,
+                  ioproc, dist::comm());
+      
+      if (dist::rank() == ioproc) {
+        
+        double max_total_bytes = 0;
+        double avg_total_bytes = 0;
+        double cnt_total_bytes = 0;
+        double max_max_bytes = 0;
+        double avg_max_bytes = 0;
+        double cnt_max_bytes = 0;
+        double max_admin_bytes = 0;
+        double avg_admin_bytes = 0;
+        double cnt_admin_bytes = 0;
+        double max_used_bytes = 0;
+        double avg_used_bytes = 0;
+        double cnt_used_bytes = 0;
+        double max_free_bytes = 0;
+        double avg_free_bytes = 0;
+        double cnt_free_bytes = 0;
+        for (size_t n=0; n<allbuf.size(); ++n) {
+          max_total_bytes = max (max_total_bytes, allbuf[n].total_bytes);
+          avg_total_bytes += allbuf[n].total_bytes;
+          ++ cnt_total_bytes;
+          max_max_bytes = max (max_max_bytes, allbuf[n].max_bytes);
+          avg_max_bytes += allbuf[n].max_bytes;
+          ++ cnt_max_bytes;
+          max_admin_bytes = max (max_admin_bytes, allbuf[n].total_admin_bytes);
+          avg_admin_bytes += allbuf[n].total_admin_bytes;
+          ++ cnt_admin_bytes;
+          max_used_bytes = max (max_used_bytes, allbuf[n].malloc_used_bytes);
+          avg_used_bytes += allbuf[n].malloc_used_bytes;
+          ++ cnt_used_bytes;
+          max_free_bytes = max (max_free_bytes, allbuf[n].malloc_free_bytes);
+          avg_free_bytes += allbuf[n].malloc_free_bytes;
+          ++ cnt_free_bytes;
+        }
+        avg_total_bytes /= cnt_total_bytes;
+        avg_max_bytes /= cnt_max_bytes;
+        avg_admin_bytes /= cnt_admin_bytes;
+        avg_used_bytes /= cnt_used_bytes;
+        avg_free_bytes /= cnt_free_bytes;
+        
+        ostringstream filenamebuf;
+        filenamebuf << out_dir << "/" << memstat_file;
+        string const filename = filenamebuf.str();
+        ofstream file;
+        static bool did_truncate = false;
+        if (not did_truncate) {
+          did_truncate = true;
+          file.open (filename.c_str(), ios::out | ios::trunc);
+          if (CCTK_IsFunctionAliased ("UniqueBuildID")) {
+            char const * const build_id
+              = static_cast<char const *> (UniqueBuildID (cctkGH));
+            file << "# Build ID: " << build_id << eol;
+          }
+          if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
+            char const * const job_id
+              = static_cast<char const *> (UniqueSimulationID (cctkGH));
+            file << "# Simulation ID: " << job_id << eol;
+          }
+          file << "# Running on " << dist::size() << " processors" << eol;
+          file << "#" << eol;
+          file << "# iteration   maxtotalbytes avgtotalbytes   maxmaxbytes avgm avgfreebytes" << eol;
+        } else {
+          file.open (filename.c_str(), ios::out | ios::app);
+        }
+        
+        file << cctk_iteration
+              << "\t "<< max_total_bytes << " " << avg_total_bytes
+              << "\t "<< max_max_bytes << " " << avg_max_bytes
+              << "\t "<< max_admin_bytes << " " << avg_admin_bytes
+              << "\t "<< max_used_bytes << " " << avg_used_bytes
+              << "\t "<< max_free_bytes << " " << avg_free_bytes
+             << eol;
+        
+        file.close ();
+        
+      } // if on root processor
+    } // if output to file
+    
+  }
+}