CarpetIOF5: Output fewer files

Honour IO::our_proc_every.
Combine files/groups/thorns/everything into one output file.
Compress output if desired.
Create a metafile with the directory.

8 files changed, 307 insertions, 40 deletions

diff --git a/CarpetDev/CarpetIOF5/param.ccl b/CarpetDev/CarpetIOF5/param.ccl
index 36663f2bf..45b0eeca1 100644
--- a/CarpetDev/CarpetIOF5/param.ccl
+++ b/CarpetDev/CarpetIOF5/param.ccl
@@ -8,13 +8,24 @@ BOOLEAN veryverbose "Produce much screen output while running"
 {
 } no
 
+
+
 SHARES: IO
 
+USES KEYWORD out_mode
+USES INT     out_proc_every
+
+USES INT     out_timesteps_per_file
+
+USES KEYWORD out_save_parameters
+
 USES STRING  out_dir       AS IO_out_dir
 USES KEYWORD out_criterion AS IO_out_criterion
 USES INT     out_every     AS IO_out_every
 USES REAL    out_dt        AS IO_out_dt
 
+
+
 PRIVATE:
 
 STRING out_dir "Output directory (overrides IO::out_dir)" STEERABLE=always
@@ -23,16 +34,36 @@ STRING out_dir "Output directory (overrides IO::out_dir)" STEERABLE=always
   ".+" :: "Not empty: directory name"
 } ""
 
-STRING out_filename "File name (without extension)" STEERABLE=always
+KEYWORD file_content "Create one file for every x" STEERABLE=always
+{
+  "variable"   :: ""
+  "group"      :: ""
+  "thorn"      :: ""
+  "everything" :: ""
+} "group"
+
+INT iteration_digits "Minimum number of digits for iteration number" STEERABLE=always
+{
+  0:* :: ""
+} 10
+
+STRING out_filename "File name (without extension) for file_content='everything'" STEERABLE=always
 {
   "" :: ""
 } "output"
 
+INT processor_digits "Minimum number of digits for processor number" STEERABLE=always
+{
+  0:* :: ""
+} 6
+
 STRING out_extension "File name extension" STEERABLE=always
 {
   "" :: "File extension (including a leading dot, if desired)"
 } ".f5"
 
+
+
 STRING out_vars "Variables and groups which should be output in the F5 file format" STEERABLE=always
 {
   "" :: "List of group or variable names"
@@ -67,3 +98,21 @@ REAL dt_fudge "Fudge factor for time comparisons" STEERABLE=always
 {
   0:* :: ""
 } 1.0e-12
+
+
+
+INT compression_level "Compression level to use for writing HDF5 data" STEERABLE=always
+{
+  0:9 :: "Higher numbers compress better, a value of zero disables compression"
+} 0
+
+
+
+KEYWORD output_regions "Regions which should be output" STEERABLE=always
+{
+  "exterior"     :: "whole region (including boundaries)"
+  "communicated" :: "exterior without outer boundary"
+  "owned"        :: "evolved in time"
+  # "active"       :: "owned minus buffers"
+  "interior"     :: "interior (without ghost zones)"
+} "exterior"
diff --git a/CarpetDev/CarpetIOF5/src/IOF5.cc b/CarpetDev/CarpetIOF5/src/IOF5.cc
index c21332adb..94e1089f3 100644
--- a/CarpetDev/CarpetIOF5/src/IOF5.cc
+++ b/CarpetDev/CarpetIOF5/src/IOF5.cc
@@ -1,5 +1,6 @@
 #include <algorithm>
 #include <cassert>
+#include <cctype>
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
@@ -84,6 +85,12 @@ namespace CarpetIOF5 {
   
   
   
+  static string
+  generate_filename (cGH const * cctkGH,
+                     int variable);
+  
+  
+  
   int
   CarpetIOF5_Startup ()
   {
@@ -316,17 +323,17 @@ namespace CarpetIOF5 {
     assert (cctkGH != 0);
     assert (variable >= 0 and variable < CCTK_NumVars());
     
+    char * const fullname = CCTK_FullName (variable);
+    assert (fullname);
+    
     if (verbose)
     {
-      char * const fullname = CCTK_FullName(variable);
       CCTK_VInfo (CCTK_THORNSTRING, "TriggerOutput \"%s\"", fullname);
-      free (fullname);
     }
     
-    char * fullname = CCTK_FullName (variable);
-    assert (fullname);
+    string const alias = generate_filename (cctkGH, variable);
     
-    int const ierr = OutputVarAs (cctkGH, fullname, out_filename);
+    int const ierr = OutputVarAs (cctkGH, fullname, alias.c_str());
     
     free (fullname);
     
@@ -369,18 +376,34 @@ namespace CarpetIOF5 {
     
     extending_t extending (cctkGH);
     
-    bool const use_IO_out_dir = std::strcmp (out_dir, "") == 0;
+    bool const use_IO_out_dir = strcmp (out_dir, "") == 0;
     string const basename
       = (use_IO_out_dir ? IO_out_dir : out_dir) + string ("/") + alias;
-    bool const want_metafile = CCTK_MyProc (cctkGH) == 0;
     
     bool const did_truncate = extending.get_did_truncate (basename);
     bool const do_truncate
       = not did_truncate and IO_TruncateOutputFiles (cctkGH);
     extending.set_did_truncate (basename);
     
-    F5::file_t file (cctkGH, basename, string (out_extension),
-                     want_metafile, do_truncate);
+    F5::file_t file (cctkGH, basename, string (out_extension), do_truncate);
+    
+    if (do_truncate)
+    {
+      // Output parameters once after the output file has been created
+      if (CCTK_EQUALS (out_save_parameters, "all") or
+          CCTK_EQUALS (out_save_parameters, "only set"))
+      {
+        WriteParameters (file);
+      }
+      else if (CCTK_EQUALS (out_save_parameters, "no"))
+      {
+        // do nothing
+      }
+      else
+      {
+        assert (0);
+      }
+    }
     
     writer_t writer (cctkGH, variable);
     writer.write (file);
@@ -393,8 +416,12 @@ namespace CarpetIOF5 {
   void
   WriteParameters (F5::file_t & file)
   {
+    bool const use_metafile = file.get_have_metafile();
+    hid_t const hdf5_file
+      = use_metafile ? file.get_hdf5_metafile() : file.get_hdf5_file();
+    
     hid_t const parameter_group
-      = F5::open_or_create_group (file.get_hdf5_file(), "Cactus parameters");
+      = F5::open_or_create_group (hdf5_file, "Cactus parameters");
     
     int first = 1;
     for (;;)
@@ -456,4 +483,73 @@ namespace CarpetIOF5 {
     assert (not herr);
   }
   
+  
+  
+  string
+  generate_filename (cGH const * const cctkGH,
+                     int const variable)
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    assert (variable >= 0);
+    
+    ostringstream filename_buf;
+    
+    if (CCTK_EQUALS (file_content, "variable"))
+    {
+      char * const varname = CCTK_FullName (variable);
+      assert (varname);
+      for (char * p = varname; * p; ++ p)
+      {
+        *p = tolower (* p);
+      }
+      filename_buf << varname;
+      free (varname);
+    }
+    else if (CCTK_EQUALS (file_content, "group"))
+    {
+      char * const groupname = CCTK_GroupNameFromVarI (variable);
+      assert (groupname);
+      for (char * p = groupname; * p; ++ p)
+      {
+        *p = tolower (* p);
+      }
+      filename_buf << groupname;
+      free (groupname);
+    }
+    else if (CCTK_EQUALS (file_content, "thorn"))
+    {
+      char const * const impname = CCTK_ImpFromVarI (variable);
+      char * const thornname = strdup (impname);
+      assert (thornname);
+      char * const colon = strchr (thornname, ':');
+      assert (colon);
+      * colon = '\0';
+      for (char * p = thornname; * p; ++ p)
+      {
+        *p = tolower (* p);
+      }
+      filename_buf << thornname;
+      free (thornname);
+    }
+    else if (CCTK_EQUALS (file_content, "everything"))
+    {
+      filename_buf << out_filename;
+    }
+    else
+    {
+      assert (0);
+    }
+    
+    if (out_timesteps_per_file > 0)
+    {
+      int const iteration = (cctkGH->cctk_iteration
+                             / out_timesteps_per_file * out_timesteps_per_file);
+      filename_buf << ".it"
+                   << setw (iteration_digits) << setfill ('0') << iteration;
+    }
+    
+    return filename_buf.str();
+  }
+  
 } // namespace CarpetIOF5
diff --git a/CarpetDev/CarpetIOF5/src/data_region.cc b/CarpetDev/CarpetIOF5/src/data_region.cc
index 7a68a66b9..21728c2e6 100644
--- a/CarpetDev/CarpetIOF5/src/data_region.cc
+++ b/CarpetDev/CarpetIOF5/src/data_region.cc
@@ -4,6 +4,7 @@
 #include <string>
 
 #include "cctk.h"
+#include "cctk_Parameters.h"
 
 #include "carpet.hh"
 
@@ -25,6 +26,8 @@ namespace CarpetIOF5 {
       : m_tensor_component (tensor_component),
         m_region (region)
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       assert (not region.empty());
       
       ostringstream namebuf;
@@ -43,14 +46,21 @@ namespace CarpetIOF5 {
       hid_t const hdf5_datatype = hdf5_datatype_from_cactus_datatype (vartype);
       assert (hdf5_datatype >= 0);
       
-      m_properties = H5Pcreate (H5P_DATASET_CREATE);
-      assert (m_properties >= 0);
-      
       vect<hsize_t, dim> const dims
         = (region.shape() / region.stride()).reverse();
       m_dataspace = H5Screate_simple (dim, & dims [0], 0);
       assert (m_dataspace >= 0);
       
+      m_properties = H5Pcreate (H5P_DATASET_CREATE);
+      assert (m_properties >= 0);
+      if (compression_level > 0)
+      {
+        herr_t const herr1 = H5Pset_chunk (m_properties, dim, & dims [0]);
+        assert (not herr1);
+        herr_t const herr2 = H5Pset_deflate (m_properties, compression_level);
+        assert (not herr2);
+      }
+      
       m_dataset
         = H5Dcreate (m_tensor_component.get_hdf5_tensor_component(), name,
                      hdf5_datatype, m_dataspace,
diff --git a/CarpetDev/CarpetIOF5/src/file.cc b/CarpetDev/CarpetIOF5/src/file.cc
index 55c00a124..f200277b5 100644
--- a/CarpetDev/CarpetIOF5/src/file.cc
+++ b/CarpetDev/CarpetIOF5/src/file.cc
@@ -8,6 +8,7 @@
 #include <hdf5.h>
 
 #include "cctk.h"
+#include "cctk_Parameters.h"
 
 #include "file.hh"
 
@@ -23,16 +24,17 @@ namespace CarpetIOF5 {
     file_t (cGH const * const cctkGH,
             string const basename,
             string const extension,
-            bool const want_metafile,
             bool const do_truncate)
       : m_cctkGH (cctkGH),
-        m_have_metafile (want_metafile),
         m_basename (basename),
         m_extension (extension)
     {
       assert (cctkGH);
       
       int const proc = CCTK_MyProc (cctkGH);
+      m_have_metafile = determine_want_metafile (proc);
+      m_output_processor = determine_output_processor (proc);
+      
       m_metafilename = make_metafilename ();
       m_filename = make_filename (proc);
       
@@ -92,17 +94,51 @@ namespace CarpetIOF5 {
     
     
     
+    bool file_t::
+    determine_want_metafile (int const proc)
+      const
+    {
+      DECLARE_CCTK_PARAMETERS;
+      
+      if (CCTK_EQUALS (out_mode, "proc") or
+          CCTK_EQUALS (out_mode, "np"))
+      {
+        return proc == 0;
+      }
+      else if (CCTK_EQUALS (out_mode, "onefile"))
+      {
+        return false;
+      }
+      else
+      {
+        assert (0);
+      }
+    }
+    
+    
+    
     int file_t::
-    base_10_digits (int number)
+    determine_output_processor (int const proc)
+      const
     {
-      number = abs (number);
-      int digits = 1;
-      while (number >= 10)
+      DECLARE_CCTK_PARAMETERS;
+      
+      if (CCTK_EQUALS (out_mode, "proc"))
+      {
+        return proc;
+      }
+      else if (CCTK_EQUALS (out_mode, "np"))
+      {
+        return proc / out_proc_every * out_proc_every;
+      }
+      else if (CCTK_EQUALS (out_mode, "onefile"))
       {
-        number /= 10;
-        ++ digits;
+        return 0;
+      }
+      else
+      {
+        assert (0);
       }
-      return digits;
     }
     
     
@@ -120,12 +156,28 @@ namespace CarpetIOF5 {
     make_filename (int const proc)
       const
     {
-      int const digits = base_10_digits (CCTK_nProcs (m_cctkGH) - 1);
+      DECLARE_CCTK_PARAMETERS;
+      
       ostringstream filenamebuf;
-      filenamebuf << m_basename
-                  << "."
-                  << setw (digits) << setfill ('0') << proc
-                  << m_extension;
+      
+      filenamebuf << m_basename;
+      
+      if (CCTK_EQUALS (out_mode, "proc") or
+          CCTK_EQUALS (out_mode, "np"))
+      {
+        filenamebuf << ".p" << setw (processor_digits) << setfill ('0') << proc;
+      }
+      else if (CCTK_EQUALS (out_mode, "onefile"))
+      {
+        // do nothing
+      }
+      else
+      {
+        assert (0);
+      }
+      
+      filenamebuf << m_extension;
+      
       return filenamebuf.str();
     }
     
@@ -149,6 +201,15 @@ namespace CarpetIOF5 {
     
     
     
+    int file_t::
+    get_output_processor ()
+      const
+    {
+      return m_output_processor;
+    }
+    
+    
+    
     string file_t::
     get_filename (int const proc)
       const
diff --git a/CarpetDev/CarpetIOF5/src/file.hh b/CarpetDev/CarpetIOF5/src/file.hh
index 66d9f0334..19add249e 100644
--- a/CarpetDev/CarpetIOF5/src/file.hh
+++ b/CarpetDev/CarpetIOF5/src/file.hh
@@ -25,7 +25,8 @@ namespace CarpetIOF5 {
       
       cGH const * const m_cctkGH;
       
-      bool const m_have_metafile;
+      bool m_have_metafile;
+      int m_output_processor;
       
       string const m_basename;
       string const m_extension;
@@ -33,7 +34,7 @@ namespace CarpetIOF5 {
       string m_metafilename;
       string m_filename;
       
-      mutable vector <string> m_filenames;
+      vector <string> mutable m_filenames;
       
       hid_t m_hdf5_metafile;
       hid_t m_hdf5_file;
@@ -42,8 +43,13 @@ namespace CarpetIOF5 {
       file_t (file_t const &);
       file_t operator= (file_t const &);
       
-      static int
-      base_10_digits (int number);
+      bool
+      determine_want_metafile (int proc)
+        const;
+      
+      int
+      determine_output_processor (int proc)
+        const;
       
       string
       make_metafilename ()
@@ -58,7 +64,6 @@ namespace CarpetIOF5 {
       file_t (cGH const * cctkGH,
               string filename,
               string extension,
-              bool want_metafile,
               bool do_truncate);
       
       virtual
@@ -72,6 +77,10 @@ namespace CarpetIOF5 {
       get_have_metafile ()
         const;
       
+      int
+      get_output_processor ()
+        const;
+      
       string
       get_filename (int proc)
         const;
diff --git a/CarpetDev/CarpetIOF5/src/meta_data_region.cc b/CarpetDev/CarpetIOF5/src/meta_data_region.cc
index 4f32e33a2..505508eb9 100644
--- a/CarpetDev/CarpetIOF5/src/meta_data_region.cc
+++ b/CarpetDev/CarpetIOF5/src/meta_data_region.cc
@@ -68,11 +68,11 @@ namespace CarpetIOF5 {
         = m_tensor_component.get_hdf5_tensor_component();
       
       herr_t const herr
-        = H5Lcreate_external(filename.c_str(),
-                             objectname.c_str(),
-                             hdf5_tensor_component,
-                             name.c_str(),
-                             H5P_DEFAULT, H5P_DEFAULT);
+        = H5Lcreate_external (filename.c_str(),
+                              objectname.c_str(),
+                              hdf5_tensor_component,
+                              name.c_str(),
+                              H5P_DEFAULT, H5P_DEFAULT);
       assert (not herr);
     }
     
diff --git a/CarpetDev/CarpetIOF5/src/writer.cc b/CarpetDev/CarpetIOF5/src/writer.cc
index e4ef100f6..df1949369 100644
--- a/CarpetDev/CarpetIOF5/src/writer.cc
+++ b/CarpetDev/CarpetIOF5/src/writer.cc
@@ -164,8 +164,9 @@ namespace CarpetIOF5 {
     int const reflevel = 0;
     int const myproc = CCTK_MyProc (m_cctkGH);
     dh * const dd = Carpet::arrdata.at(group).at(map).dd;
-    bbox<int, dim> const & region
-      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc).exterior;
+    dh::dboxes const & boxes
+      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc);
+    bbox<int, dim> const & region = determine_region (boxes);
     
     if (have_metafile)
     {
@@ -320,4 +321,37 @@ namespace CarpetIOF5 {
     }
   }
   
+  
+  
+  bbox<int,dim> const & writer_t::
+  determine_region (dh::dboxes const & boxes)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    bbox<int,dim> dh::dboxes::* boxptr;
+    if (CCTK_EQUALS (output_regions, "exterior"))
+    {
+      boxptr = & dh::dboxes::exterior;
+    }
+    else if (CCTK_EQUALS (output_regions, "communicated"))
+    {
+      boxptr = & dh::dboxes::communicated;
+    }
+    else if (CCTK_EQUALS (output_regions, "owned"))
+    {
+      boxptr = & dh::dboxes::owned;
+    }
+    else if (CCTK_EQUALS (output_regions, "interior"))
+    {
+      boxptr = & dh::dboxes::interior;
+    }
+    else
+    {
+      assert (0);
+    }
+    
+    return boxes.*boxptr;
+  }
+  
 } // namespace CarpetIOF5
diff --git a/CarpetDev/CarpetIOF5/src/writer.hh b/CarpetDev/CarpetIOF5/src/writer.hh
index 056d39a3f..94b31d3cd 100644
--- a/CarpetDev/CarpetIOF5/src/writer.hh
+++ b/CarpetDev/CarpetIOF5/src/writer.hh
@@ -3,6 +3,10 @@
 
 #include "cctk.h"
 
+#include "bbox.hh"
+#include "defs.hh"
+#include "dh.hh"
+
 #include "file.hh"
 #include "simulation.hh"
 #include "tensor_component.hh"
@@ -59,6 +63,10 @@ namespace CarpetIOF5 {
                          bool have_metafile)
       const;
     
+    bbox<int,dim> const &
+    determine_region (dh::dboxes const & boxes)
+      const;
+    
   };
   
 } // namespace CarpetIOF5