Import Carpet

Ignore-this: 309b4dd613f4af2b84aa5d6743fdb6b3

36 files changed, 2966 insertions, 629 deletions

diff --git a/CarpetDev/CarpetIOF5/README b/CarpetDev/CarpetIOF5/README
index a03d4b2f5..a625117a0 100644
--- a/CarpetDev/CarpetIOF5/README
+++ b/CarpetDev/CarpetIOF5/README
@@ -11,16 +11,20 @@ Input and output data using the HDF5 based F5 file format.
 
 An F5 file contains the following hierarchy:
 
-        file
-        timestep
-        simulation
-        topology
-        coordinate_system
-        physical_quantity
-        tensor_component
-        data_region
+        Bundle
+        Slice
+        Grid
+        Topology
+        Representation
+        Field
 
-2006-06-05: Werner says that tensor_component and data_region should
-be interchanged.
+We have a class hierarchy:
 
-2007-04-25: Werner confirms this.
+        file
+        timestep (slice)
+        simulation (grid) [grid hierarchy of one patch]
+        topology (topology) [one refinement level]
+        coordinate_system (representation)
+        physical_quantity (field)
+        data_region
+        tensor_component
diff --git a/CarpetDev/CarpetIOF5/configuration.ccl b/CarpetDev/CarpetIOF5/configuration.ccl
index 1ab5bc27d..d5c33c995 100644
--- a/CarpetDev/CarpetIOF5/configuration.ccl
+++ b/CarpetDev/CarpetIOF5/configuration.ccl
@@ -1,3 +1,3 @@
 # Configuration definitions for thorn CarpetIOF5
 
-REQUIRES Carpet CarpetLib HDF5 IOUtil
+REQUIRES Carpet CarpetLib F5 HDF5 IOUtil
diff --git a/CarpetDev/CarpetIOF5/interface.ccl b/CarpetDev/CarpetIOF5/interface.ccl
index db9d9c014..aa7a39fee 100644
--- a/CarpetDev/CarpetIOF5/interface.ccl
+++ b/CarpetDev/CarpetIOF5/interface.ccl
@@ -3,7 +3,7 @@
 IMPLEMENTS: IOF5
 
 USES INCLUDE: bbox.hh defs.hh dh.hh vect.hh
-USES INCLUDE: carpet.hh
+USES INCLUDE: carpet.hh modes.hh
 
 
 
@@ -26,7 +26,25 @@ REQUIRES FUNCTION IO_TruncateOutputFiles
 
 
 # Return a pointer to an unmodifiable C string
+# which contains a unique ID for this configuration
+CCTK_POINTER_TO_CONST \
+FUNCTION UniqueConfigID (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION UniqueConfigID
+
+# Return a pointer to an unmodifiable C string
+# which contains a unique ID for this build
+CCTK_POINTER_TO_CONST \
+FUNCTION UniqueBuildID (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION UniqueBuildID
+
+# Return a pointer to an unmodifiable C string
 # which contains a unique ID for this simulation
 CCTK_POINTER_TO_CONST \
-FUNCTION UniqueSimulationID (CCTK_POINTER_TO_CONST IN  cctkGH)
+FUNCTION UniqueSimulationID (CCTK_POINTER_TO_CONST IN cctkGH)
 USES FUNCTION UniqueSimulationID
+
+# Return a pointer to an unmodifiable C string
+# which contains a unique ID for this run
+CCTK_POINTER_TO_CONST \
+FUNCTION UniqueRunID (CCTK_POINTER_TO_CONST IN cctkGH)
+USES FUNCTION UniqueRunID
diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par b/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par
new file mode 100644
index 000000000..1e8c2783d
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par
@@ -0,0 +1,244 @@
+Cactus::cctk_run_title = "WaveToy cell-centred refined"
+
+Cactus::cctk_full_warnings         = yes
+Cactus::highlight_warning_messages = no
+#Cactus::cctk_timer_output          = "full"
+
+Cactus::terminate       = "time"
+Cactus::cctk_final_time = 1.0
+
+
+
+ActiveThorns = "IOUtil"
+
+IO::out_dir = $parfile
+
+
+
+ActiveThorns = "InitBase"
+
+InitBase::initial_data_setup_method = "init_single_level"
+
+
+
+ActiveThorns = "AEILocalInterp"
+
+ActiveThorns = "Fortran"
+
+ActiveThorns = "HDF5"
+
+ActiveThorns = "LocalInterp"
+
+ActiveThorns = "LoopControl"
+
+ActiveThorns = "NaNCatcher"
+
+ActiveThorns = "Slab"
+
+
+
+ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab"
+
+Carpet::verbose           = no
+Carpet::schedule_barriers = no
+Carpet::veryverbose       = no
+CarpetLib::output_bboxes  = no
+
+Carpet::domain_from_coordbase = yes
+Carpet::max_refinement_levels = 3
+
+driver::ghost_size = 2
+
+Carpet::refinement_centering     = "cell"
+Carpet::prolongation_order_space = 2
+Carpet::prolongation_order_time  = 2
+
+Carpet::convergence_level = 0
+
+Carpet::init_each_timelevel          = yes
+Carpet::regrid_during_initialisation = no
+Carpet::enable_all_storage           = no
+
+Carpet::poison_new_timelevels = yes
+CarpetLib::poison_new_memory  = yes
+
+Carpet::grid_structure_filename  = "carpet-grid-structure"
+Carpet::output_timers_every      = 100
+CarpetLib::print_timestats_every = 100
+CarpetLib::print_memstats_every  = 100
+
+
+
+ActiveThorns = "NaNChecker"
+
+NaNChecker::check_every     = 100
+NaNChecker::action_if_found = "terminate"
+NaNChecker::check_vars      = "
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "Boundary CartGrid3D CoordBase SymBase"
+
+CoordBase::domainsize = "minmax"
+
+CoordBase::xmin = -0.50
+CoordBase::ymin = -0.50
+CoordBase::zmin = -0.50
+CoordBase::xmax = +0.50
+CoordBase::ymax = +0.50
+CoordBase::zmax = +0.50
+CoordBase::dx   =  0.05
+CoordBase::dy   =  0.05
+CoordBase::dz   =  0.05
+
+CoordBase::boundary_staggered_x_lower = yes
+CoordBase::boundary_staggered_y_lower = yes
+CoordBase::boundary_staggered_z_lower = yes
+CoordBase::boundary_staggered_x_upper = yes
+CoordBase::boundary_staggered_y_upper = yes
+CoordBase::boundary_staggered_z_upper = yes
+
+CartGrid3D::type = "coordbase"
+
+
+
+ActiveThorns = "CarpetRegrid2"
+
+CarpetRegrid2::regrid_every = 0
+
+CarpetRegrid2::snap_to_coarse = yes
+
+CarpetRegrid2::num_centres = 1
+
+CarpetRegrid2::num_levels_1 = 3
+CarpetRegrid2::radius_1[1]  = 0.25
+CarpetRegrid2::radius_1[2]  = 0.125
+
+
+
+ActiveThorns = "WavetoyC"
+
+WaveToy::bound = "radiation"
+
+
+
+ActiveThorns = "IDScalarWaveC"
+
+IDScalarWave::initial_data = "gaussian"
+IDScalarWave::radius    = 0.0
+IDScalarWave::sigma     = 0.1
+IDScalarWave::amplitude = 1.0
+
+
+
+ActiveThorns = "Time"
+
+Time::dtfac = 0.5
+
+
+
+ActiveThorns = "CarpetIOBasic"
+
+IOBasic::outInfo_every      = 1
+IOBasic::outInfo_reductions = "norm2 minimum maximum"
+IOBasic::outInfo_vars       = "
+        WaveToy::phi
+"
+
+
+
+ActiveThorns = "CarpetIOScalar"
+
+IOScalar::one_file_per_group = yes
+
+IOScalar::outScalar_every = 1
+IOScalar::outScalar_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "CarpetIOASCII"
+
+IOASCII::one_file_per_group = yes
+
+#IOASCII::output_symmetry_points = no
+#IOASCII::out3D_ghosts           = no
+
+IOASCII::out0D_every = 1
+IOASCII::out0D_vars  = "
+        Carpet::timing
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out1D_every = 1
+IOASCII::out1D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out2D_every = 10
+IOASCII::out2D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out3D_every = 100
+IOASCII::out3D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+Activethorns = "CarpetIOHDF5"
+
+IOHDF5::out_every = 100
+IOHDF5::compression_level = 1
+IOHDF5::out_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOHDF5::checkpoint          = yes
+IO::checkpoint_dir          = $parfile
+IO::checkpoint_ID           = yes
+IO::checkpoint_every        = 100
+IO::checkpoint_on_terminate = yes
+
+Carpet::regrid_during_recovery             = no
+IOHDF5::use_grid_structure_from_checkpoint = yes
+
+IO::recover     = "autoprobe"
+IO::recover_dir = $parfile
+
+
+
+ActiveThorns = "Formaline"
+
+Formaline::send_as_rdf     = yes
+Formaline::rdf_hostname[0] = "buran.aei.mpg.de"
+Formaline::rdf_port    [0] = 24997
+Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu"
+Formaline::rdf_port    [1] = 8080
+
+
+
+ActiveThorns = "Twitter"
+
+
+
+ActiveThorns = "TimerReport"
+
+TimerReport::out_every    = 100
+TimerReport::out_filename = "TimerReport"
diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par b/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par
new file mode 100644
index 000000000..d5265e3c8
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par
@@ -0,0 +1,230 @@
+Cactus::cctk_run_title = "WaveToy cell-centred unigrid"
+
+Cactus::cctk_full_warnings         = yes
+Cactus::highlight_warning_messages = no
+#Cactus::cctk_timer_output          = "full"
+
+Cactus::terminate       = "time"
+Cactus::cctk_final_time = 1.0
+
+
+
+ActiveThorns = "IOUtil"
+
+IO::out_dir = $parfile
+
+
+
+ActiveThorns = "InitBase"
+
+InitBase::initial_data_setup_method = "init_single_level"
+
+
+
+ActiveThorns = "AEILocalInterp"
+
+ActiveThorns = "Fortran"
+
+ActiveThorns = "HDF5"
+
+ActiveThorns = "LocalInterp"
+
+ActiveThorns = "LoopControl"
+
+ActiveThorns = "NaNCatcher"
+
+ActiveThorns = "Slab"
+
+
+
+ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab"
+
+Carpet::verbose           = no
+Carpet::schedule_barriers = no
+Carpet::veryverbose       = no
+CarpetLib::output_bboxes  = no
+
+Carpet::domain_from_coordbase = yes
+Carpet::max_refinement_levels = 3
+
+driver::ghost_size = 2
+
+Carpet::refinement_centering     = "cell"
+Carpet::prolongation_order_space = 3
+Carpet::prolongation_order_time  = 2
+
+Carpet::convergence_level = 0
+
+Carpet::init_each_timelevel          = yes
+Carpet::regrid_during_initialisation = no
+Carpet::enable_all_storage           = no
+
+Carpet::poison_new_timelevels = yes
+CarpetLib::poison_new_memory  = yes
+
+Carpet::grid_structure_filename  = "carpet-grid-structure"
+Carpet::output_timers_every      = 100
+CarpetLib::print_timestats_every = 100
+CarpetLib::print_memstats_every  = 100
+
+
+
+ActiveThorns = "NaNChecker"
+
+NaNChecker::check_every     = 100
+NaNChecker::action_if_found = "terminate"
+NaNChecker::check_vars      = "
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "Boundary CartGrid3D CoordBase SymBase"
+
+CoordBase::domainsize = "minmax"
+
+CoordBase::xmin = -0.50
+CoordBase::ymin = -0.50
+CoordBase::zmin = -0.50
+CoordBase::xmax = +0.50
+CoordBase::ymax = +0.50
+CoordBase::zmax = +0.50
+CoordBase::dx   =  0.0125
+CoordBase::dy   =  0.0125
+CoordBase::dz   =  0.0125
+
+CoordBase::boundary_staggered_x_lower = yes
+CoordBase::boundary_staggered_y_lower = yes
+CoordBase::boundary_staggered_z_lower = yes
+CoordBase::boundary_staggered_x_upper = yes
+CoordBase::boundary_staggered_y_upper = yes
+CoordBase::boundary_staggered_z_upper = yes
+
+CartGrid3D::type = "coordbase"
+
+
+
+ActiveThorns = "WavetoyC"
+
+WaveToy::bound = "radiation"
+
+
+
+ActiveThorns = "IDScalarWaveC"
+
+IDScalarWave::initial_data = "gaussian"
+IDScalarWave::radius    = 0.0
+IDScalarWave::sigma     = 0.1
+IDScalarWave::amplitude = 1.0
+
+
+
+ActiveThorns = "Time"
+
+Time::dtfac = 0.5
+
+
+
+ActiveThorns = "CarpetIOBasic"
+
+IOBasic::outInfo_every      = 1
+IOBasic::outInfo_reductions = "norm2 minimum maximum"
+IOBasic::outInfo_vars       = "
+        WaveToy::phi
+"
+
+
+
+ActiveThorns = "CarpetIOScalar"
+
+IOScalar::one_file_per_group = yes
+
+IOScalar::outScalar_every = 1
+IOScalar::outScalar_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "CarpetIOASCII"
+
+IOASCII::one_file_per_group = yes
+
+IOASCII::output_symmetry_points = no
+IOASCII::out3D_ghosts           = no
+
+IOASCII::out0D_every = 1
+IOASCII::out0D_vars  = "
+        Carpet::timing
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out1D_every = 1
+IOASCII::out1D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out2D_every = 10
+IOASCII::out2D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out3D_every = 100
+IOASCII::out3D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+Activethorns = "CarpetIOHDF5"
+
+IOHDF5::out_every = 100
+IOHDF5::compression_level = 1
+IOHDF5::out_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOHDF5::checkpoint          = yes
+IO::checkpoint_dir          = $parfile
+IO::checkpoint_ID           = yes
+IO::checkpoint_every        = 100
+IO::checkpoint_on_terminate = yes
+
+Carpet::regrid_during_recovery             = no
+IOHDF5::use_grid_structure_from_checkpoint = yes
+
+IO::recover     = "autoprobe"
+IO::recover_dir = $parfile
+
+
+
+ActiveThorns = "Formaline"
+
+Formaline::send_as_rdf     = yes
+Formaline::rdf_hostname[0] = "buran.aei.mpg.de"
+Formaline::rdf_port    [0] = 24997
+Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu"
+Formaline::rdf_port    [1] = 8080
+
+
+
+ActiveThorns = "Twitter"
+
+
+
+ActiveThorns = "TimerReport"
+
+TimerReport::out_every    = 100
+TimerReport::out_filename = "TimerReport"
diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par b/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par
new file mode 100644
index 000000000..46ccdd6b2
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par
@@ -0,0 +1,244 @@
+Cactus::cctk_run_title = "WaveToy vertex-centred refined"
+
+Cactus::cctk_full_warnings         = yes
+Cactus::highlight_warning_messages = no
+#Cactus::cctk_timer_output          = "full"
+
+Cactus::terminate       = "time"
+Cactus::cctk_final_time = 1.0
+
+
+
+ActiveThorns = "IOUtil"
+
+IO::out_dir = $parfile
+
+
+
+ActiveThorns = "InitBase"
+
+InitBase::initial_data_setup_method = "init_single_level"
+
+
+
+ActiveThorns = "AEILocalInterp"
+
+ActiveThorns = "Fortran"
+
+ActiveThorns = "HDF5"
+
+ActiveThorns = "LocalInterp"
+
+ActiveThorns = "LoopControl"
+
+ActiveThorns = "NaNCatcher"
+
+ActiveThorns = "Slab"
+
+
+
+ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab"
+
+Carpet::verbose           = no
+Carpet::schedule_barriers = no
+Carpet::veryverbose       = no
+CarpetLib::output_bboxes  = no
+
+Carpet::domain_from_coordbase = yes
+Carpet::max_refinement_levels = 3
+
+driver::ghost_size = 2
+
+Carpet::refinement_centering     = "vertex"
+Carpet::prolongation_order_space = 3
+Carpet::prolongation_order_time  = 2
+
+Carpet::convergence_level = 0
+
+Carpet::init_each_timelevel          = yes
+Carpet::regrid_during_initialisation = no
+Carpet::enable_all_storage           = no
+
+Carpet::poison_new_timelevels = yes
+CarpetLib::poison_new_memory  = yes
+
+Carpet::grid_structure_filename  = "carpet-grid-structure"
+Carpet::output_timers_every      = 100
+CarpetLib::print_timestats_every = 100
+CarpetLib::print_memstats_every  = 100
+
+
+
+ActiveThorns = "NaNChecker"
+
+NaNChecker::check_every     = 100
+NaNChecker::action_if_found = "terminate"
+NaNChecker::check_vars      = "
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "Boundary CartGrid3D CoordBase SymBase"
+
+CoordBase::domainsize = "minmax"
+
+CoordBase::xmin = -0.50
+CoordBase::ymin = -0.50
+CoordBase::zmin = -0.50
+CoordBase::xmax = +0.50
+CoordBase::ymax = +0.50
+CoordBase::zmax = +0.50
+CoordBase::dx   =  0.05
+CoordBase::dy   =  0.05
+CoordBase::dz   =  0.05
+
+CoordBase::boundary_staggered_x_lower = no
+CoordBase::boundary_staggered_y_lower = no
+CoordBase::boundary_staggered_z_lower = no
+CoordBase::boundary_staggered_x_upper = no
+CoordBase::boundary_staggered_y_upper = no
+CoordBase::boundary_staggered_z_upper = no
+
+CartGrid3D::type = "coordbase"
+
+
+
+ActiveThorns = "CarpetRegrid2"
+
+CarpetRegrid2::regrid_every = 0
+
+CarpetRegrid2::snap_to_coarse = yes
+
+CarpetRegrid2::num_centres = 1
+
+CarpetRegrid2::num_levels_1 = 3
+CarpetRegrid2::radius_1[1]  = 0.25
+CarpetRegrid2::radius_1[2]  = 0.125
+
+
+
+ActiveThorns = "WavetoyC"
+
+WaveToy::bound = "radiation"
+
+
+
+ActiveThorns = "IDScalarWaveC"
+
+IDScalarWave::initial_data = "gaussian"
+IDScalarWave::radius    = 0.0
+IDScalarWave::sigma     = 0.1
+IDScalarWave::amplitude = 1.0
+
+
+
+ActiveThorns = "Time"
+
+Time::dtfac = 0.5
+
+
+
+ActiveThorns = "CarpetIOBasic"
+
+IOBasic::outInfo_every      = 1
+IOBasic::outInfo_reductions = "norm2 minimum maximum"
+IOBasic::outInfo_vars       = "
+        WaveToy::phi
+"
+
+
+
+ActiveThorns = "CarpetIOScalar"
+
+IOScalar::one_file_per_group = yes
+
+IOScalar::outScalar_every = 1
+IOScalar::outScalar_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "CarpetIOASCII"
+
+IOASCII::one_file_per_group = yes
+
+#IOASCII::output_symmetry_points = no
+#IOASCII::out3D_ghosts           = no
+
+IOASCII::out0D_every = 1
+IOASCII::out0D_vars  = "
+        Carpet::timing
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out1D_every = 1
+IOASCII::out1D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out2D_every = 10
+IOASCII::out2D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOASCII::out3D_every = 100
+IOASCII::out3D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+
+Activethorns = "CarpetIOHDF5"
+
+IOHDF5::out_every = 100
+IOHDF5::compression_level = 1
+IOHDF5::out_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+IOHDF5::checkpoint          = yes
+IO::checkpoint_dir          = $parfile
+IO::checkpoint_ID           = yes
+IO::checkpoint_every        = 100
+IO::checkpoint_on_terminate = yes
+
+Carpet::regrid_during_recovery             = no
+IOHDF5::use_grid_structure_from_checkpoint = yes
+
+IO::recover     = "autoprobe"
+IO::recover_dir = $parfile
+
+
+
+ActiveThorns = "Formaline"
+
+Formaline::send_as_rdf     = yes
+Formaline::rdf_hostname[0] = "buran.aei.mpg.de"
+Formaline::rdf_port    [0] = 24997
+Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu"
+Formaline::rdf_port    [1] = 8080
+
+
+
+ActiveThorns = "Twitter"
+
+
+
+ActiveThorns = "TimerReport"
+
+TimerReport::out_every    = 100
+TimerReport::out_filename = "TimerReport"
diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par b/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par
new file mode 100644
index 000000000..e2339f342
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par
@@ -0,0 +1,245 @@
+Cactus::cctk_run_title = "WaveToy vertex-centred unigrid"
+
+Cactus::cctk_full_warnings         = yes
+Cactus::highlight_warning_messages = no
+
+Cactus::terminate       = "time"
+Cactus::cctk_final_time = 1.0
+
+
+
+ActiveThorns = "IOUtil"
+
+IO::out_dir = $parfile
+
+
+
+ActiveThorns = "InitBase"
+
+InitBase::initial_data_setup_method = "init_single_level"
+
+
+
+ActiveThorns = "AEILocalInterp"
+
+ActiveThorns = "Fortran"
+
+ActiveThorns = "HDF5"
+
+ActiveThorns = "LocalInterp"
+
+ActiveThorns = "LoopControl"
+
+ActiveThorns = "NaNCatcher"
+
+ActiveThorns = "Slab"
+
+
+
+ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab"
+
+Carpet::verbose           = no
+Carpet::schedule_barriers = no
+Carpet::veryverbose       = no
+CarpetLib::output_bboxes  = no
+
+Carpet::domain_from_coordbase = yes
+Carpet::max_refinement_levels = 3
+
+driver::ghost_size = 2
+
+Carpet::refinement_centering     = "vertex"
+Carpet::prolongation_order_space = 3
+Carpet::prolongation_order_time  = 2
+
+Carpet::convergence_level = 0
+
+Carpet::init_each_timelevel          = yes
+Carpet::regrid_during_initialisation = no
+Carpet::enable_all_storage           = no
+
+Carpet::poison_new_timelevels = yes
+CarpetLib::poison_new_memory  = yes
+
+Carpet::grid_structure_filename  = "carpet-grid-structure"
+Carpet::output_timers_every      = 100
+CarpetLib::print_timestats_every = 100
+CarpetLib::print_memstats_every  = 100
+
+
+
+ActiveThorns = "NaNChecker"
+
+NaNChecker::check_every     = 100
+NaNChecker::action_if_found = "terminate"
+NaNChecker::check_vars      = "
+        WaveToy::scalarevolve
+"
+
+
+
+ActiveThorns = "Boundary CartGrid3D CoordBase SymBase"
+
+CoordBase::domainsize = "minmax"
+
+CoordBase::xmin = -0.50
+CoordBase::ymin = -0.50
+CoordBase::zmin = -0.50
+CoordBase::xmax = +0.50
+CoordBase::ymax = +0.50
+CoordBase::zmax = +0.50
+CoordBase::dx   =  0.0125
+CoordBase::dy   =  0.0125
+CoordBase::dz   =  0.0125
+
+CoordBase::boundary_staggered_x_lower = no
+CoordBase::boundary_staggered_y_lower = no
+CoordBase::boundary_staggered_z_lower = no
+CoordBase::boundary_staggered_x_upper = no
+CoordBase::boundary_staggered_y_upper = no
+CoordBase::boundary_staggered_z_upper = no
+
+CartGrid3D::type = "coordbase"
+
+
+
+ActiveThorns = "WavetoyC"
+
+WaveToy::bound = "radiation"
+
+
+
+ActiveThorns = "IDScalarWaveC"
+
+IDScalarWave::initial_data = "gaussian"
+IDScalarWave::radius    = 0.0
+IDScalarWave::sigma     = 0.1
+IDScalarWave::amplitude = 1.0
+
+
+
+ActiveThorns = "Time"
+
+Time::dtfac = 0.5
+
+
+
+ActiveThorns = "CarpetIOBasic"
+
+IOBasic::outInfo_every      = 1
+IOBasic::outInfo_reductions = "norm2 minimum maximum"
+IOBasic::outInfo_vars       = "
+        WaveToy::phi
+"
+
+
+
+#ActiveThorns = "CarpetIOScalar"
+#
+#IOScalar::one_file_per_group = yes
+#
+#IOScalar::outScalar_every = 1
+#IOScalar::outScalar_vars  = "
+#        grid::coordinates
+#        CarpetReduce::weight
+#        WaveToy::scalarevolve
+#"
+
+
+
+ActiveThorns = "CarpetIOASCII"
+
+IOASCII::one_file_per_group = yes
+
+IOASCII::output_symmetry_points = yes # no
+IOASCII::out3D_ghosts           = yes # no
+
+#IOASCII::out0D_every = 1
+#IOASCII::out0D_vars  = "
+#        Carpet::timing
+#        grid::coordinates
+#        CarpetReduce::weight
+#        WaveToy::scalarevolve
+#"
+
+IOASCII::out1D_every = 1
+IOASCII::out1D_vars  = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+#IOASCII::out2D_every = 10
+#IOASCII::out2D_vars  = "
+#        grid::coordinates
+#        CarpetReduce::weight
+#        WaveToy::scalarevolve
+#"
+#
+#IOASCII::out3D_every = 100
+#IOASCII::out3D_vars  = "
+#        grid::coordinates
+#        CarpetReduce::weight
+#        WaveToy::scalarevolve
+#"
+
+
+
+Activethorns = "CarpetIOHDF5"
+
+IOHDF5::out_every         = 64
+IOHDF5::compression_level = 1
+IOHDF5::out_vars          = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+#IOHDF5::checkpoint          = yes
+#IO::checkpoint_dir          = $parfile
+#IO::checkpoint_ID           = yes
+#IO::checkpoint_every        = 100
+#IO::checkpoint_on_terminate = yes
+#
+#Carpet::regrid_during_recovery             = no
+#IOHDF5::use_grid_structure_from_checkpoint = yes
+#
+#IO::recover     = "autoprobe"
+#IO::recover_dir = $parfile
+
+
+
+Activethorns = "CarpetIOF5"
+
+IOF5::out_every         = 64
+IOF5::chunk_size_x      = 8
+IOF5::chunk_size_y      = 8
+IOF5::chunk_size_z      = 8
+IOF5::compression_level = 1
+IOF5::write_checksum    = yes
+IOF5::out_vars          = "
+        grid::coordinates
+        CarpetReduce::weight
+        WaveToy::scalarevolve
+"
+
+
+ActiveThorns = "Formaline"
+
+Formaline::send_as_rdf     = yes
+Formaline::rdf_hostname[0] = "buran.aei.mpg.de"
+Formaline::rdf_port    [0] = 24997
+Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu"
+Formaline::rdf_port    [1] = 8080
+
+
+
+ActiveThorns = "Twitter"
+
+
+
+ActiveThorns = "TimerReport"
+
+TimerReport::out_every                  = 100
+TimerReport::output_all_timers_together = yes
+TimerReport::out_filename               = "TimerReport"
diff --git a/CarpetDev/CarpetIOF5/param.ccl b/CarpetDev/CarpetIOF5/param.ccl
index 45b0eeca1..f9d1249bd 100644
--- a/CarpetDev/CarpetIOF5/param.ccl
+++ b/CarpetDev/CarpetIOF5/param.ccl
@@ -62,6 +62,15 @@ STRING out_extension "File name extension" STEERABLE=always
   "" :: "File extension (including a leading dot, if desired)"
 } ".f5"
 
+BOOLEAN create_subdirs "Create subdirectories for the output files to reduce the number of files per directory" STEERABLE=always
+{
+} "no"
+
+BOOLEAN one_dir_per_file "Create one subdirectory per output file to reduce locking overhead" STEERABLE=always
+{
+} "no"
+
+
 
 
 STRING out_vars "Variables and groups which should be output in the F5 file format" STEERABLE=always
@@ -101,17 +110,44 @@ REAL dt_fudge "Fudge factor for time comparisons" STEERABLE=always
 
 
 
+INT chunk_size_x "Chunk size for chunked datasets" STEERABLE=always
+{
+  -1  :: "no chunking (use actual dataset size)"
+  1:* :: "desired chunk size"
+} -1
+
+INT chunk_size_y "Chunk size for chunked datasets" STEERABLE=always
+{
+  -1  :: "no chunking (use actual dataset size)"
+  1:* :: "desired chunk size"
+} -1
+
+INT chunk_size_z "Chunk size for chunked datasets" STEERABLE=always
+{
+  -1  :: "no chunking (use actual dataset size)"
+  1:* :: "desired chunk size"
+} -1
+
 INT compression_level "Compression level to use for writing HDF5 data" STEERABLE=always
 {
-  0:9 :: "Higher numbers compress better, a value of zero disables compression"
+  0   :: "no compression"
+  1:9 :: "higher numbers compress better"
 } 0
 
+BOOLEAN write_checksum "Add a checksum when writing HDF5 files" STEERABLE=always
+{
+} "no"
+
+BOOLEAN check_checksum "Check checksum (if present) when reading HDF5 files" STEERABLE=always
+{
+} "no"
+
 
 
 KEYWORD output_regions "Regions which should be output" STEERABLE=always
 {
   "exterior"     :: "whole region (including boundaries)"
-  "communicated" :: "exterior without outer boundary"
+  #"communicated" :: "exterior without outer boundary"
   "owned"        :: "evolved in time"
   # "active"       :: "owned minus buffers"
   "interior"     :: "interior (without ghost zones)"
diff --git a/CarpetDev/CarpetIOF5/src/IOF5.cc b/CarpetDev/CarpetIOF5/src/IOF5.cc
index 94e1089f3..1173f5228 100644
--- a/CarpetDev/CarpetIOF5/src/IOF5.cc
+++ b/CarpetDev/CarpetIOF5/src/IOF5.cc
@@ -9,12 +9,22 @@
 #include <string>
 #include <vector>
 
+// force HDF5 1.8.x installations to use the new API
+#define H5Dcreate_vers 2
+
+#include <hdf5.h>
+
 #include "cctk.h"
 #include "cctk_Arguments.h"
 #include "cctk_Parameters.h"
 
+#include "CactusBase/IOUtil/src/ioGH.h"
+#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h"
+
 #include "carpet.hh"
 
+#include "defs.hh"
+
 #include "extending.hh"
 #include "file.hh"
 #include "utils.hh"
@@ -377,36 +387,93 @@ namespace CarpetIOF5 {
     extending_t extending (cctkGH);
     
     bool const use_IO_out_dir = strcmp (out_dir, "") == 0;
-    string const basename
-      = (use_IO_out_dir ? IO_out_dir : out_dir) + string ("/") + alias;
+    string const path = use_IO_out_dir ? IO_out_dir : out_dir;
+    string const basename = alias;
     
     bool const did_truncate = extending.get_did_truncate (basename);
     bool const do_truncate
       = not did_truncate and IO_TruncateOutputFiles (cctkGH);
     extending.set_did_truncate (basename);
     
-    F5::file_t file (cctkGH, basename, string (out_extension), do_truncate);
+    int const proc = CCTK_MyProc (cctkGH);
+    bool have_metafile;         // whether there is a metadata file
+    int metadata_processor;     // the processor which outputs the metadata file
+    int output_processor;       // the processor which outputs our data
+    if (CCTK_EQUALS (out_mode, "proc"))
+    {
+      have_metafile = true;
+      metadata_processor = 0;
+      output_processor = proc;
+    }
+    else if (CCTK_EQUALS (out_mode, "np"))
+    {
+      have_metafile = true;
+      metadata_processor = 0;
+      output_processor = proc / out_proc_every * out_proc_every;
+    }
+    else if (CCTK_EQUALS (out_mode, "onefile"))
+    {
+      have_metafile = false;
+      metadata_processor = 0;
+      output_processor = 0;
+    }
+    else
+    {
+      assert (0);
+    }
+    
+    F5::file_t * metafile = NULL;
+    if (have_metafile and proc == metadata_processor)
+    {
+      metafile
+        = new F5::file_t (cctkGH, path, basename, string (out_extension),
+                          do_truncate, true, false);
+    }
+    
+    F5::file_t * file = NULL;
+    if (proc == output_processor)
+    {
+      file = new F5::file_t (cctkGH, path, basename, string (out_extension),
+                             do_truncate, not have_metafile, true);
+    }
     
     if (do_truncate)
     {
       // Output parameters once after the output file has been created
-      if (CCTK_EQUALS (out_save_parameters, "all") or
-          CCTK_EQUALS (out_save_parameters, "only set"))
-      {
-        WriteParameters (file);
-      }
-      else if (CCTK_EQUALS (out_save_parameters, "no"))
+      if (proc == metadata_processor)
       {
-        // do nothing
-      }
-      else
-      {
-        assert (0);
+        if (CCTK_EQUALS (out_save_parameters, "all") or
+            CCTK_EQUALS (out_save_parameters, "only set"))
+        {
+          WriteParameters (have_metafile ? * metafile : * file);
+        }
+        else if (CCTK_EQUALS (out_save_parameters, "no"))
+        {
+          // do nothing
+        }
+        else
+        {
+          assert (0);
+        }
       }
     }
     
-    writer_t writer (cctkGH, variable);
-    writer.write (file);
+    if (metafile)
+    {
+      writer_t writer (cctkGH, variable);
+      writer.write (* metafile);
+      delete metafile;
+      metafile = NULL;
+    }
+    {
+      writer_t writer (cctkGH, variable);
+#warning "TODO: handle the case where not all processors are writing to their own file"
+      assert (proc == output_processor);
+      assert (file);
+      writer.write (* file);
+      delete file;
+      file = NULL;
+    }
     
     return Error_none;
   }
@@ -416,12 +483,84 @@ namespace CarpetIOF5 {
   void
   WriteParameters (F5::file_t & file)
   {
-    bool const use_metafile = file.get_have_metafile();
-    hid_t const hdf5_file
-      = use_metafile ? file.get_hdf5_metafile() : file.get_hdf5_file();
+    DECLARE_CCTK_PARAMETERS;
+    
+    cGH const * const cctkGH = file.get_cctkGH ();
+    
+    hid_t const hdf5_file = file.get_hdf5_file();
+    
+    hid_t const attribute_group
+      = F5::open_or_create_group (hdf5_file,
+                                  "Parameters and Global Attributes");
+    assert (attribute_group >= 0);
+    
+    // unique configuration identifier
+    if (CCTK_IsFunctionAliased ("UniqueConfigID")) {
+      F5::write_or_check_attribute
+        (attribute_group, "config id",
+         static_cast<char const *> (UniqueConfigID (cctkGH)));
+    }
+    
+    // unique build identifier
+    if (CCTK_IsFunctionAliased ("UniqueBuildID")) {
+      F5::write_or_check_attribute
+        (attribute_group, "build id",
+         static_cast<char const *> (UniqueBuildID (cctkGH)));
+    }
+    
+    // unique simulation identifier
+    if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
+      F5::write_or_check_attribute
+        (attribute_group, "simulation id",
+         static_cast<char const *> (UniqueSimulationID (cctkGH)));
+    }
+    
+    // unique run identifier
+    if (CCTK_IsFunctionAliased ("UniqueRunID")) {
+      F5::write_or_check_attribute
+        (attribute_group, "run id",
+         static_cast<char const *> (UniqueRunID (cctkGH)));
+    }
+    
+    // Output Cactus parameters as single string
+    {
+      char * const parameters = IOUtil_GetAllParameters (cctkGH, true);
+      assert (parameters);
+      // Create a dataset, since the data may not fit into an attribute
+      hsize_t const size = strlen (parameters) + 1;
+      hid_t const dataspace = H5Screate_simple (1, & size, NULL);
+      assert (dataspace >= 0);
+      hid_t properties = H5Pcreate (H5P_DATASET_CREATE);
+      assert (properties >= 0);
+      check (not H5Pset_chunk (properties, 1, & size));
+      if (compression_level > 0)
+      {
+        check (not H5Pset_deflate (properties, compression_level));
+      }
+      if (write_checksum)
+      {
+        check (not H5Pset_fletcher32 (properties));
+      }
+      hid_t const dataset
+        = H5Dcreate (attribute_group, "All Parameters", H5T_NATIVE_CHAR,
+                     dataspace, H5P_DEFAULT,
+                     properties, H5P_DEFAULT);
+      assert (dataset >= 0);
+      check (not H5Dwrite (dataset, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL,
+                           H5P_DEFAULT, parameters));
+      check (not H5Dclose (dataset));
+      check (not H5Pclose (properties));
+      check (not H5Sclose (dataspace));
+      free (parameters);
+    }
+    
+    check (not H5Gclose (attribute_group));
     
+    // This is far too slow to be useful
+#if 0
     hid_t const parameter_group
       = F5::open_or_create_group (hdf5_file, "Cactus parameters");
+    assert (parameter_group >= 0);
     
     int first = 1;
     for (;;)
@@ -479,8 +618,8 @@ namespace CarpetIOF5 {
       first = 0;
     }
     
-    herr_t const herr = H5Gclose (parameter_group);
-    assert (not herr);
+    check (not H5Gclose (parameter_group));
+#endif
   }
   
   
diff --git a/CarpetDev/CarpetIOF5/src/coordinate_system.cc b/CarpetDev/CarpetIOF5/src/coordinate_system.cc
index bb5b6808f..e030c08a9 100644
--- a/CarpetDev/CarpetIOF5/src/coordinate_system.cc
+++ b/CarpetDev/CarpetIOF5/src/coordinate_system.cc
@@ -102,8 +102,7 @@ namespace CarpetIOF5 {
     Cartesian_coordinate_system_t::
     ~ Cartesian_coordinate_system_t ()
     {
-      herr_t const herr = H5Gclose (m_hdf5_coordinate_system);
-      assert (not herr);
+      check (not H5Gclose (m_hdf5_coordinate_system));
     }
     
     
@@ -114,8 +113,10 @@ namespace CarpetIOF5 {
       assert (all (m_level_delta > (CCTK_REAL) 0));
       
       ostringstream namebuf;
-      namebuf << "Cartesian 3D, x0=" << m_level_origin
-              << ", dx=" << m_level_delta;
+      namebuf << "Cartesian3D-"
+              << name_from_ivect (m_level_origin)
+              << "-"
+              << name_from_ivect (m_level_delta);
       string const namestr = namebuf.str();
       m_name = namestr;
       char const * const name = namestr.c_str();
@@ -124,6 +125,7 @@ namespace CarpetIOF5 {
         = open_or_create_group (m_topology.get_hdf5_topology(), name);
       assert (m_hdf5_coordinate_system >= 0);
       
+#warning "TODO: don't output coordinates as attributes"
       write_or_check_attribute
         (m_hdf5_coordinate_system, "origin", m_level_origin);
       write_or_check_attribute
@@ -133,21 +135,20 @@ namespace CarpetIOF5 {
     
     
     void coordinate_system_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
+      get_topology().get_link_destination (proc, filename, objectname);
+      if (objectname.empty())
       {
-        initialised = true;
-        get_topology().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
+        objectname = m_name;
+      }
+      else
+      {
+        objectname += string ("/") + m_name;
       }
-      filename = l_filename;
-      objectname = l_objectname;
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/coordinate_system.hh b/CarpetDev/CarpetIOF5/src/coordinate_system.hh
index 3a36a458a..2b67b4561 100644
--- a/CarpetDev/CarpetIOF5/src/coordinate_system.hh
+++ b/CarpetDev/CarpetIOF5/src/coordinate_system.hh
@@ -46,7 +46,8 @@ namespace CarpetIOF5 {
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
diff --git a/CarpetDev/CarpetIOF5/src/data_region.cc b/CarpetDev/CarpetIOF5/src/data_region.cc
index 21728c2e6..99b188323 100644
--- a/CarpetDev/CarpetIOF5/src/data_region.cc
+++ b/CarpetDev/CarpetIOF5/src/data_region.cc
@@ -3,12 +3,14 @@
 #include <sstream>
 #include <string>
 
+#include <hdf5.h>
+
 #include "cctk.h"
-#include "cctk_Parameters.h"
 
 #include "carpet.hh"
 
 #include "data_region.hh"
+#include "utils.hh"
 
 
 
@@ -21,54 +23,19 @@ namespace CarpetIOF5 {
     
     
     data_region_t::
-    data_region_t (tensor_component_t & tensor_component,
+    data_region_t (physical_quantity_t & physical_quantity,
                    bbox<int, dim> const & region)
-      : m_tensor_component (tensor_component),
-        m_region (region)
+      : m_physical_quantity (physical_quantity),
+        m_region (region),
+        m_name (string ("region-") + F5::name_from_ibbox (region))
     {
-      DECLARE_CCTK_PARAMETERS;
-      
       assert (not region.empty());
       
-      ostringstream namebuf;
-#if 0
-      namebuf << "map=" << Carpet::map << " "
-              << "region=" << m_region;
-#else
-      namebuf << "region=" << m_region;
-#endif
-      string const namestr = namebuf.str();
-      char const * const name = namestr.c_str();
-      assert (name != 0);
-      
-      int const vartype = CCTK_VarTypeI (m_tensor_component.get_variable());
-      assert (vartype >= 0);
-      hid_t const hdf5_datatype = hdf5_datatype_from_cactus_datatype (vartype);
-      assert (hdf5_datatype >= 0);
-      
-      vect<hsize_t, dim> const dims
-        = (region.shape() / region.stride()).reverse();
-      m_dataspace = H5Screate_simple (dim, & dims [0], 0);
-      assert (m_dataspace >= 0);
-      
-      m_properties = H5Pcreate (H5P_DATASET_CREATE);
-      assert (m_properties >= 0);
-      if (compression_level > 0)
-      {
-        herr_t const herr1 = H5Pset_chunk (m_properties, dim, & dims [0]);
-        assert (not herr1);
-        herr_t const herr2 = H5Pset_deflate (m_properties, compression_level);
-        assert (not herr2);
-      }
-      
-      m_dataset
-        = H5Dcreate (m_tensor_component.get_hdf5_tensor_component(), name,
-                     hdf5_datatype, m_dataspace,
-                     H5P_DEFAULT, m_properties, H5P_DEFAULT);
-      assert (m_dataset >= 0);
-      
-      write_or_check_attribute
-        (m_dataset, "iorigin", region.lower() / region.stride());
+      m_hdf5_data_region
+        = open_or_create_group (m_physical_quantity
+                                .get_hdf5_physical_quantity(),
+                                m_name.c_str());
+      assert (m_hdf5_data_region >= 0);
       
       assert (invariant());
     }
@@ -78,73 +45,35 @@ namespace CarpetIOF5 {
     data_region_t::
     ~ data_region_t ()
     {
-      herr_t herr;
-      
-      herr = H5Dclose (m_dataset);
-      assert (not herr);
-      
-      herr = H5Sclose (m_dataspace);
-      assert (not herr);
-      
-      herr = H5Pclose (m_properties);
+      herr_t const herr = H5Gclose (m_hdf5_data_region);
       assert (not herr);
     }
     
     
     
-    string data_region_t::
-    name_from_region (bbox<int, dim> const & region)
+    physical_quantity_t & data_region_t::
+    get_physical_quantity ()
+      const
     {
-      ostringstream namebuf;
-#if 0
-      namebuf << "map=" << Carpet::map << " "
-              << "region=" << region;
-#else
-      namebuf << "region=" << region;
-#endif
-      return namebuf.str();
+      return m_physical_quantity;
     }
     
     
     
-    tensor_component_t & data_region_t::
-    get_tensor_component ()
+    bbox<int, dim> const & data_region_t::
+    get_region ()
       const
     {
-      return m_tensor_component;
+      return m_region;
     }
     
     
     
-    void data_region_t::
-    write (void const * const data,
-           int const cactus_datatype)
+    hid_t data_region_t::
+    get_hdf5_data_region ()
       const
     {
-      hid_t const memory_hdf5_datatype
-        = hdf5_datatype_from_cactus_datatype (cactus_datatype);
-      assert (memory_hdf5_datatype >= 0);
-      
-      vect<hsize_t, dim> const dims
-        = (m_region.shape() / m_region.stride()).reverse();
-      hid_t const memory_dataspace
-        = H5Screate_simple (dim, & dims [0], & dims [0]);
-      assert (memory_dataspace >= 0);
-      
-      hid_t const transfer_properties = H5Pcreate (H5P_DATASET_XFER);
-      assert (transfer_properties >= 0);
-      
-      herr_t herr;
-      herr
-        = H5Dwrite (m_dataset, memory_hdf5_datatype, memory_dataspace,
-                    m_dataspace, transfer_properties, data);
-      assert (not herr);
-      
-      herr = H5Pclose (transfer_properties);
-      assert (not herr);
-      
-      herr = H5Sclose (memory_dataspace);
-      assert (not herr);
+      return m_hdf5_data_region;
     }
     
     
@@ -154,9 +83,7 @@ namespace CarpetIOF5 {
       const
     {
       return (not m_region.empty()
-              and m_properties >= 0
-              and m_dataset >= 0
-              and m_dataspace >= 0);
+              and m_hdf5_data_region >= 0);
     }
     
   } // namespace F5
diff --git a/CarpetDev/CarpetIOF5/src/data_region.hh b/CarpetDev/CarpetIOF5/src/data_region.hh
index 29d5504c7..35e5ecc79 100644
--- a/CarpetDev/CarpetIOF5/src/data_region.hh
+++ b/CarpetDev/CarpetIOF5/src/data_region.hh
@@ -1,18 +1,14 @@
 #ifndef DATA_REGION_HH
 #define DATA_REGION_HH
 
-// force HDF5 1.8.x installations to use the new API
-#ifdef H5Dcreate_vers
-#undef H5Dcreate_vers
-#endif
-#define H5Dcreate_vers 2
+#include <string>
 
 #include <hdf5.h>
 
 #include "bbox.hh"
 #include "defs.hh"
 
-#include "tensor_component.hh"
+#include "physical_quantity.hh"
 
 
 
@@ -20,15 +16,16 @@ namespace CarpetIOF5 {
   
   namespace F5 {
     
+    using std::string;
+    
     class data_region_t {
       
-      tensor_component_t & m_tensor_component;
+      physical_quantity_t & m_physical_quantity;
       
       bbox<int, dim> const m_region;
+      string const m_name;
       
-      hid_t m_properties;
-      hid_t m_dataset;
-      hid_t m_dataspace;
+      hid_t m_hdf5_data_region;
       
       data_region_t ();
       data_region_t (data_region_t const &);
@@ -36,22 +33,22 @@ namespace CarpetIOF5 {
       
     public:
       
-      data_region_t (tensor_component_t & tensor_component,
+      data_region_t (physical_quantity_t & physical_quantity,
                      bbox<int, dim> const & region);
       
       virtual
       ~ data_region_t ();
       
-      static string
-      name_from_region (bbox<int, dim> const & region);
+      physical_quantity_t &
+      get_physical_quantity ()
+        const;
       
-      tensor_component_t &
-      get_tensor_component ()
+      bbox<int, dim> const &
+      get_region ()
         const;
       
-      void
-      write (void const * data,
-             int cactus_datatype)
+      hid_t
+      get_hdf5_data_region ()
         const;
       
       virtual bool
diff --git a/CarpetDev/CarpetIOF5/src/extending.cc b/CarpetDev/CarpetIOF5/src/extending.cc
index bff256190..26d63cb60 100644
--- a/CarpetDev/CarpetIOF5/src/extending.cc
+++ b/CarpetDev/CarpetIOF5/src/extending.cc
@@ -1,6 +1,5 @@
 #include <cassert>
 #include <cstring>
-#include <iostream>
 
 #include "cctk.h"
 
diff --git a/CarpetDev/CarpetIOF5/src/f5writer.cc b/CarpetDev/CarpetIOF5/src/f5writer.cc
new file mode 100644
index 000000000..b351c7391
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/src/f5writer.cc
@@ -0,0 +1,445 @@
+#include <sstream>
+#include <string>
+
+#include <hdf5.h>
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+#include "carpet.hh"
+
+#include "data_region.hh"
+#include "f5writer.hh"
+#include "file.hh"
+#include "meta_data_region.hh"
+#include "physical_quantity.hh"
+#include "simulation.hh"
+#include "tensor_component.hh"
+#include "timestep.hh"
+#include "topology.hh"
+
+
+
+namespace CarpetIOF5 {
+  
+  f5writer_t::
+  f5writer_t (cGH const * const cctkGH,
+              int const variable)
+    : m_cctkGH (cctkGH),
+      m_variable (variable)
+  {
+  }
+  
+  
+  
+  void f5writer_t::
+  write (F5::file_t & file)
+    const
+  {
+    write_meta (file, file.get_have_metafile());
+  }
+  
+  
+  
+  void f5writer_t::
+  write_meta (F5::file_t & file,
+              bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_meta");
+    }
+    
+    if (Carpet::is_meta_mode())
+    {
+      for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH);
+           not mglevel_iter.done();
+           mglevel_iter.step())
+      {
+        write_one_mglevel (file, have_metafile);
+      }
+    }
+    else
+    {
+      write_one_mglevel (file, have_metafile);
+    }
+  }
+  
+  
+  
+  void f5writer_t::
+  write_one_mglevel (F5::file_t & file,
+                     bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_mglevel mglevel=%d",
+                  Carpet::mglevel);
+    }
+    
+    // ostringstream namebuf;
+    // namebuf << "convlevel=" << m_cctkGH->cctk_convlevel;
+    // string const namestr = namebuf.str();
+    // char const * const name = namestr.c_str();
+    
+    int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
+    assert (grouptype >= 0);
+    switch (grouptype)
+    {
+    case CCTK_ARRAY:
+    case CCTK_SCALAR:
+      {
+        if (Carpet::do_global_mode)
+        {
+          write_global (file, have_metafile);
+        }
+      }
+      break;
+    case CCTK_GF:
+      {
+        if (Carpet::is_global_mode())
+        {
+          for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH);
+               not reflevel_iter.done();
+               reflevel_iter.step())
+          {
+            write_one_reflevel (file, have_metafile);
+          }
+        }
+        else
+        {
+          write_one_reflevel (simulation, have_metafile);
+        }
+      }
+      break;
+    default:
+      assert (0);
+    }
+  }
+  
+  
+  
+  void f5writer_t::
+  write_global (F5::simulation_t & simulation,
+                bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_INFO ("OutputVarAs/write_global");
+    }
+    
+    F5::unigrid_topology_t topology (simulation);
+    
+    int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
+    assert (grouptype >= 0);
+    assert (grouptype == CCTK_SCALAR or grouptype == CCTK_ARRAY);
+    
+    vect<CCTK_REAL, dim> level_origin, level_delta;
+    for (int d=0; d<dim; ++d)
+    {
+      level_origin[d] = 0.0;
+      level_delta[d]  = 1.0;
+    }
+    F5::Cartesian_coordinate_system_t coordinate_system
+      (topology, level_origin, level_delta);
+    
+    int const group = CCTK_GroupIndexFromVarI (m_variable);
+    assert (group >= 0 and group < CCTK_NumGroups());
+    F5::physical_quantity_t physical_quantity (coordinate_system, group);
+    
+    int const map = 0;
+    int const reflevel = 0;
+    int const myproc = CCTK_MyProc (m_cctkGH);
+    dh * const dd = Carpet::arrdata.at(group).at(map).dd;
+    dh::dboxes const & boxes
+      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc);
+    bbox<int, dim> const & region = determine_region (boxes);
+    
+    if (have_metafile)
+    {
+      F5::meta_data_region_t meta_data_region (physical_quantity, region);
+      gh * const hh = Carpet::vhh.at(Carpet::map);
+      int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+      meta_data_region.write (proc);
+    }
+    
+    F5::data_region_t data_region (physical_quantity, region);
+    
+    F5::tensor_component_t tensor_component (data_region, m_variable);
+    int const timelevel = 0;
+    void const * const varptr
+      = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+    assert (varptr != 0);
+    int const vartype = CCTK_VarTypeI (m_variable);
+    assert (vartype >= 0);
+    tensor_component.write (varptr, vartype);
+  }
+  
+  
+  
+  void f5writer_t::
+  write_one_reflevel (F5::file_t & file,
+                      bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_reflevel reflevel=%d",
+                  Carpet::reflevel);
+    }
+    
+    // int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
+    // assert (grouptype >= 0);
+    // assert (grouptype == CCTK_GF);
+    
+    // A name for the simulation
+    string sim_id;
+    if (CCTK_IsFunctionAliased ("UniqueSimulationID"))
+    {
+      sim_id = static_cast<char const *> (UniqueSimulationID (m_cctkGH));
+    }
+    else {
+      char parfilename[10000];
+      int const len = CCTK_ParameterFilename (sizeof parfilename, parfilename);
+      assert (len >= 0 and len < sizeof parfilename);
+      sim_id = parfilename;
+      if (sim_id.length() >= 4
+          and sim_id.substr (sim_id.length() - 4) == ".par")
+      {
+        sim_id = sim_id.substr (0, sim_id.length() - 4);
+      }
+    }
+    
+    // Refinement factor of current refinement level
+    vect <hsize_t, dim> const current_refinement
+      = vect <int, dim>::ref (m_cctkGH->cctk_levfac);
+    
+    // Refinement factor of root refinement level
+    vect <hsize_t, dim> const rootlevel_refinement (1);
+    
+    // Remember time when the root level was output the last time
+    static CCTK_REAL rootlevel_time = 0.0;
+    if (all (current_refinement == 1))
+    {
+      rootlevel_time = m_cctkGH->cctk_time;
+    }
+    
+    F5Path * refinement_path = NULL;
+    if (any (current_refinement != 1))
+    {
+      refinement_path
+        = F5Rcreate_relative_vertex_Qrefinement3D (file,
+                                                   m_cctkGH->cctk_time,
+                                                   sim_id.c_str(),
+                                                   current_refinement,
+                                                   rootlevel_time,
+                                                   rootlevel_refinement);
+    }
+    
+#warning "TODO: what is this doing?"
+    F5T_REFINEMENT3D_POINT = refinement_path->myChart->Point_hid_t; 
+    
+    if (Carpet::is_level_mode())
+    {
+      for (Carpet::map_iterator map_iter (m_cctkGH, grouptype);
+           not map_iter.done();
+           map_iter.step())
+      {
+        write_one_map (simulation, have_metafile);
+      }
+    }
+    else
+    {
+      write_one_map (simulation, have_metafile);
+    }
+  }
+  
+  
+  
+  void f5writer_t::
+  write_one_map (F5::simulation_t & simulation,
+                 bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_map map=%d", Carpet::map);
+    }
+    
+    // F5::mesh_refinement_topology_t topology
+    //   (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels,
+    //    Carpet::spacereflevelfact, Carpet::maxspacereflevelfact);
+    
+    // vect<CCTK_REAL, dim> level_origin, level_delta;
+    // for (int d=0; d<dim; ++d)
+    // {
+    //   cGH const * const cctkGH = m_cctkGH;
+    //   DECLARE_CCTK_ARGUMENTS;
+    //   level_origin[d] = CCTK_ORIGIN_SPACE(d);
+    //   level_delta[d]  = CCTK_DELTA_SPACE(d);
+    // }
+    // F5::Cartesian_coordinate_system_t coordinate_system
+    //   (topology, level_origin, level_delta);
+    // 
+    // int const group = CCTK_GroupIndexFromVarI (m_variable);
+    // assert (group >= 0 and group < CCTK_NumGroups());
+    // F5::physical_quantity_t physical_quantity (coordinate_system, group);
+    
+#warning "TODO: this should depend on the patch number"
+    char const * const coordinate_system = NULL;
+    
+    // Depends on the refinement level
+    string const gridname = sim_id + "-" + current_refinement;
+    
+    F5Path * const myPath
+      = F5Rcreate_vertex_refinement3D (file,
+                                      m_cctkGH->cctk_time,
+                                       sim_id.c_str(),
+                                       refinement,
+                                       coordinate_system);
+    
+    if (not refinement_path)
+    {
+      // Make root level appear as unigrid topology as well, just to
+      // be nice to tools that do not understand AMR
+      F5Rlink_default_vertex_topology (myPath, current_refinement);
+    }
+    
+    // Global information for this refinement level 
+    vect <hsize_t, dim> const level_dims
+      = vect <int, dim>::ref (m_cctkGH->cctk_gsh);
+#warning "TODO: switch to double precision"
+    vect <float, dim> const level_min
+      (CCTK_ORIGIN_SPACE(0), CCTK_ORIGIN_SPACE(1), CCTK_ORIGIN_SPACE(2));
+    vect <float, dim> const level_spacing
+      (CCTK_DELTA_SPACE(0), CCTK_DELTA_SPACE(1), CCTK_DELTA_SPACE(2));
+    vect <float, dim> const level_max
+      = level_min + level_spacing * (level_dims - 1);
+    
+    // Output geometric information for an entire level; it is
+    // uniformely covered in coordinate space
+    assert (dim == 3);
+    F5Fwrite_linear (myPath, FIBER_HDF5_POSITIONS_STRING,
+                     dim, & level_dims [0],
+                     F5T_COORD3_FLOAT, & level_min [0], & level_spacing [0]);
+    
+    F5Fset_range (myPath, & level_min [0], & level_max [0]);
+    
+#error "CONTINUE HERE"
+    
+    if (Carpet::is_singlemap_mode())
+    {
+      int const grouptype = CCTK_GroupTypeI (group);
+      assert (grouptype >= 0);
+      
+      for (Carpet::component_iterator component_iter (m_cctkGH, grouptype);
+           not component_iter.done();
+           component_iter.step())
+      {
+        write_one_component (physical_quantity, have_metafile);
+      }
+    }
+    else
+    {
+      write_one_component (physical_quantity, have_metafile);
+    }
+  }
+  
+  
+  
+  void f5writer_t::
+  write_one_component (F5::physical_quantity_t & physical_quantity,
+                       bool const have_metafile)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_component component=%d",
+                  Carpet::component);
+    }
+    
+    gh * const hh = Carpet::vhh.at(Carpet::map);
+    bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component);
+    if (have_metafile or is_local)
+    {
+      dh * const dd = Carpet::vdd.at(Carpet::map);
+      bbox<int, dim> const & region
+        = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel)
+           .at(Carpet::component).exterior);
+      
+      if (have_metafile)
+      {
+        F5::meta_data_region_t meta_data_region (physical_quantity, region);
+        int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+        meta_data_region.write (proc);
+      }
+      
+      if (is_local)
+      {
+        F5::data_region_t data_region (physical_quantity, region);
+        
+        F5::tensor_component_t tensor_component (data_region, m_variable);
+        int const timelevel = 0;
+        void const * const varptr
+          = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+        assert (varptr != 0);
+        int const vartype = CCTK_VarTypeI (m_variable);
+        assert (vartype >= 0);
+        tensor_component.write (varptr, vartype);
+      }
+    }
+  }
+  
+  
+  
+  bbox<int,dim> const & f5writer_t::
+  determine_region (dh::dboxes const & boxes)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+#warning "TODO: use superregions instead of regions (?  only if the regions are on the same processor?); use HDF5 chunks as well"
+    
+    bbox<int,dim> dh::dboxes::* boxptr;
+    if (CCTK_EQUALS (output_regions, "exterior"))
+    {
+      boxptr = & dh::dboxes::exterior;
+    }
+    else if (CCTK_EQUALS (output_regions, "owned"))
+    {
+      boxptr = & dh::dboxes::owned;
+    }
+    else if (CCTK_EQUALS (output_regions, "interior"))
+    {
+      boxptr = & dh::dboxes::interior;
+    }
+    else
+    {
+      assert (0);
+    }
+    
+    return boxes.*boxptr;
+  }
+  
+} // namespace CarpetIOF5
diff --git a/CarpetDev/CarpetIOF5/src/f5writer.hh b/CarpetDev/CarpetIOF5/src/f5writer.hh
new file mode 100644
index 000000000..ae016721e
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/src/f5writer.hh
@@ -0,0 +1,74 @@
+#ifndef F5WRITER_HH
+#define F5WRITER_HH
+
+#include "cctk.h"
+
+#include "bbox.hh"
+#include "defs.hh"
+#include "dh.hh"
+
+#include "file.hh"
+#include "physical_quantity.hh"
+#include "simulation.hh"
+#include "timestep.hh"
+#include "topology.hh"
+
+
+
+namespace CarpetIOF5 {
+  
+  class f5writer_t {
+    
+    cGH const * const m_cctkGH;
+    int const m_variable;
+    
+  public:
+    
+    f5writer_t (cGH const * cctkGH,
+                int variable);
+    
+    void
+    write (F5::file_t & file)
+      const;
+    
+  private:
+    
+    void
+    write_meta (F5::file_t & file,
+                bool have_metafile)
+      const;
+    
+    void
+    write_one_mglevel (F5::timestep_t & timestep,
+                       bool have_metafile)
+      const;
+    
+    void
+    write_global (F5::simulation_t & simulation,
+                  bool have_metafile)
+      const;
+    
+    void
+    write_one_reflevel (F5::simulation_t & simulation,
+                        bool have_metafile)
+      const;
+    
+    void
+    write_one_map (F5::simulation_t & simulation,
+                   bool have_metafile)
+      const;
+    
+    void
+    write_one_component (F5::physical_quantity_t & physical_quantity,
+                         bool have_metafile)
+      const;
+    
+    bbox<int,dim> const &
+    determine_region (dh::dboxes const & boxes)
+      const;
+    
+  };
+  
+} // namespace CarpetIOF5
+
+#endif  // #ifndef F5WRITER_HH
diff --git a/CarpetDev/CarpetIOF5/src/file.cc b/CarpetDev/CarpetIOF5/src/file.cc
index f200277b5..d78dcb001 100644
--- a/CarpetDev/CarpetIOF5/src/file.cc
+++ b/CarpetDev/CarpetIOF5/src/file.cc
@@ -1,16 +1,19 @@
 #include <algorithm>
 #include <cassert>
 #include <iomanip>
-#include <iostream>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <hdf5.h>
 
 #include "cctk.h"
 #include "cctk_Parameters.h"
 
+#include "defs.hh"
+
 #include "file.hh"
+#include "utils.hh"
 
 
 
@@ -22,58 +25,86 @@ namespace CarpetIOF5 {
     
     file_t::
     file_t (cGH const * const cctkGH,
+            string const path,
             string const basename,
             string const extension,
-            bool const do_truncate)
+            bool const do_truncate,
+            bool const is_metafile,
+            bool const is_datafile)
       : m_cctkGH (cctkGH),
+        m_path (path),
         m_basename (basename),
-        m_extension (extension)
+        m_extension (extension),
+        m_is_metafile (is_metafile),
+        m_is_datafile (is_datafile)
     {
-      assert (cctkGH);
-      
-      int const proc = CCTK_MyProc (cctkGH);
-      m_have_metafile = determine_want_metafile (proc);
-      m_output_processor = determine_output_processor (proc);
+      DECLARE_CCTK_PARAMETERS;
       
-      m_metafilename = make_metafilename ();
-      m_filename = make_filename (proc);
+      assert (cctkGH);
       
-      char const * const metafilenameptr = m_metafilename.c_str();
-      char const * const filenameptr = m_filename.c_str();
+      // Create file names, creating subdirectories to avoid placing
+      // too many files into the same directory
+      check (CCTK_CreateDirectory (mode, path.c_str()) >= 0);
+      if (is_metafile)
+      {
+        m_filename = basename + extension;
+      }
+      else
+      {
+        int const myproc = CCTK_MyProc (cctkGH);
+        m_filename = create_filename (myproc, true);
+      }
+      string const filepath = m_path + "/" + m_filename;
       
       htri_t is_hdf5;
       H5E_BEGIN_TRY {
-        is_hdf5 = H5Fis_hdf5 (filenameptr);
+        is_hdf5 = H5Fis_hdf5 (filepath.c_str());
       } H5E_END_TRY;
       bool const file_exists = is_hdf5 > 0;
       
-      // Ignore the metafile when determining whether the file already
-      // exists
       if (do_truncate or not file_exists)
       {
-        m_hdf5_metafile = -1;
-        if (m_have_metafile)
+        if (veryverbose)
         {
-          m_hdf5_metafile
-            = H5Fcreate (metafilenameptr,
-                         H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+          CCTK_VInfo (CCTK_THORNSTRING,
+                      "H5Fcreate (name=\"%s\")", filepath.c_str());
         }
         m_hdf5_file
-          = H5Fcreate (filenameptr, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+          = H5Fcreate (filepath.c_str(),
+                       H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
       }
       else
       {
-        m_hdf5_metafile = -1;
-        if (m_have_metafile)
+        if (veryverbose)
         {
-          m_hdf5_metafile
-            = H5Fopen (metafilenameptr, H5F_ACC_RDWR, H5P_DEFAULT);
+          CCTK_VInfo (CCTK_THORNSTRING,
+                      "H5Fopen (name=\"%s\")", filepath.c_str());
         }
-        m_hdf5_file = H5Fopen (filenameptr, H5F_ACC_RDWR, H5P_DEFAULT);
+        m_hdf5_file = H5Fopen (filepath.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
       }
+      assert (m_hdf5_file >= 0);
       
-      m_filenames.resize (CCTK_nProcs (cctkGH));
-      m_filenames.at(proc) = m_filename;
+      m_hdf5_fiber_contents = -1;
+      m_hdf5_fiber_global_charts = -1;
+      m_hdf5_fiber_parameter_space = -1;
+      if (m_is_metafile)
+      {
+        create_or_check_version (m_hdf5_file);
+        m_hdf5_fiber_contents
+          = open_or_create_group (m_hdf5_file, "TableOfContents");
+        assert (m_hdf5_fiber_contents >= 0);
+        m_hdf5_fiber_global_charts
+          = open_or_create_group (m_hdf5_file, "Charts");
+        assert (m_hdf5_fiber_global_charts >= 0);
+        m_hdf5_fiber_parameter_space
+          = open_or_create_group (m_hdf5_fiber_contents, "Parameters");
+        assert (m_hdf5_fiber_parameter_space >= 0);
+        
+        hid_t const hdf5_time_group
+          = open_or_create_group (m_hdf5_fiber_parameter_space, "Time");
+        assert (hdf5_time_group >= 0);
+        check (not H5Gclose (hdf5_time_group));
+      }
       
       assert (invariant());
     }
@@ -83,102 +114,91 @@ namespace CarpetIOF5 {
     file_t::
     ~ file_t()
     {
-      if (m_have_metafile)
-      {
-        herr_t const herr = H5Fclose (m_hdf5_metafile);
-        assert (not herr);
-      }
-      herr_t const herr = H5Fclose (m_hdf5_file);
-      assert (not herr);
-    }
-    
-    
-    
-    bool file_t::
-    determine_want_metafile (int const proc)
-      const
-    {
-      DECLARE_CCTK_PARAMETERS;
-      
-      if (CCTK_EQUALS (out_mode, "proc") or
-          CCTK_EQUALS (out_mode, "np"))
-      {
-        return proc == 0;
-      }
-      else if (CCTK_EQUALS (out_mode, "onefile"))
-      {
-        return false;
-      }
-      else
+      if (m_is_metafile)
       {
-        assert (0);
+        check (not H5Gclose (m_hdf5_fiber_parameter_space));
+        check (not H5Gclose (m_hdf5_fiber_contents));
+        check (not H5Gclose (m_hdf5_fiber_global_charts));
       }
+      check (not H5Fclose (m_hdf5_file));
     }
     
     
     
-    int file_t::
-    determine_output_processor (int const proc)
+    string file_t::
+    create_filename (int const proc, bool const create_directories)
       const
     {
       DECLARE_CCTK_PARAMETERS;
       
-      if (CCTK_EQUALS (out_mode, "proc"))
-      {
-        return proc;
-      }
-      else if (CCTK_EQUALS (out_mode, "np"))
-      {
-        return proc / out_proc_every * out_proc_every;
-      }
-      else if (CCTK_EQUALS (out_mode, "onefile"))
+      int const nprocs = CCTK_nProcs (m_cctkGH);
+      string extrapath;
+      if (create_subdirs)
       {
-        return 0;
+        if (nprocs >= 10000)
+        {
+          ostringstream buf;
+          buf << extrapath
+              << m_basename
+              << ".p" << setw (max (0, processor_digits - 4)) << setfill ('0')
+              << proc / 10000
+              << "nnnn/";
+          extrapath = buf.str();
+          if (create_directories)
+          {
+            string const filepath = m_path + "/" + extrapath;
+            check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0);
+          }
+        }
+        if (nprocs >= 100)
+        {
+          ostringstream buf;
+          buf << extrapath
+              << m_basename
+              << ".p" << setw (max (0, processor_digits - 2)) << setfill ('0')
+              << proc / 100
+              << "nn/";
+          extrapath = buf.str();
+          if (create_directories)
+          {
+            string const filepath = m_path + "/" + extrapath;
+            check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0);
+          }
+        }
       }
-      else
+      if (one_dir_per_file and nprocs >= 1)
       {
-        assert (0);
+        ostringstream buf;
+        buf << extrapath
+            << m_basename
+            << ".p" << setw (processor_digits) << setfill ('0')
+            << proc
+            << "/";
+        extrapath = buf.str();
+        if (create_directories)
+        {
+          string const filepath = m_path + "/" + extrapath;
+          check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0);
+        }
       }
+      ostringstream buf;
+      buf << extrapath
+          << m_basename
+          << ".p" << setw (processor_digits) << setfill ('0') << proc
+          << m_extension;
+      return buf.str();
     }
     
-    
-    
-    string file_t::
-    make_metafilename ()
-      const
+
+
+    void file_t::
+    create_or_check_version (hid_t const hdf5_file)
     {
-      return m_basename + m_extension;
-    }
-    
-    
-    
-    string file_t::
-    make_filename (int const proc)
-      const
-    {
-      DECLARE_CCTK_PARAMETERS;
-      
-      ostringstream filenamebuf;
-      
-      filenamebuf << m_basename;
-      
-      if (CCTK_EQUALS (out_mode, "proc") or
-          CCTK_EQUALS (out_mode, "np"))
-      {
-        filenamebuf << ".p" << setw (processor_digits) << setfill ('0') << proc;
-      }
-      else if (CCTK_EQUALS (out_mode, "onefile"))
-      {
-        // do nothing
-      }
-      else
-      {
-        assert (0);
-      }
-      
-      filenamebuf << m_extension;
-      
-      return filenamebuf.str();
+      hid_t const hdf5_group = open_or_create_group (hdf5_file, "version");
+      assert (hdf5_group >= 0);
+      string const version ("http://www.zib.de/visual/F5-0.1.2/");
+      write_or_check_attribute (hdf5_group, "version", version.c_str());
+      check (not H5Gclose (hdf5_group));
     }
     
     
@@ -192,72 +212,50 @@ namespace CarpetIOF5 {
     
     
     
-    bool file_t::
-    get_have_metafile ()
-      const
-    {
-      return m_have_metafile;
-    }
-    
-    
-    
-    int file_t::
-    get_output_processor ()
+    hid_t file_t::
+    get_hdf5_file ()
       const
     {
-      return m_output_processor;
+      return m_hdf5_file;
     }
     
     
     
-    string file_t::
-    get_filename (int const proc)
+    hid_t file_t::
+    get_hdf5_fiber_parameter_space ()
       const
     {
-      assert (proc >= 0 and proc < CCTK_nProcs (m_cctkGH));
-      if (m_filenames.at(proc).empty())
-      {
-        m_filenames.at(proc) = make_filename (proc);
-      }
-      return m_filenames.at(proc);
+      return m_hdf5_fiber_parameter_space;
     }
     
     
     
-    hid_t file_t::
-    get_hdf5_metafile()
+    bool file_t::
+    get_is_metafile ()
       const
     {
-      return m_hdf5_metafile;
+      return m_is_metafile;
     }
     
     
     
-    hid_t file_t::
-    get_hdf5_file()
+    bool file_t::
+    get_is_datafile ()
       const
     {
-      return m_hdf5_file;
+      return m_is_datafile;
     }
     
     
     
     void file_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
-      {
-        initialised = true;
-        l_filename = m_filename;
-        l_objectname = string ("");
-      }
-      filename = l_filename;
-      objectname = l_objectname;
+      filename = create_filename (proc);
+      objectname = "";
     }
     
     
@@ -266,9 +264,7 @@ namespace CarpetIOF5 {
     invariant()
       const
     {
-      return (m_cctkGH != 0
-              and (not m_have_metafile or m_hdf5_metafile >= 0)
-              and m_hdf5_file >= 0);
+      return m_cctkGH != 0 and m_hdf5_file >= 0;
     }
     
   } // namespace F5
diff --git a/CarpetDev/CarpetIOF5/src/file.hh b/CarpetDev/CarpetIOF5/src/file.hh
index 19add249e..44d8ce00c 100644
--- a/CarpetDev/CarpetIOF5/src/file.hh
+++ b/CarpetDev/CarpetIOF5/src/file.hh
@@ -23,48 +23,46 @@ namespace CarpetIOF5 {
     
     class file_t {
       
-      cGH const * const m_cctkGH;
+      // File mode for creating directories
+      static int const mode = 0755;
       
-      bool m_have_metafile;
-      int m_output_processor;
+      cGH const * const m_cctkGH;
       
+      string const m_path;
       string const m_basename;
       string const m_extension;
-      
-      string m_metafilename;
       string m_filename;
       
-      vector <string> mutable m_filenames;
+      bool const m_is_metafile;
+      bool const m_is_datafile;
       
-      hid_t m_hdf5_metafile;
       hid_t m_hdf5_file;
       
+      hid_t m_hdf5_fiber_contents;
+      hid_t m_hdf5_fiber_global_charts;
+      hid_t m_hdf5_fiber_parameter_space;
+      
       file_t ();
       file_t (file_t const &);
       file_t operator= (file_t const &);
       
-      bool
-      determine_want_metafile (int proc)
-        const;
-      
-      int
-      determine_output_processor (int proc)
-        const;
-      
       string
-      make_metafilename ()
+      create_filename (int proc,
+                       bool create_directories = false)
         const;
       
-      string
-      make_filename (int proc)
-        const;
+      static void
+      create_or_check_version (hid_t const hdf5_file);
       
     public:
       
       file_t (cGH const * cctkGH,
-              string filename,
+              string path,
+              string basename,
               string extension,
-              bool do_truncate);
+              bool do_truncate,
+              bool is_metafile,
+              bool is_datafile);
       
       virtual
       ~ file_t ();
@@ -73,28 +71,23 @@ namespace CarpetIOF5 {
       get_cctkGH ()
         const;
       
-      bool
-      get_have_metafile ()
-        const;
-      
-      int
-      get_output_processor ()
+      hid_t
+      get_hdf5_file ()
         const;
       
-      string
-      get_filename (int proc)
+      hid_t
+      get_hdf5_fiber_parameter_space ()
         const;
       
-      hid_t
-      get_hdf5_metafile ()
+      bool get_is_metafile ()
         const;
       
-      hid_t
-      get_hdf5_file ()
+      bool get_is_datafile ()
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
diff --git a/CarpetDev/CarpetIOF5/src/make.code.defn b/CarpetDev/CarpetIOF5/src/make.code.defn
index 0641b4f9d..d0350d235 100644
--- a/CarpetDev/CarpetIOF5/src/make.code.defn
+++ b/CarpetDev/CarpetIOF5/src/make.code.defn
@@ -4,16 +4,16 @@
 SRCS =	IOF5.cc					\
 	coordinate_system.cc			\
 	data_region.cc				\
-        extending.cc				\
-        file.cc					\
+	extending.cc				\
+	file.cc					\
 	meta_data_region.cc			\
-        physical_quantity.cc 			\
-        simulation.cc 				\
-        tensor_component.cc			\
-        timestep.cc 				\
-        topology.cc 				\
-        utils.cc				\
-        writer.cc
+	physical_quantity.cc			\
+	simulation.cc				\
+	tensor_component.cc			\
+	timestep.cc				\
+	topology.cc				\
+	utils.cc				\
+	writer.cc
 
 # Subdirectories containing source files
 SUBDIRS = 
diff --git a/CarpetDev/CarpetIOF5/src/meta_data_region.cc b/CarpetDev/CarpetIOF5/src/meta_data_region.cc
index 505508eb9..88963ec34 100644
--- a/CarpetDev/CarpetIOF5/src/meta_data_region.cc
+++ b/CarpetDev/CarpetIOF5/src/meta_data_region.cc
@@ -3,11 +3,14 @@
 #include <sstream>
 #include <string>
 
+#include <hdf5.h>
+
 #include "cctk.h"
 
 #include "carpet.hh"
 
-#include "data_region.hh"
+#include "defs.hh"
+
 #include "meta_data_region.hh"
 #include "utils.hh"
 
@@ -22,17 +25,14 @@ namespace CarpetIOF5 {
     
     
     meta_data_region_t::
-    meta_data_region_t (tensor_component_t & tensor_component,
+    meta_data_region_t (physical_quantity_t & physical_quantity,
                         bbox<int, dim> const & region)
-      : m_tensor_component (tensor_component),
-        m_region (region)
+      : m_physical_quantity (physical_quantity),
+        m_region (region),
+        m_name (string ("region-") + F5::name_from_ibbox (region))
     {
       assert (not region.empty());
       
-      string const namestr = data_region_t::name_from_region (region);
-      char const * const name = namestr.c_str();
-      assert (name != 0);
-      
       assert (invariant());
     }
     
@@ -45,11 +45,11 @@ namespace CarpetIOF5 {
     
     
     
-    tensor_component_t & meta_data_region_t::
-    get_tensor_component ()
+    physical_quantity_t & meta_data_region_t::
+    get_physical_quantity ()
       const
     {
-      return m_tensor_component;
+      return m_physical_quantity;
     }
     
     
@@ -58,22 +58,48 @@ namespace CarpetIOF5 {
     write (int const proc)
       const
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       string filename;
       string objectname;
-      get_tensor_component().get_link_destination (filename, objectname);
+      get_physical_quantity().get_link_destination (proc, filename, objectname);
       
-      string const name = data_region_t::name_from_region (m_region);
+      hid_t const hdf5_physical_quantity
+        = m_physical_quantity.get_hdf5_physical_quantity();
       
-      hid_t const hdf5_tensor_component
-        = m_tensor_component.get_hdf5_tensor_component();
+      bool link_exists;
+      H5E_BEGIN_TRY {
+        H5L_info_t linkbuf;
+        herr_t const herr
+          = H5Lget_info (hdf5_physical_quantity, m_name.c_str(),
+                         & linkbuf, H5P_DEFAULT);
+        link_exists = herr == 0;
+      } H5E_END_TRY;
+      if (veryverbose)
+      {
+        CCTK_VInfo (CCTK_THORNSTRING,
+                    "Link \"%s\" %s",
+                    m_name.c_str(),
+                    link_exists ? "exists" : "does not exist");
+      }
       
-      herr_t const herr
-        = H5Lcreate_external (filename.c_str(),
-                              objectname.c_str(),
-                              hdf5_tensor_component,
-                              name.c_str(),
-                              H5P_DEFAULT, H5P_DEFAULT);
-      assert (not herr);
+      if (not link_exists)
+      {
+        if (veryverbose)
+        {
+          CCTK_VInfo (CCTK_THORNSTRING,
+                      "H5Lcreate_external "
+                      "(filename=\"%s\", objectname=\"%s\", linkname=\"%s\")",
+                      filename.c_str(),
+                      objectname.c_str(),
+                      m_name.c_str());
+        }
+        check (not H5Lcreate_external (filename.c_str(),
+                                       objectname.c_str(),
+                                       hdf5_physical_quantity,
+                                       m_name.c_str(),
+                                       H5P_DEFAULT, H5P_DEFAULT));
+      }
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/meta_data_region.hh b/CarpetDev/CarpetIOF5/src/meta_data_region.hh
index b109c5da4..ac59c18e2 100644
--- a/CarpetDev/CarpetIOF5/src/meta_data_region.hh
+++ b/CarpetDev/CarpetIOF5/src/meta_data_region.hh
@@ -1,12 +1,14 @@
 #ifndef META_DATA_REGION_HH
 #define META_DATA_REGION_HH
 
+#include <string>
+
 #include <hdf5.h>
 
 #include "bbox.hh"
 #include "defs.hh"
 
-#include "tensor_component.hh"
+#include "physical_quantity.hh"
 
 
 
@@ -14,11 +16,14 @@ namespace CarpetIOF5 {
   
   namespace F5 {
     
+    using std::string;
+    
     class meta_data_region_t {
       
-      tensor_component_t & m_tensor_component;
+      physical_quantity_t & m_physical_quantity;
       
       bbox<int, dim> const m_region;
+      string const m_name;
       
       hid_t m_properties;
       hid_t m_dataset;
@@ -30,14 +35,14 @@ namespace CarpetIOF5 {
       
     public:
       
-      meta_data_region_t (tensor_component_t & tensor_component,
+      meta_data_region_t (physical_quantity_t & physical_quantity,
                           bbox<int, dim> const & region);
       
       virtual
       ~ meta_data_region_t ();
       
-      tensor_component_t &
-      get_tensor_component ()
+      physical_quantity_t &
+      get_physical_quantity ()
         const;
       
       void
diff --git a/CarpetDev/CarpetIOF5/src/physical_quantity.cc b/CarpetDev/CarpetIOF5/src/physical_quantity.cc
index 0b6720ece..d0cb78dcd 100644
--- a/CarpetDev/CarpetIOF5/src/physical_quantity.cc
+++ b/CarpetDev/CarpetIOF5/src/physical_quantity.cc
@@ -1,6 +1,8 @@
 #include <cassert>
 #include <cstdlib>
 
+#include <hdf5.h>
+
 #include "cctk.h"
 
 #include "physical_quantity.hh"
@@ -67,6 +69,15 @@ namespace CarpetIOF5 {
     
     
     
+    string physical_quantity_t::
+    get_name ()
+      const
+    {
+      return m_name;
+    }
+    
+    
+    
     hid_t physical_quantity_t::
     get_hdf5_physical_quantity ()
       const
@@ -77,21 +88,20 @@ namespace CarpetIOF5 {
     
     
     void physical_quantity_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
+      get_coordinate_system().get_link_destination (proc, filename, objectname);
+      if (objectname.empty())
+      {
+        objectname = m_name;
+      }
+      else
       {
-        initialised = true;
-        get_coordinate_system().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
+        objectname += string ("/") + m_name;
       }
-      filename = l_filename;
-      objectname = l_objectname;
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/physical_quantity.hh b/CarpetDev/CarpetIOF5/src/physical_quantity.hh
index 38b670cb2..e2b025b81 100644
--- a/CarpetDev/CarpetIOF5/src/physical_quantity.hh
+++ b/CarpetDev/CarpetIOF5/src/physical_quantity.hh
@@ -40,12 +40,17 @@ namespace CarpetIOF5 {
       get_group ()
         const;
       
+      string
+      get_name ()
+        const;
+      
       hid_t
       get_hdf5_physical_quantity ()
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
diff --git a/CarpetDev/CarpetIOF5/src/simulation.cc b/CarpetDev/CarpetIOF5/src/simulation.cc
index 98a0a3888..1adde5e5a 100644
--- a/CarpetDev/CarpetIOF5/src/simulation.cc
+++ b/CarpetDev/CarpetIOF5/src/simulation.cc
@@ -4,6 +4,8 @@
 
 #include "cctk.h"
 
+#include "defs.hh"
+
 #include "simulation.hh"
 #include "utils.hh"
 
@@ -28,6 +30,10 @@ namespace CarpetIOF5 {
                                 m_name.c_str());
       assert (m_hdf5_simulation >= 0);
       
+      cGH const * const cctkGH = m_timestep.get_file().get_cctkGH();
+      write_or_check_attribute
+        (m_hdf5_simulation, "TimeStep", cctkGH->cctk_iteration);
+      
       assert (invariant());
     }
     
@@ -36,8 +42,7 @@ namespace CarpetIOF5 {
     simulation_t::
     ~ simulation_t()
     {
-      herr_t const herr = H5Gclose (m_hdf5_simulation);
-      assert (not herr);
+      check (not H5Gclose (m_hdf5_simulation));
     }
     
     
@@ -61,21 +66,20 @@ namespace CarpetIOF5 {
     
     
     void simulation_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
+      get_timestep().get_link_destination (proc, filename, objectname);
+      if (objectname.empty())
+      {
+        objectname = m_name;
+      }
+      else
       {
-        initialised = true;
-        get_timestep().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
+        objectname += string ("/") + m_name;
       }
-      filename = l_filename;
-      objectname = l_objectname;
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/simulation.hh b/CarpetDev/CarpetIOF5/src/simulation.hh
index 272b3385c..4c4639958 100644
--- a/CarpetDev/CarpetIOF5/src/simulation.hh
+++ b/CarpetDev/CarpetIOF5/src/simulation.hh
@@ -47,7 +47,8 @@ namespace CarpetIOF5 {
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
diff --git a/CarpetDev/CarpetIOF5/src/tensor_component.cc b/CarpetDev/CarpetIOF5/src/tensor_component.cc
index 5d4b281fc..6cd2b6486 100644
--- a/CarpetDev/CarpetIOF5/src/tensor_component.cc
+++ b/CarpetDev/CarpetIOF5/src/tensor_component.cc
@@ -1,7 +1,15 @@
 #include <cassert>
 #include <cstdlib>
 
+// force HDF5 1.8.x installations to use the new API
+#define H5Dcreate_vers 2
+
+#include <hdf5.h>
+
 #include "cctk.h"
+#include "cctk_Parameters.h"
+
+#include "vect.hh"
 
 #include "tensor_component.hh"
 
@@ -12,22 +20,65 @@ namespace CarpetIOF5 {
   namespace F5 {
     
     tensor_component_t::
-    tensor_component_t (physical_quantity_t & physical_quantity,
+    tensor_component_t (data_region_t & data_region,
                         int const variable)
-      : m_physical_quantity (physical_quantity),
+      : m_data_region (data_region),
         m_variable (variable)
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       assert (variable >= 0 and variable < CCTK_NumVars());
       
       char const * const name = CCTK_VarName (variable);
       assert (name != 0);
       m_name = string (name);
       
-      m_hdf5_tensor_component
-        = open_or_create_group (m_physical_quantity
-                                .get_hdf5_physical_quantity(),
-                                name);
-      assert (m_hdf5_tensor_component >= 0);
+      int const vartype = CCTK_VarTypeI (variable);
+      assert (vartype >= 0);
+      hid_t const hdf5_datatype = hdf5_datatype_from_cactus_datatype (vartype);
+      assert (hdf5_datatype >= 0);
+      
+      bbox<int, dim> const & region = m_data_region.get_region();
+      
+      vect<hsize_t, dim> const dims
+        = (region.shape() / region.stride()).reverse();
+      m_dataspace = H5Screate_simple (dim, & dims [0], 0);
+      assert (m_dataspace >= 0);
+      
+      m_properties = H5Pcreate (H5P_DATASET_CREATE);
+      assert (m_properties >= 0);
+      vect<int, dim> const user_chunk_size
+        (chunk_size_x, chunk_size_y, chunk_size_z);
+      bool const need_chunks
+        = any (user_chunk_size > 0) or compression_level > 0 or write_checksum;
+      if (need_chunks)
+      {
+        vect<hsize_t, dim> const chunk_size
+          = either (user_chunk_size > 0,
+                    vect<hsize_t, dim> (user_chunk_size),
+                    dims);
+        herr_t const herr = H5Pset_chunk (m_properties, dim, & chunk_size [0]);
+        assert (not herr);
+      }
+      if (compression_level > 0)
+      {
+        herr_t const herr = H5Pset_deflate (m_properties, compression_level);
+        assert (not herr);
+      }
+      if (write_checksum)
+      {
+        herr_t const herr = H5Pset_fletcher32 (m_properties);
+        assert (not herr);
+      }
+      
+      m_dataset
+        = H5Dcreate (m_data_region.get_hdf5_data_region(), m_name.c_str(),
+                     hdf5_datatype, m_dataspace,
+                     H5P_DEFAULT, m_properties, H5P_DEFAULT);
+      assert (m_dataset >= 0);
+      
+      write_or_check_attribute
+        (m_dataset, "iorigin", region.lower() / region.stride());
       
       assert (invariant());
     }
@@ -37,17 +88,25 @@ namespace CarpetIOF5 {
     tensor_component_t::
     ~ tensor_component_t ()
     {
-      herr_t const herr = H5Gclose (m_hdf5_tensor_component);
+      herr_t herr;
+      
+      herr = H5Dclose (m_dataset);
+      assert (not herr);
+      
+      herr = H5Sclose (m_dataspace);
+      assert (not herr);
+      
+      herr = H5Pclose (m_properties);
       assert (not herr);
     }
     
     
       
-    physical_quantity_t & tensor_component_t::
-    get_physical_quantity ()
+    data_region_t & tensor_component_t::
+    get_data_region ()
         const
     {
-      return m_physical_quantity;
+      return m_data_region;
     }
     
     
@@ -61,31 +120,38 @@ namespace CarpetIOF5 {
     
     
     
-    hid_t tensor_component_t::
-    get_hdf5_tensor_component ()
-      const
-    {
-      return m_hdf5_tensor_component;
-    }
-    
-    
-    
+#warning "TODO: This assumes that the shape of data is the same as the shape of the region; this may not be so if not all of the data are written out"
     void tensor_component_t::
-    get_link_destination (string & filename,
-                          string & objectname)
+    write (void const * const data,
+           int const cactus_datatype)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
-      {
-        initialised = true;
-        get_physical_quantity().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
-      }
-      filename = l_filename;
-      objectname = l_objectname;
+      hid_t const memory_hdf5_datatype
+        = hdf5_datatype_from_cactus_datatype (cactus_datatype);
+      assert (memory_hdf5_datatype >= 0);
+      
+      bbox<int, dim> const & region = m_data_region.get_region();
+      
+      vect<hsize_t, dim> const dims
+        = (region.shape() / region.stride()).reverse();
+      hid_t const memory_dataspace
+        = H5Screate_simple (dim, & dims [0], & dims [0]);
+      assert (memory_dataspace >= 0);
+      
+      hid_t const transfer_properties = H5Pcreate (H5P_DATASET_XFER);
+      assert (transfer_properties >= 0);
+      
+      herr_t herr;
+      herr
+        = H5Dwrite (m_dataset, memory_hdf5_datatype, memory_dataspace,
+                    m_dataspace, transfer_properties, data);
+      assert (not herr);
+      
+      herr = H5Pclose (transfer_properties);
+      assert (not herr);
+      
+      herr = H5Sclose (memory_dataspace);
+      assert (not herr);
     }
     
     
@@ -95,7 +161,9 @@ namespace CarpetIOF5 {
       const
     {
       return (m_variable >= 0 and m_variable < CCTK_NumVars()
-              and m_hdf5_tensor_component >= 0);
+              and m_properties >= 0
+              and m_dataset >= 0
+              and m_dataspace >= 0);
     }
     
   } // namespace F5
diff --git a/CarpetDev/CarpetIOF5/src/tensor_component.hh b/CarpetDev/CarpetIOF5/src/tensor_component.hh
index f659c0076..8a95d6faa 100644
--- a/CarpetDev/CarpetIOF5/src/tensor_component.hh
+++ b/CarpetDev/CarpetIOF5/src/tensor_component.hh
@@ -1,10 +1,11 @@
 #ifndef TENSOR_COMPONENT_HH
 #define TENSOR_COMPONENT_HH
 
+#include <string>
+
 #include <hdf5.h>
 
-#include "physical_quantity.hh"
-#include "utils.hh"
+#include "data_region.hh"
 
 
 
@@ -14,12 +15,14 @@ namespace CarpetIOF5 {
     
     class tensor_component_t {
       
-      physical_quantity_t & m_physical_quantity;
+      data_region_t & m_data_region;
       
       int const m_variable;
       string m_name;
       
-      hid_t m_hdf5_tensor_component;
+      hid_t m_dataspace;
+      hid_t m_properties;
+      hid_t m_dataset;
       
       tensor_component_t ();
       tensor_component_t (tensor_component_t const &);
@@ -27,27 +30,23 @@ namespace CarpetIOF5 {
       
     public:
       
-      tensor_component_t (physical_quantity_t & physical_quantity,
+      tensor_component_t (data_region_t & data_region,
                           int variable);
       
       virtual
       ~ tensor_component_t ();
       
-      physical_quantity_t &
-      get_physical_quantity ()
+      data_region_t &
+      get_data_region ()
         const;
       
       hid_t
       get_variable ()
         const;
       
-      hid_t
-      get_hdf5_tensor_component ()
-        const;
-      
       void
-      get_link_destination (string & filename,
-                            string & objectname)
+      write (void const * data,
+             int cactus_datatype)
         const;
       
       virtual bool
diff --git a/CarpetDev/CarpetIOF5/src/timestep.cc b/CarpetDev/CarpetIOF5/src/timestep.cc
index 73d106a92..409f47221 100644
--- a/CarpetDev/CarpetIOF5/src/timestep.cc
+++ b/CarpetDev/CarpetIOF5/src/timestep.cc
@@ -9,6 +9,8 @@
 #include "cctk.h"
 #include "cctk_Functions.h"
 
+#include "defs.hh"
+
 #include "timestep.hh"
 #include "utils.hh"
 
@@ -42,21 +44,14 @@ namespace CarpetIOF5 {
       {
         // Create a string from the time without losing information
         int const precision = numeric_limits<CCTK_REAL>::digits10 + 2;
-        buf << setprecision (precision) << "t=" << time;
+        buf << setprecision (precision) << "T=" << time;
       }
       m_name = buf.str();
       
       m_hdf5_timestep
         = open_or_create_group (m_file.get_hdf5_file(), m_name.c_str());
       assert (m_hdf5_timestep >= 0);
-      write_or_check_attribute (m_hdf5_timestep, "time", time);
-      
-      if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
-        cGH const * const cctkGH = get_file().get_cctkGH();
-        char const * const job_id
-          = static_cast<char const *> (UniqueSimulationID (cctkGH));
-        write_or_check_attribute (m_hdf5_timestep, "simulation id", job_id);
-      }
+      write_or_check_attribute (m_hdf5_timestep, "Time", time);
       
       assert (invariant());
     }
@@ -66,8 +61,7 @@ namespace CarpetIOF5 {
     timestep_t::
     ~ timestep_t()
     {
-      herr_t const herr = H5Gclose (m_hdf5_timestep);
-      assert (not herr);
+      check (not H5Gclose (m_hdf5_timestep));
     }
     
     
@@ -100,21 +94,20 @@ namespace CarpetIOF5 {
     
     
     void timestep_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
+      get_file().get_link_destination (proc, filename, objectname);
+      if (objectname.empty())
+      {
+        objectname = m_name;
+      }
+      else
       {
-        initialised = true;
-        get_file().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
+        objectname += string ("/") + m_name;
       }
-      filename = l_filename;
-      objectname = l_objectname;
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/timestep.hh b/CarpetDev/CarpetIOF5/src/timestep.hh
index 3a44ec29b..d70a00024 100644
--- a/CarpetDev/CarpetIOF5/src/timestep.hh
+++ b/CarpetDev/CarpetIOF5/src/timestep.hh
@@ -53,7 +53,8 @@ namespace CarpetIOF5 {
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
diff --git a/CarpetDev/CarpetIOF5/src/topology.cc b/CarpetDev/CarpetIOF5/src/topology.cc
index 401f65271..bc05c73a7 100644
--- a/CarpetDev/CarpetIOF5/src/topology.cc
+++ b/CarpetDev/CarpetIOF5/src/topology.cc
@@ -62,7 +62,7 @@ namespace CarpetIOF5 {
     unigrid_topology_t (simulation_t & simulation)
       : topology_t (simulation)
     {
-      char const * const name = "Vertices";
+      char const * const name = "uniform";
       m_name = string (name);
       m_hdf5_topology
         = open_or_create_group (m_simulation.get_hdf5_simulation(), name);
@@ -94,6 +94,7 @@ namespace CarpetIOF5 {
     mesh_refinement_topology_t::
     mesh_refinement_topology_t (simulation_t & simulation,
                                 int const map,
+                                int const maps,
                                 int const refinement_level,
                                 int const max_refinement_levels,
                                 vect<int, dim> const & level_refinement_factor,
@@ -110,7 +111,15 @@ namespace CarpetIOF5 {
       assert (all (level_refinement_factor <= max_refinement_factor));
       
       ostringstream namebuf;
-      namebuf << "Vertices map=" << map << " level=" << refinement_level;
+      namebuf << "Vertices";
+      if (maps > 1)
+      {
+        namebuf << "-map" << map;
+      }
+      if (max_refinement_levels > 1)
+      {
+        namebuf << "-level" << refinement_level;
+      }
       string const namestr = namebuf.str();
       m_name = namestr;
       char const * const name = namestr.c_str();
@@ -157,21 +166,20 @@ namespace CarpetIOF5 {
     
     
     void topology_t::
-    get_link_destination (string & filename,
+    get_link_destination (int const proc,
+                          string & filename,
                           string & objectname)
       const
     {
-      static bool initialised = false;
-      static string l_filename;
-      static string l_objectname;
-      if (not initialised)
+      get_simulation().get_link_destination (proc, filename, objectname);
+      if (objectname.empty())
+      {
+        objectname = m_name;
+      }
+      else
       {
-        initialised = true;
-        get_simulation().get_link_destination (l_filename, l_objectname);
-        l_objectname += string ("/") + m_name;
+        objectname += string ("/") + m_name;
       }
-      filename = l_filename;
-      objectname = l_objectname;
     }
     
     
diff --git a/CarpetDev/CarpetIOF5/src/topology.hh b/CarpetDev/CarpetIOF5/src/topology.hh
index df841107b..e179e71db 100644
--- a/CarpetDev/CarpetIOF5/src/topology.hh
+++ b/CarpetDev/CarpetIOF5/src/topology.hh
@@ -45,7 +45,8 @@ namespace CarpetIOF5 {
         const;
       
       void
-      get_link_destination (string & filename,
+      get_link_destination (int proc,
+                            string & filename,
                             string & objectname)
         const;
       
@@ -97,6 +98,7 @@ namespace CarpetIOF5 {
       
       mesh_refinement_topology_t (simulation_t & simulation,
                                   int map,
+                                  int maps,
                                   int refinement_level,
                                   int max_refinement_levels,
                                   vect<int, dim> const & level_refinement_factors,
diff --git a/CarpetDev/CarpetIOF5/src/utils.cc b/CarpetDev/CarpetIOF5/src/utils.cc
index b57e06a9e..d311f0992 100644
--- a/CarpetDev/CarpetIOF5/src/utils.cc
+++ b/CarpetDev/CarpetIOF5/src/utils.cc
@@ -1,11 +1,23 @@
 #include <cassert>
 #include <complex>
 #include <cstring>
+#include <sstream>
 #include <vector>
 
+// force HDF5 1.8.x installations to use the new API
+#define H5Acreate_vers 2
+#define H5Gcreate_vers 2
+#define H5Gopen_vers 2
+#define H5Tarray_create_vers 2
+
+#include <hdf5.h>
+
 #include "cctk.h"
+#include "cctk_Parameters.h"
 
+#include "bbox.hh"
 #include "defs.hh"
+#include "vect.hh"
 
 #include "utils.hh"
 
@@ -175,28 +187,96 @@ namespace CarpetIOF5 {
     
     
     
+    template <typename T, int D>
+    string
+    name_from_ivect (vect <T, D> const & ivect)
+    {
+      ostringstream buf;
+      for (int d = 0; d < dim; ++ d)
+      {
+        if (d > 0)
+        {
+          buf << ":";
+        }
+        buf << ivect[d];
+      }
+      return buf.str ();
+    }
+    
+    template
+    string
+    name_from_ivect (vect <int, dim> const & ivect);
+    
+    template
+    string
+    name_from_ivect (vect <CCTK_REAL, dim> const & ivect);
+    
+    
+    
+    template <typename T, int D>
+    string
+    name_from_ibbox (bbox <T, D> const & ibbox)
+    {
+      ostringstream buf;
+      buf << name_from_ivect (ibbox.lower ())
+          << "-"
+          << name_from_ivect (ibbox.upper ())
+          << "-"
+          << name_from_ivect (ibbox.stride ());
+      return buf.str ();
+    }
+    
+    template
+    string
+    name_from_ibbox (bbox <int, dim> const & ivect);
+    
+    
+    
     hid_t
     open_or_create_group (hid_t const where,
                           char const * const name)
     {
+      DECLARE_CCTK_PARAMETERS;
+      
       assert (where >= 0);
       assert (name != 0);
       
+      // bool group_exists;
+      // H5E_BEGIN_TRY {
+      //   H5G_stat_t statbuf;
+      //   herr_t const herr = H5Gget_objinfo (where, name, true, & statbuf);
+      //   group_exists = herr == 0;
+      // } H5E_END_TRY;
       bool group_exists;
       H5E_BEGIN_TRY {
-        H5G_stat_t statbuf;
+        H5G_info_t groupinfo;
         herr_t const herr
-          = H5Gget_objinfo (where, name, true, & statbuf);
+          = H5Gget_info_by_name (where, name, & groupinfo, H5P_DEFAULT);
         group_exists = herr == 0;
       } H5E_END_TRY;
+      if (veryverbose)
+      {
+        CCTK_VInfo (CCTK_THORNSTRING,
+                    "Group \"%s\" %s",
+                    name,
+                    group_exists ? "exists" : "does not exist");
+      }
       
       hid_t group;
       if (group_exists)
       {
+        if (veryverbose)
+        {
+          CCTK_VInfo (CCTK_THORNSTRING, "H5Gopen (name=\"%s\")", name);
+        }
         group = H5Gopen (where, name, H5P_DEFAULT);
       }
       else
       {
+        if (veryverbose)
+        {
+          CCTK_VInfo (CCTK_THORNSTRING, "H5Gcreate (name=\"%s\")", name);
+        }
         group = H5Gcreate (where, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
       }
       
diff --git a/CarpetDev/CarpetIOF5/src/utils.hh b/CarpetDev/CarpetIOF5/src/utils.hh
index ae7cb65a1..8d9d0a925 100644
--- a/CarpetDev/CarpetIOF5/src/utils.hh
+++ b/CarpetDev/CarpetIOF5/src/utils.hh
@@ -1,24 +1,6 @@
 #ifndef UTILS_HH
 #define UTILS_HH
 
-// force HDF5 1.8.x installations to use the new API
-#ifdef H5Gopen_vers
-#undef H5Gopen_vers
-#endif
-#ifdef H5Gcreate_vers
-#undef H5Gcreate_vers
-#endif
-#ifdef H5Acreate_vers
-#undef H5Acreate_vers
-#endif
-#ifdef H5Tarray_create_vers
-#undef H5Tarray_create_vers
-#endif
-#define H5Gopen_vers 2
-#define H5Gcreate_vers 2
-#define H5Acreate_vers 2
-#define H5Tarray_create_vers 2
-
 #include <hdf5.h>
 
 #include "cctk.h"
@@ -69,6 +51,16 @@ namespace CarpetIOF5 {
     
     
     
+    template <typename T, int D>
+    string
+    name_from_ivect (vect <T, D> const & ivect);
+    
+    template <typename T, int D>
+    string
+    name_from_ibbox (bbox <T, D> const & ibbox);
+    
+    
+    
     hid_t
     open_or_create_group (hid_t where,
                           char const * name);
diff --git a/CarpetDev/CarpetIOF5/src/writer-old.cc b/CarpetDev/CarpetIOF5/src/writer-old.cc
new file mode 100644
index 000000000..4c4f639a8
--- /dev/null
+++ b/CarpetDev/CarpetIOF5/src/writer-old.cc
@@ -0,0 +1,511 @@
+#include <cstdlib>
+#include <sstream>
+#include <string>
+
+#include <hdf5.h>
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+#include "carpet.hh"
+
+#include "data_region.hh"
+#include "file.hh"
+#include "meta_data_region.hh"
+#include "physical_quantity.hh"
+#include "simulation.hh"
+#include "tensor_component.hh"
+#include "timestep.hh"
+#include "topology.hh"
+#include "writer.hh"
+
+
+
+namespace CarpetIOF5 {
+  
+  writer_t::
+  writer_t (cGH const * const cctkGH,
+            int const variable)
+    : m_cctkGH (cctkGH),
+      m_variable (variable)
+  {
+  }
+  
+  
+  
+  void writer_t::
+  write (F5::file_t & file)
+    const
+  {
+    write_meta (file);
+  }
+  
+  
+  
+  void writer_t::
+  write_meta (F5::file_t & file)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_meta");
+    }
+    
+    F5::timestep_t timestep (file, m_cctkGH->cctk_time);
+    
+    if (Carpet::is_meta_mode())
+    {
+      for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH);
+           not mglevel_iter.done();
+           mglevel_iter.step())
+      {
+        write_one_mglevel (timestep);
+      }
+    }
+    else
+    {
+      write_one_mglevel (timestep);
+    }
+  }
+  
+  
+  
+  void writer_t::
+  write_one_mglevel (F5::timestep_t & timestep)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_mglevel mglevel=%d",
+                  Carpet::mglevel);
+    }
+    
+    // ostringstream namebuf;
+    // namebuf << sim_id;
+    // if (m_cctkGH->cctk_convlevel != 0)
+    // {
+    //   namebuf << "-convlevel" << m_cctkGH->cctk_convlevel;
+    // }
+    // string const name = namebuf.str();
+    // F5::simulation_t simulation (timestep, name.c_str());
+    
+    int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
+    assert (grouptype >= 0);
+    switch (grouptype)
+    {
+    case CCTK_ARRAY:
+    case CCTK_SCALAR:
+      {
+        if (Carpet::is_global_mode())
+        {
+          int const rl = 0;
+          enter_level_mode (m_cctkGH, rl);
+          write_global_reflevel (timestep);
+          leave_level_mode (m_cctkGH);
+        }
+        else
+        {
+          if (Carpet::do_global_mode)
+          {
+            write_global_reflevel (timestep);
+          }
+        }
+      }
+      break;
+    case CCTK_GF:
+      {
+        if (Carpet::is_global_mode())
+        {
+          for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH);
+               not reflevel_iter.done();
+               reflevel_iter.step())
+          {
+            write_one_reflevel (timestep);
+          }
+        }
+        else
+        {
+          write_one_reflevel (simulation);
+        }
+      }
+      break;
+    default:
+      assert (0);
+    }
+  }
+  
+  
+  
+  void writer_t::
+  write_global_reflevel (F5::timestep_t & timestep)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_INFO ("OutputVarAs/write_global_reflevel");
+    }
+    
+    if (Carpet::is_level_mode())
+    {
+      for (Carpet::map_iterator map_iter (m_cctkGH, grouptype);
+           not map_iter.done();
+           map_iter.step())
+      {
+        write_global_map (timestep);
+      }
+    }
+    else
+    {
+      write_global_map (timestep);
+    }
+  }
+  
+  
+  
+  void writer_t::
+  write_global_map (F5::timestep_t & timestep)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_INFO ("OutputVarAs/write_global_map");
+    }
+    
+    write_global_component (timestep);
+  }
+  
+  
+  
+  void writer_t::
+  write_global_component (F5::timestep_t & timestep)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_INFO ("OutputVarAs/write_global_component");
+    }
+    
+    int const group = CCTK_GroupIndexFromVarI (m_variable);
+    assert (group >= 0 and group < CCTK_NumGroups());
+    cGroup groupdata;
+    check (not CCTK_GropuData (group, & groupdata));
+    assert (groupdata.grouptype == CCTK_SCALAR
+            or groupdata.grouptype == CCTK_ARRAY);
+    
+      for (Carpet::component_iterator component_iter (m_cctkGH, grouptype);
+           not component_iter.done();
+           component_iter.step())
+
+
+
+    int const myproc = CCTK_MyProc (m_cctkGH);
+    if (groupdata.disttype == CCTK_DISTRIB_CONSTANT and myproc != 0)
+    {
+      // Output DISTRIB=constant groups only on the root processor
+      return;
+    }
+    
+    // Name the grid after the variable group
+    char * const c_name = CCTK_GroupNameFromVarI (m_variable);
+    string const name = c_name;
+    free (c_name);
+    F5::simulation_t simulation (timestep, name.c_str());
+    
+    F5::unigrid_topology_t topology (simulation);
+    
+    vect<CCTK_REAL, dim> const level_origin (0.0), level_delta (1.0);
+    F5::Cartesian_coordinate_system_t coordinate_system
+      (topology, level_origin, level_delta);
+    
+    F5::physical_quantity_t physical_quantity (coordinate_system, group);
+    
+    int const map = 0;
+    int const reflevel = 0;
+    dh * const dd = Carpet::arrdata.at(group).at(map).dd;
+    dh::dboxes const & boxes
+      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc);
+    bbox<int, dim> const & region = determine_region (boxes);
+    
+    F5::file_t & file = timestep.get_file();
+    bool const have_metafile
+      = file.get_is_metafile() and not file.get_is_datafile();
+    if (have_metafile)
+    {
+      F5::meta_data_region_t meta_data_region (physical_quantity, region);
+      gh * const hh = Carpet::vhh.at(Carpet::map);
+      int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+        meta_data_region.write (proc);
+    }
+    
+    F5::data_region_t data_region (physical_quantity, region);
+    
+    F5::tensor_component_t tensor_component (data_region, m_variable);
+    int const timelevel = 0;
+    void const * const varptr
+      = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+    assert (varptr != 0);
+    int const vartype = CCTK_VarTypeI (m_variable);
+    assert (vartype >= 0);
+    tensor_component.write (varptr, vartype);
+  }
+  
+  
+  
+  void writer_t::
+  write_global (F5::timestep_t & timestep)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_INFO ("OutputVarAs/write_global");
+    }
+    
+    int const group = CCTK_GroupIndexFromVarI (m_variable);
+    assert (group >= 0 and group < CCTK_NumGroups());
+    cGroup groupdata;
+    check (not CCTK_GropuData (group, & groupdata));
+    assert (groupdata.grouptype == CCTK_SCALAR
+            or groupdata.grouptype == CCTK_ARRAY);
+    
+      for (Carpet::component_iterator component_iter (m_cctkGH, grouptype);
+           not component_iter.done();
+           component_iter.step())
+
+
+
+    int const myproc = CCTK_MyProc (m_cctkGH);
+    if (groupdata.disttype == CCTK_DISTRIB_CONSTANT and myproc != 0)
+    {
+      // Output DISTRIB=constant groups only on the root processor
+      return;
+    }
+    
+    // Name the grid after the variable group
+    char * const c_name = CCTK_GroupNameFromVarI (m_variable);
+    string const name = c_name;
+    free (c_name);
+    F5::simulation_t simulation (timestep, name.c_str());
+    
+    F5::unigrid_topology_t topology (simulation);
+    
+    vect<CCTK_REAL, dim> const level_origin (0.0), level_delta (1.0);
+    F5::Cartesian_coordinate_system_t coordinate_system
+      (topology, level_origin, level_delta);
+    
+    F5::physical_quantity_t physical_quantity (coordinate_system, group);
+    
+    int const map = 0;
+    int const reflevel = 0;
+    dh * const dd = Carpet::arrdata.at(group).at(map).dd;
+    dh::dboxes const & boxes
+      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc);
+    bbox<int, dim> const & region = determine_region (boxes);
+    
+    F5::file_t & file = timestep.get_file();
+    bool const have_metafile
+      = file.get_is_metafile() and not file.get_is_datafile();
+    if (have_metafile)
+    {
+      F5::meta_data_region_t meta_data_region (physical_quantity, region);
+      gh * const hh = Carpet::vhh.at(Carpet::map);
+      int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+        meta_data_region.write (proc);
+    }
+    
+    F5::data_region_t data_region (physical_quantity, region);
+    
+    F5::tensor_component_t tensor_component (data_region, m_variable);
+    int const timelevel = 0;
+    void const * const varptr
+      = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+    assert (varptr != 0);
+    int const vartype = CCTK_VarTypeI (m_variable);
+    assert (vartype >= 0);
+    tensor_component.write (varptr, vartype);
+  }
+  
+  
+  
+  void writer_t::
+  write_one_map (F5::simulation_t & simulation)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_map map=%d", Carpet::map);
+    }
+    
+    F5::mesh_refinement_topology_t topology
+      (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels,
+       Carpet::spacereflevelfact, Carpet::maxspacereflevelfact);
+    
+    vect<CCTK_REAL, dim> level_origin, level_delta;
+    for (int d=0; d<dim; ++d)
+    {
+      cGH const * const cctkGH = m_cctkGH;
+      DECLARE_CCTK_ARGUMENTS;
+      level_origin[d] = CCTK_ORIGIN_SPACE(d);
+      level_delta[d]  = CCTK_DELTA_SPACE(d);
+    }
+    F5::Cartesian_coordinate_system_t coordinate_system
+      (topology, level_origin, level_delta);
+    
+    int const group = CCTK_GroupIndexFromVarI (m_variable);
+    assert (group >= 0 and group < CCTK_NumGroups());
+    F5::physical_quantity_t physical_quantity (coordinate_system, group);
+    
+    if (Carpet::is_singlemap_mode())
+    {
+      int const grouptype = CCTK_GroupTypeI (group);
+      assert (grouptype >= 0);
+      
+      for (Carpet::component_iterator component_iter (m_cctkGH, grouptype);
+           not component_iter.done();
+           component_iter.step())
+      {
+        write_one_component (physical_quantity);
+      }
+    }
+    else
+    {
+      write_one_component (physical_quantity);
+    }
+  }
+  
+  
+  
+  void writer_t::
+  write_one_reflevel (F5::simulation_t & simulation)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_reflevel reflevel=%d",
+                  Carpet::reflevel);
+    }
+    
+    int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
+    assert (grouptype >= 0);
+    assert (grouptype == CCTK_GF);
+    
+    if (Carpet::is_level_mode())
+    {
+      for (Carpet::map_iterator map_iter (m_cctkGH, grouptype);
+           not map_iter.done();
+           map_iter.step())
+      {
+        write_one_map (simulation);
+      }
+    }
+    else
+    {
+      write_one_map (simulation);
+    }
+  }
+  
+  
+  
+  void writer_t::
+  write_one_component (F5::physical_quantity_t & physical_quantity)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (verbose or veryverbose)
+    {
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_one_component component=%d",
+                  Carpet::component);
+    }
+    
+    F5::file_t & file
+      = (physical_quantity.get_coordinate_system().get_topology()
+         .get_simulation().get_timestep().get_file());
+    bool const have_metafile
+      = file.get_is_metafile() and not file.get_is_datafile();
+    gh * const hh = Carpet::vhh.at(Carpet::map);
+    bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component);
+    if (have_metafile or is_local)
+    {
+      dh * const dd = Carpet::vdd.at(Carpet::map);
+      bbox<int, dim> const & region
+        = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel)
+           .at(Carpet::component).exterior);
+      
+      if (have_metafile)
+      {
+        F5::meta_data_region_t meta_data_region (physical_quantity, region);
+        int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+        meta_data_region.write (proc);
+      }
+      
+      if (is_local)
+      {
+        F5::data_region_t data_region (physical_quantity, region);
+        
+        F5::tensor_component_t tensor_component (data_region, m_variable);
+        int const timelevel = 0;
+        void const * const varptr
+          = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+        assert (varptr != 0);
+        int const vartype = CCTK_VarTypeI (m_variable);
+        assert (vartype >= 0);
+        tensor_component.write (varptr, vartype);
+      }
+    }
+  }
+  
+  
+  
+  bbox<int,dim> const & writer_t::
+  determine_region (dh::dboxes const & boxes)
+    const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    // TODO: use superregions instead of regions (?  only if the
+    // regions are on the same processor?)
+    
+    bbox<int,dim> dh::dboxes::* boxptr;
+    if (CCTK_EQUALS (output_regions, "exterior"))
+    {
+      boxptr = & dh::dboxes::exterior;
+    }
+    else if (CCTK_EQUALS (output_regions, "owned"))
+    {
+      boxptr = & dh::dboxes::owned;
+    }
+    else if (CCTK_EQUALS (output_regions, "interior"))
+    {
+      boxptr = & dh::dboxes::interior;
+    }
+    else
+    {
+      assert (0);
+    }
+    
+    return boxes.*boxptr;
+  }
+  
+} // namespace CarpetIOF5
diff --git a/CarpetDev/CarpetIOF5/src/writer.cc b/CarpetDev/CarpetIOF5/src/writer.cc
index df1949369..cf00271a0 100644
--- a/CarpetDev/CarpetIOF5/src/writer.cc
+++ b/CarpetDev/CarpetIOF5/src/writer.cc
@@ -1,22 +1,29 @@
+#include <cstdlib>
 #include <sstream>
+#include <string>
+
+#include <hdf5.h>
 
 #include "cctk.h"
 #include "cctk_Arguments.h"
 #include "cctk_Parameters.h"
 
 #include "carpet.hh"
+#include "modes.hh"
 
 #include "data_region.hh"
 #include "file.hh"
 #include "meta_data_region.hh"
+#include "physical_quantity.hh"
 #include "simulation.hh"
 #include "tensor_component.hh"
 #include "timestep.hh"
 #include "topology.hh"
+#include "utils.hh"
 #include "writer.hh"
 
 
-  
+
 namespace CarpetIOF5 {
   
   writer_t::
@@ -33,20 +40,18 @@ namespace CarpetIOF5 {
   write (F5::file_t & file)
     const
   {
-    write_meta (file, file.get_have_metafile());
+    write_meta (file);
   }
   
   
   
   void writer_t::
-  write_meta (F5::file_t & file,
-              bool const have_metafile)
+  write_meta (F5::file_t & file)
     const
   {
     DECLARE_CCTK_PARAMETERS;
     
-    if (verbose or veryverbose)
-    {
+    if (verbose or veryverbose) {
       CCTK_VInfo (CCTK_THORNSTRING,
                   "OutputVarAs/write_meta");
     }
@@ -55,24 +60,22 @@ namespace CarpetIOF5 {
     
     if (Carpet::is_meta_mode())
     {
-      for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH);
-           not mglevel_iter.done();
-           mglevel_iter.step())
+      BEGIN_MGLEVEL_LOOP (m_cctkGH)
       {
-        write_one_mglevel (timestep, have_metafile);
+        write_one_mglevel (timestep);
       }
+      END_MGLEVEL_LOOP;
     }
     else
     {
-      write_one_mglevel (timestep, have_metafile);
+      write_one_mglevel (timestep);
     }
   }
   
   
   
   void writer_t::
-  write_one_mglevel (F5::timestep_t & timestep,
-                     bool const have_metafile)
+  write_one_mglevel (F5::timestep_t & timestep)
     const
   {
     DECLARE_CCTK_PARAMETERS;
@@ -84,12 +87,6 @@ namespace CarpetIOF5 {
                   Carpet::mglevel);
     }
     
-    ostringstream namebuf;
-    namebuf << "convlevel=" << m_cctkGH->cctk_convlevel;
-    string const namestr = namebuf.str();
-    char const * const name = namestr.c_str();
-    F5::simulation_t simulation (timestep, name);
-    
     int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
     assert (grouptype >= 0);
     switch (grouptype)
@@ -97,9 +94,20 @@ namespace CarpetIOF5 {
     case CCTK_ARRAY:
     case CCTK_SCALAR:
       {
-        if (Carpet::do_global_mode)
+        if (Carpet::is_global_mode())
         {
-          write_global (simulation, have_metafile);
+          ENTER_LEVEL_MODE (m_cctkGH, 0)
+          {
+            write_global (timestep);
+          }
+          LEAVE_LEVEL_MODE;
+        }
+        else
+        {
+          if (Carpet::do_global_mode)
+          {
+            write_global (timestep);
+          }
         }
       }
       break;
@@ -107,16 +115,15 @@ namespace CarpetIOF5 {
       {
         if (Carpet::is_global_mode())
         {
-          for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH);
-               not reflevel_iter.done();
-               reflevel_iter.step())
+          BEGIN_REFLEVEL_LOOP (m_cctkGH)
           {
-            write_one_reflevel (simulation, have_metafile);
+            write_one_reflevel (timestep);
           }
+          END_REFLEVEL_LOOP;
         }
         else
         {
-          write_one_reflevel (simulation, have_metafile);
+          write_one_reflevel (timestep);
         }
       }
       break;
@@ -128,69 +135,96 @@ namespace CarpetIOF5 {
   
   
   void writer_t::
-  write_global (F5::simulation_t & simulation,
-                bool const have_metafile)
+  write_global (F5::timestep_t & timestep)
     const
   {
     DECLARE_CCTK_PARAMETERS;
     
     if (verbose or veryverbose)
     {
-      CCTK_INFO ("OutputVarAs/write_global");
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "OutputVarAs/write_global");
     }
     
-    F5::unigrid_topology_t topology (simulation);
-    
     int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
-    assert (grouptype >= 0);
     assert (grouptype == CCTK_SCALAR or grouptype == CCTK_ARRAY);
     
-    vect<CCTK_REAL, dim> level_origin, level_delta;
-    for (int d=0; d<dim; ++d)
-    {
-      level_origin[d] = 0.0;
-      level_delta[d]  = 1.0;
-    }
-    F5::Cartesian_coordinate_system_t coordinate_system
-      (topology, level_origin, level_delta);
-    
-    int const group = CCTK_GroupIndexFromVarI (m_variable);
-    assert (group >= 0 and group < CCTK_NumGroups());
-    F5::physical_quantity_t physical_quantity (coordinate_system, group);
-    
-    F5::tensor_component_t tensor_component (physical_quantity, m_variable);
-    
-    int const map = 0;
-    int const reflevel = 0;
-    int const myproc = CCTK_MyProc (m_cctkGH);
-    dh * const dd = Carpet::arrdata.at(group).at(map).dd;
-    dh::dboxes const & boxes
-      = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc);
-    bbox<int, dim> const & region = determine_region (boxes);
-    
-    if (have_metafile)
+    BEGIN_MAP_LOOP (m_cctkGH, grouptype)
     {
-      F5::meta_data_region_t meta_data_region (tensor_component, region);
-      gh * const hh = Carpet::vhh.at(Carpet::map);
-      int const proc = hh->processor (Carpet::reflevel, Carpet::component);
-      meta_data_region.write (proc);
+      
+      // Name the grid after the variable group
+      ostringstream namebuf;
+      char * const c_name = CCTK_GroupNameFromVarI (m_variable);
+      namebuf << "Cactus-" << c_name;
+      free (c_name);
+      string const name = namebuf.str();
+      F5::simulation_t simulation (timestep, name.c_str());
+      
+      F5::unigrid_topology_t topology (simulation);
+      
+      vect<CCTK_REAL, dim> const level_origin (0.0), level_delta (1.0);
+      F5::Cartesian_coordinate_system_t coordinate_system
+        (topology, level_origin, level_delta);
+      
+      int const group = CCTK_GroupIndexFromVarI (m_variable);
+      assert (group >= 0 and group < CCTK_NumGroups());
+      F5::physical_quantity_t physical_quantity (coordinate_system, group);
+      
+      int const myproc = CCTK_MyProc (m_cctkGH);
+      
+      F5::file_t & file = timestep.get_file();
+      bool const write_metafile
+        = file.get_is_metafile() and not file.get_is_datafile();
+      if (write_metafile)
+      {
+        
+        BEGIN_COMPONENT_LOOP (m_cctkGH, grouptype)
+        {
+          dh * const dd = Carpet::arrdata.at(group).at(Carpet::map).dd;
+          dh::dboxes const & boxes
+            = dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel).at(myproc);
+          bbox<int, dim> const & region = determine_region (boxes);
+          F5::meta_data_region_t meta_data_region (physical_quantity, region);
+          
+          gh * const hh = Carpet::vhh.at(Carpet::map);
+          int const proc = hh->processor (Carpet::reflevel, Carpet::component);
+          meta_data_region.write (proc);
+        }
+        END_COMPONENT_LOOP;
+        
+      }
+      else                      // if not write_metafile
+      {
+        
+        BEGIN_LOCAL_COMPONENT_LOOP (m_cctkGH, grouptype)
+        {
+          dh * const dd = Carpet::arrdata.at(group).at(Carpet::map).dd;
+          dh::dboxes const & boxes
+            = dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel).at(myproc);
+          bbox<int, dim> const & region = determine_region (boxes);
+          F5::data_region_t data_region (physical_quantity, region);
+          
+          F5::tensor_component_t tensor_component (data_region, m_variable);
+          int const timelevel = 0;
+          void const * const varptr
+            = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
+          assert (varptr != 0);
+          int const vartype = CCTK_VarTypeI (m_variable);
+          assert (vartype >= 0);
+          tensor_component.write (varptr, vartype);
+        }
+        END_LOCAL_COMPONENT_LOOP;
+        
+      } // if not write_metafile
+      
     }
-    
-    F5::data_region_t data_region (tensor_component, region);
-    int const timelevel = 0;
-    void const * const varptr
-      = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
-    assert (varptr != 0);
-    int const vartype = CCTK_VarTypeI (m_variable);
-    assert (vartype >= 0);
-    data_region.write (varptr, vartype);
+    END_MAP_LOOP;
   }
   
   
   
   void writer_t::
-  write_one_reflevel (F5::simulation_t & simulation,
-                      bool const have_metafile)
+  write_one_reflevel (F5::timestep_t & timestep)
     const
   {
     DECLARE_CCTK_PARAMETERS;
@@ -203,29 +237,26 @@ namespace CarpetIOF5 {
     }
     
     int const grouptype = CCTK_GroupTypeFromVarI (m_variable);
-    assert (grouptype >= 0);
     assert (grouptype == CCTK_GF);
     
     if (Carpet::is_level_mode())
     {
-      for (Carpet::map_iterator map_iter (m_cctkGH, grouptype);
-           not map_iter.done();
-           map_iter.step())
+      BEGIN_MAP_LOOP (m_cctkGH, grouptype)
       {
-        write_one_map (simulation, have_metafile);
+        write_one_map (timestep);
       }
+      END_MAP_LOOP;
     }
     else
     {
-      write_one_map (simulation, have_metafile);
+      write_one_map (timestep);
     }
   }
   
   
   
   void writer_t::
-  write_one_map (F5::simulation_t & simulation,
-                 bool const have_metafile)
+  write_one_map (F5::timestep_t & timestep)
     const
   {
     DECLARE_CCTK_PARAMETERS;
@@ -236,8 +267,19 @@ namespace CarpetIOF5 {
                   "OutputVarAs/write_one_map map=%d", Carpet::map);
     }
     
+    // Name the grid after the map number
+    ostringstream namebuf;
+    namebuf << "Carpet";
+    if (Carpet::maps > 1)
+    {
+      namebuf << "-map" << Carpet::map;
+    }
+    string const name = namebuf.str();
+    F5::simulation_t simulation (timestep, name.c_str());
+    
     F5::mesh_refinement_topology_t topology
-      (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels,
+      (simulation,
+       Carpet::map, Carpet::maps, Carpet::reflevel, Carpet::maxreflevels,
        Carpet::spacereflevelfact, Carpet::maxspacereflevelfact);
     
     vect<CCTK_REAL, dim> level_origin, level_delta;
@@ -255,31 +297,27 @@ namespace CarpetIOF5 {
     assert (group >= 0 and group < CCTK_NumGroups());
     F5::physical_quantity_t physical_quantity (coordinate_system, group);
     
-    F5::tensor_component_t tensor_component (physical_quantity, m_variable);
-    
     if (Carpet::is_singlemap_mode())
     {
       int const grouptype = CCTK_GroupTypeI (group);
       assert (grouptype >= 0);
       
-      for (Carpet::component_iterator component_iter (m_cctkGH, grouptype);
-           not component_iter.done();
-           component_iter.step())
+      BEGIN_COMPONENT_LOOP (m_cctkGH, grouptype)
       {
-        write_one_component (tensor_component, have_metafile);
+        write_one_component (physical_quantity);
       }
+      END_COMPONENT_LOOP;
     }
     else
     {
-      write_one_component (tensor_component, have_metafile);
+      write_one_component (physical_quantity);
     }
   }
   
   
   
   void writer_t::
-  write_one_component (F5::tensor_component_t & tensor_component,
-                       bool const have_metafile)
+  write_one_component (F5::physical_quantity_t & physical_quantity)
     const
   {
     DECLARE_CCTK_PARAMETERS;
@@ -293,30 +331,37 @@ namespace CarpetIOF5 {
     
     gh * const hh = Carpet::vhh.at(Carpet::map);
     bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component);
-    if (have_metafile or is_local)
+    
+    F5::file_t & file
+      = (physical_quantity.get_coordinate_system().get_topology()
+         .get_simulation().get_timestep().get_file());
+    bool const write_metafile
+      = file.get_is_metafile() and not file.get_is_datafile();
+    if (write_metafile or is_local)
     {
       dh * const dd = Carpet::vdd.at(Carpet::map);
       bbox<int, dim> const & region
         = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel)
            .at(Carpet::component).exterior);
       
-      if (have_metafile)
+      if (write_metafile)
       {
-        F5::meta_data_region_t meta_data_region (tensor_component, region);
+        F5::meta_data_region_t meta_data_region (physical_quantity, region);
         int const proc = hh->processor (Carpet::reflevel, Carpet::component);
         meta_data_region.write (proc);
       }
-      
-      if (is_local)
+      else if (is_local)
       {
-        F5::data_region_t data_region (tensor_component, region);
+        F5::data_region_t data_region (physical_quantity, region);
+        
+        F5::tensor_component_t tensor_component (data_region, m_variable);
         int const timelevel = 0;
         void const * const varptr
           = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable);
         assert (varptr != 0);
         int const vartype = CCTK_VarTypeI (m_variable);
         assert (vartype >= 0);
-        data_region.write (varptr, vartype);
+        tensor_component.write (varptr, vartype);
       }
     }
   }
@@ -325,19 +370,17 @@ namespace CarpetIOF5 {
   
   bbox<int,dim> const & writer_t::
   determine_region (dh::dboxes const & boxes)
-    const
   {
     DECLARE_CCTK_PARAMETERS;
     
+    // TODO: use superregions instead of regions (?  only if the
+    // regions are on the same processor?)
+    
     bbox<int,dim> dh::dboxes::* boxptr;
     if (CCTK_EQUALS (output_regions, "exterior"))
     {
       boxptr = & dh::dboxes::exterior;
     }
-    else if (CCTK_EQUALS (output_regions, "communicated"))
-    {
-      boxptr = & dh::dboxes::communicated;
-    }
     else if (CCTK_EQUALS (output_regions, "owned"))
     {
       boxptr = & dh::dboxes::owned;
diff --git a/CarpetDev/CarpetIOF5/src/writer.hh b/CarpetDev/CarpetIOF5/src/writer.hh
index 94b31d3cd..77bad3915 100644
--- a/CarpetDev/CarpetIOF5/src/writer.hh
+++ b/CarpetDev/CarpetIOF5/src/writer.hh
@@ -6,10 +6,11 @@
 #include "bbox.hh"
 #include "defs.hh"
 #include "dh.hh"
+#include "vect.hh"
 
 #include "file.hh"
+#include "physical_quantity.hh"
 #include "simulation.hh"
-#include "tensor_component.hh"
 #include "timestep.hh"
 #include "topology.hh"
 
@@ -34,38 +35,33 @@ namespace CarpetIOF5 {
   private:
     
     void
-    write_meta (F5::file_t & file,
-                bool have_metafile)
+    write_meta (F5::file_t & file)
       const;
     
     void
-    write_one_mglevel (F5::timestep_t & timestep,
-                       bool have_metafile)
+    write_one_mglevel (F5::timestep_t & timestep)
       const;
     
     void
-    write_global (F5::simulation_t & simulation,
-                  bool have_metafile)
+    write_global (F5::timestep_t & timestep)
       const;
     
     void
-    write_one_reflevel (F5::simulation_t & simulation,
-                        bool have_metafile)
+    write_one_reflevel (F5::timestep_t & timestep)
       const;
     
     void
-    write_one_map (F5::simulation_t & simulation,
-                   bool have_metafile)
+    write_one_map (F5::timestep_t & timestep)
       const;
     
     void
-    write_one_component (F5::tensor_component_t & tensor_component,
-                         bool have_metafile)
+    write_one_component (F5::physical_quantity_t & physical_quantity)
       const;
     
-    bbox<int,dim> const &
-    determine_region (dh::dboxes const & boxes)
-      const;
+    
+    
+    static bbox<int,dim> const &
+    determine_region (dh::dboxes const & boxes);
     
   };