From 11c4d98017cbb86d08e15fd1b549180184b58a26 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 3 Sep 2009 16:19:15 -0500 Subject: Import Carpet Ignore-this: 309b4dd613f4af2b84aa5d6743fdb6b3 --- CarpetDev/CarpetIOF5/README | 26 +- CarpetDev/CarpetIOF5/configuration.ccl | 2 +- CarpetDev/CarpetIOF5/interface.ccl | 22 +- CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par | 244 ++++++++++++ CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par | 230 ++++++++++++ CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par | 244 ++++++++++++ CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par | 245 ++++++++++++ CarpetDev/CarpetIOF5/param.ccl | 40 +- CarpetDev/CarpetIOF5/src/IOF5.cc | 181 +++++++-- CarpetDev/CarpetIOF5/src/coordinate_system.cc | 29 +- CarpetDev/CarpetIOF5/src/coordinate_system.hh | 3 +- CarpetDev/CarpetIOF5/src/data_region.cc | 121 ++---- CarpetDev/CarpetIOF5/src/data_region.hh | 33 +- CarpetDev/CarpetIOF5/src/extending.cc | 1 - CarpetDev/CarpetIOF5/src/f5writer.cc | 445 ++++++++++++++++++++++ CarpetDev/CarpetIOF5/src/f5writer.hh | 74 ++++ CarpetDev/CarpetIOF5/src/file.cc | 292 ++++++++------- CarpetDev/CarpetIOF5/src/file.hh | 61 ++- CarpetDev/CarpetIOF5/src/make.code.defn | 18 +- CarpetDev/CarpetIOF5/src/meta_data_region.cc | 70 ++-- CarpetDev/CarpetIOF5/src/meta_data_region.hh | 15 +- CarpetDev/CarpetIOF5/src/physical_quantity.cc | 30 +- CarpetDev/CarpetIOF5/src/physical_quantity.hh | 7 +- CarpetDev/CarpetIOF5/src/simulation.cc | 28 +- CarpetDev/CarpetIOF5/src/simulation.hh | 3 +- CarpetDev/CarpetIOF5/src/tensor_component.cc | 136 +++++-- CarpetDev/CarpetIOF5/src/tensor_component.hh | 25 +- CarpetDev/CarpetIOF5/src/timestep.cc | 35 +- CarpetDev/CarpetIOF5/src/timestep.hh | 3 +- CarpetDev/CarpetIOF5/src/topology.cc | 32 +- CarpetDev/CarpetIOF5/src/topology.hh | 4 +- CarpetDev/CarpetIOF5/src/utils.cc | 84 ++++- CarpetDev/CarpetIOF5/src/utils.hh | 28 +- CarpetDev/CarpetIOF5/src/writer-old.cc | 511 ++++++++++++++++++++++++++ CarpetDev/CarpetIOF5/src/writer.cc | 245 +++++++----- CarpetDev/CarpetIOF5/src/writer.hh | 28 +- 36 files changed, 2966 insertions(+), 629 deletions(-) create mode 100644 CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par create mode 100644 CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par create mode 100644 CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par create mode 100644 CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par create mode 100644 CarpetDev/CarpetIOF5/src/f5writer.cc create mode 100644 CarpetDev/CarpetIOF5/src/f5writer.hh create mode 100644 CarpetDev/CarpetIOF5/src/writer-old.cc (limited to 'CarpetDev') diff --git a/CarpetDev/CarpetIOF5/README b/CarpetDev/CarpetIOF5/README index a03d4b2f5..a625117a0 100644 --- a/CarpetDev/CarpetIOF5/README +++ b/CarpetDev/CarpetIOF5/README @@ -11,16 +11,20 @@ Input and output data using the HDF5 based F5 file format. An F5 file contains the following hierarchy: - file - timestep - simulation - topology - coordinate_system - physical_quantity - tensor_component - data_region + Bundle + Slice + Grid + Topology + Representation + Field -2006-06-05: Werner says that tensor_component and data_region should -be interchanged. +We have a class hierarchy: -2007-04-25: Werner confirms this. + file + timestep (slice) + simulation (grid) [grid hierarchy of one patch] + topology (topology) [one refinement level] + coordinate_system (representation) + physical_quantity (field) + data_region + tensor_component diff --git a/CarpetDev/CarpetIOF5/configuration.ccl b/CarpetDev/CarpetIOF5/configuration.ccl index 1ab5bc27d..d5c33c995 100644 --- a/CarpetDev/CarpetIOF5/configuration.ccl +++ b/CarpetDev/CarpetIOF5/configuration.ccl @@ -1,3 +1,3 @@ # Configuration definitions for thorn CarpetIOF5 -REQUIRES Carpet CarpetLib HDF5 IOUtil +REQUIRES Carpet CarpetLib F5 HDF5 IOUtil diff --git a/CarpetDev/CarpetIOF5/interface.ccl b/CarpetDev/CarpetIOF5/interface.ccl index db9d9c014..aa7a39fee 100644 --- a/CarpetDev/CarpetIOF5/interface.ccl +++ b/CarpetDev/CarpetIOF5/interface.ccl @@ -3,7 +3,7 @@ IMPLEMENTS: IOF5 USES INCLUDE: bbox.hh defs.hh dh.hh vect.hh -USES INCLUDE: carpet.hh +USES INCLUDE: carpet.hh modes.hh @@ -25,8 +25,26 @@ REQUIRES FUNCTION IO_TruncateOutputFiles +# Return a pointer to an unmodifiable C string +# which contains a unique ID for this configuration +CCTK_POINTER_TO_CONST \ +FUNCTION UniqueConfigID (CCTK_POINTER_TO_CONST IN cctkGH) +USES FUNCTION UniqueConfigID + +# Return a pointer to an unmodifiable C string +# which contains a unique ID for this build +CCTK_POINTER_TO_CONST \ +FUNCTION UniqueBuildID (CCTK_POINTER_TO_CONST IN cctkGH) +USES FUNCTION UniqueBuildID + # Return a pointer to an unmodifiable C string # which contains a unique ID for this simulation CCTK_POINTER_TO_CONST \ -FUNCTION UniqueSimulationID (CCTK_POINTER_TO_CONST IN cctkGH) +FUNCTION UniqueSimulationID (CCTK_POINTER_TO_CONST IN cctkGH) USES FUNCTION UniqueSimulationID + +# Return a pointer to an unmodifiable C string +# which contains a unique ID for this run +CCTK_POINTER_TO_CONST \ +FUNCTION UniqueRunID (CCTK_POINTER_TO_CONST IN cctkGH) +USES FUNCTION UniqueRunID diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par b/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par new file mode 100644 index 000000000..1e8c2783d --- /dev/null +++ b/CarpetDev/CarpetIOF5/par/wavetoy-cc-ref.par @@ -0,0 +1,244 @@ +Cactus::cctk_run_title = "WaveToy cell-centred refined" + +Cactus::cctk_full_warnings = yes +Cactus::highlight_warning_messages = no +#Cactus::cctk_timer_output = "full" + +Cactus::terminate = "time" +Cactus::cctk_final_time = 1.0 + + + +ActiveThorns = "IOUtil" + +IO::out_dir = $parfile + + + +ActiveThorns = "InitBase" + +InitBase::initial_data_setup_method = "init_single_level" + + + +ActiveThorns = "AEILocalInterp" + +ActiveThorns = "Fortran" + +ActiveThorns = "HDF5" + +ActiveThorns = "LocalInterp" + +ActiveThorns = "LoopControl" + +ActiveThorns = "NaNCatcher" + +ActiveThorns = "Slab" + + + +ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab" + +Carpet::verbose = no +Carpet::schedule_barriers = no +Carpet::veryverbose = no +CarpetLib::output_bboxes = no + +Carpet::domain_from_coordbase = yes +Carpet::max_refinement_levels = 3 + +driver::ghost_size = 2 + +Carpet::refinement_centering = "cell" +Carpet::prolongation_order_space = 2 +Carpet::prolongation_order_time = 2 + +Carpet::convergence_level = 0 + +Carpet::init_each_timelevel = yes +Carpet::regrid_during_initialisation = no +Carpet::enable_all_storage = no + +Carpet::poison_new_timelevels = yes +CarpetLib::poison_new_memory = yes + +Carpet::grid_structure_filename = "carpet-grid-structure" +Carpet::output_timers_every = 100 +CarpetLib::print_timestats_every = 100 +CarpetLib::print_memstats_every = 100 + + + +ActiveThorns = "NaNChecker" + +NaNChecker::check_every = 100 +NaNChecker::action_if_found = "terminate" +NaNChecker::check_vars = " + WaveToy::scalarevolve +" + + + +ActiveThorns = "Boundary CartGrid3D CoordBase SymBase" + +CoordBase::domainsize = "minmax" + +CoordBase::xmin = -0.50 +CoordBase::ymin = -0.50 +CoordBase::zmin = -0.50 +CoordBase::xmax = +0.50 +CoordBase::ymax = +0.50 +CoordBase::zmax = +0.50 +CoordBase::dx = 0.05 +CoordBase::dy = 0.05 +CoordBase::dz = 0.05 + +CoordBase::boundary_staggered_x_lower = yes +CoordBase::boundary_staggered_y_lower = yes +CoordBase::boundary_staggered_z_lower = yes +CoordBase::boundary_staggered_x_upper = yes +CoordBase::boundary_staggered_y_upper = yes +CoordBase::boundary_staggered_z_upper = yes + +CartGrid3D::type = "coordbase" + + + +ActiveThorns = "CarpetRegrid2" + +CarpetRegrid2::regrid_every = 0 + +CarpetRegrid2::snap_to_coarse = yes + +CarpetRegrid2::num_centres = 1 + +CarpetRegrid2::num_levels_1 = 3 +CarpetRegrid2::radius_1[1] = 0.25 +CarpetRegrid2::radius_1[2] = 0.125 + + + +ActiveThorns = "WavetoyC" + +WaveToy::bound = "radiation" + + + +ActiveThorns = "IDScalarWaveC" + +IDScalarWave::initial_data = "gaussian" +IDScalarWave::radius = 0.0 +IDScalarWave::sigma = 0.1 +IDScalarWave::amplitude = 1.0 + + + +ActiveThorns = "Time" + +Time::dtfac = 0.5 + + + +ActiveThorns = "CarpetIOBasic" + +IOBasic::outInfo_every = 1 +IOBasic::outInfo_reductions = "norm2 minimum maximum" +IOBasic::outInfo_vars = " + WaveToy::phi +" + + + +ActiveThorns = "CarpetIOScalar" + +IOScalar::one_file_per_group = yes + +IOScalar::outScalar_every = 1 +IOScalar::outScalar_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +ActiveThorns = "CarpetIOASCII" + +IOASCII::one_file_per_group = yes + +#IOASCII::output_symmetry_points = no +#IOASCII::out3D_ghosts = no + +IOASCII::out0D_every = 1 +IOASCII::out0D_vars = " + Carpet::timing + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out1D_every = 1 +IOASCII::out1D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out2D_every = 10 +IOASCII::out2D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out3D_every = 100 +IOASCII::out3D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +Activethorns = "CarpetIOHDF5" + +IOHDF5::out_every = 100 +IOHDF5::compression_level = 1 +IOHDF5::out_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOHDF5::checkpoint = yes +IO::checkpoint_dir = $parfile +IO::checkpoint_ID = yes +IO::checkpoint_every = 100 +IO::checkpoint_on_terminate = yes + +Carpet::regrid_during_recovery = no +IOHDF5::use_grid_structure_from_checkpoint = yes + +IO::recover = "autoprobe" +IO::recover_dir = $parfile + + + +ActiveThorns = "Formaline" + +Formaline::send_as_rdf = yes +Formaline::rdf_hostname[0] = "buran.aei.mpg.de" +Formaline::rdf_port [0] = 24997 +Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu" +Formaline::rdf_port [1] = 8080 + + + +ActiveThorns = "Twitter" + + + +ActiveThorns = "TimerReport" + +TimerReport::out_every = 100 +TimerReport::out_filename = "TimerReport" diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par b/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par new file mode 100644 index 000000000..d5265e3c8 --- /dev/null +++ b/CarpetDev/CarpetIOF5/par/wavetoy-cc-uni.par @@ -0,0 +1,230 @@ +Cactus::cctk_run_title = "WaveToy cell-centred unigrid" + +Cactus::cctk_full_warnings = yes +Cactus::highlight_warning_messages = no +#Cactus::cctk_timer_output = "full" + +Cactus::terminate = "time" +Cactus::cctk_final_time = 1.0 + + + +ActiveThorns = "IOUtil" + +IO::out_dir = $parfile + + + +ActiveThorns = "InitBase" + +InitBase::initial_data_setup_method = "init_single_level" + + + +ActiveThorns = "AEILocalInterp" + +ActiveThorns = "Fortran" + +ActiveThorns = "HDF5" + +ActiveThorns = "LocalInterp" + +ActiveThorns = "LoopControl" + +ActiveThorns = "NaNCatcher" + +ActiveThorns = "Slab" + + + +ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab" + +Carpet::verbose = no +Carpet::schedule_barriers = no +Carpet::veryverbose = no +CarpetLib::output_bboxes = no + +Carpet::domain_from_coordbase = yes +Carpet::max_refinement_levels = 3 + +driver::ghost_size = 2 + +Carpet::refinement_centering = "cell" +Carpet::prolongation_order_space = 3 +Carpet::prolongation_order_time = 2 + +Carpet::convergence_level = 0 + +Carpet::init_each_timelevel = yes +Carpet::regrid_during_initialisation = no +Carpet::enable_all_storage = no + +Carpet::poison_new_timelevels = yes +CarpetLib::poison_new_memory = yes + +Carpet::grid_structure_filename = "carpet-grid-structure" +Carpet::output_timers_every = 100 +CarpetLib::print_timestats_every = 100 +CarpetLib::print_memstats_every = 100 + + + +ActiveThorns = "NaNChecker" + +NaNChecker::check_every = 100 +NaNChecker::action_if_found = "terminate" +NaNChecker::check_vars = " + WaveToy::scalarevolve +" + + + +ActiveThorns = "Boundary CartGrid3D CoordBase SymBase" + +CoordBase::domainsize = "minmax" + +CoordBase::xmin = -0.50 +CoordBase::ymin = -0.50 +CoordBase::zmin = -0.50 +CoordBase::xmax = +0.50 +CoordBase::ymax = +0.50 +CoordBase::zmax = +0.50 +CoordBase::dx = 0.0125 +CoordBase::dy = 0.0125 +CoordBase::dz = 0.0125 + +CoordBase::boundary_staggered_x_lower = yes +CoordBase::boundary_staggered_y_lower = yes +CoordBase::boundary_staggered_z_lower = yes +CoordBase::boundary_staggered_x_upper = yes +CoordBase::boundary_staggered_y_upper = yes +CoordBase::boundary_staggered_z_upper = yes + +CartGrid3D::type = "coordbase" + + + +ActiveThorns = "WavetoyC" + +WaveToy::bound = "radiation" + + + +ActiveThorns = "IDScalarWaveC" + +IDScalarWave::initial_data = "gaussian" +IDScalarWave::radius = 0.0 +IDScalarWave::sigma = 0.1 +IDScalarWave::amplitude = 1.0 + + + +ActiveThorns = "Time" + +Time::dtfac = 0.5 + + + +ActiveThorns = "CarpetIOBasic" + +IOBasic::outInfo_every = 1 +IOBasic::outInfo_reductions = "norm2 minimum maximum" +IOBasic::outInfo_vars = " + WaveToy::phi +" + + + +ActiveThorns = "CarpetIOScalar" + +IOScalar::one_file_per_group = yes + +IOScalar::outScalar_every = 1 +IOScalar::outScalar_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +ActiveThorns = "CarpetIOASCII" + +IOASCII::one_file_per_group = yes + +IOASCII::output_symmetry_points = no +IOASCII::out3D_ghosts = no + +IOASCII::out0D_every = 1 +IOASCII::out0D_vars = " + Carpet::timing + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out1D_every = 1 +IOASCII::out1D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out2D_every = 10 +IOASCII::out2D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out3D_every = 100 +IOASCII::out3D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +Activethorns = "CarpetIOHDF5" + +IOHDF5::out_every = 100 +IOHDF5::compression_level = 1 +IOHDF5::out_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOHDF5::checkpoint = yes +IO::checkpoint_dir = $parfile +IO::checkpoint_ID = yes +IO::checkpoint_every = 100 +IO::checkpoint_on_terminate = yes + +Carpet::regrid_during_recovery = no +IOHDF5::use_grid_structure_from_checkpoint = yes + +IO::recover = "autoprobe" +IO::recover_dir = $parfile + + + +ActiveThorns = "Formaline" + +Formaline::send_as_rdf = yes +Formaline::rdf_hostname[0] = "buran.aei.mpg.de" +Formaline::rdf_port [0] = 24997 +Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu" +Formaline::rdf_port [1] = 8080 + + + +ActiveThorns = "Twitter" + + + +ActiveThorns = "TimerReport" + +TimerReport::out_every = 100 +TimerReport::out_filename = "TimerReport" diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par b/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par new file mode 100644 index 000000000..46ccdd6b2 --- /dev/null +++ b/CarpetDev/CarpetIOF5/par/wavetoy-vc-ref.par @@ -0,0 +1,244 @@ +Cactus::cctk_run_title = "WaveToy vertex-centred refined" + +Cactus::cctk_full_warnings = yes +Cactus::highlight_warning_messages = no +#Cactus::cctk_timer_output = "full" + +Cactus::terminate = "time" +Cactus::cctk_final_time = 1.0 + + + +ActiveThorns = "IOUtil" + +IO::out_dir = $parfile + + + +ActiveThorns = "InitBase" + +InitBase::initial_data_setup_method = "init_single_level" + + + +ActiveThorns = "AEILocalInterp" + +ActiveThorns = "Fortran" + +ActiveThorns = "HDF5" + +ActiveThorns = "LocalInterp" + +ActiveThorns = "LoopControl" + +ActiveThorns = "NaNCatcher" + +ActiveThorns = "Slab" + + + +ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab" + +Carpet::verbose = no +Carpet::schedule_barriers = no +Carpet::veryverbose = no +CarpetLib::output_bboxes = no + +Carpet::domain_from_coordbase = yes +Carpet::max_refinement_levels = 3 + +driver::ghost_size = 2 + +Carpet::refinement_centering = "vertex" +Carpet::prolongation_order_space = 3 +Carpet::prolongation_order_time = 2 + +Carpet::convergence_level = 0 + +Carpet::init_each_timelevel = yes +Carpet::regrid_during_initialisation = no +Carpet::enable_all_storage = no + +Carpet::poison_new_timelevels = yes +CarpetLib::poison_new_memory = yes + +Carpet::grid_structure_filename = "carpet-grid-structure" +Carpet::output_timers_every = 100 +CarpetLib::print_timestats_every = 100 +CarpetLib::print_memstats_every = 100 + + + +ActiveThorns = "NaNChecker" + +NaNChecker::check_every = 100 +NaNChecker::action_if_found = "terminate" +NaNChecker::check_vars = " + WaveToy::scalarevolve +" + + + +ActiveThorns = "Boundary CartGrid3D CoordBase SymBase" + +CoordBase::domainsize = "minmax" + +CoordBase::xmin = -0.50 +CoordBase::ymin = -0.50 +CoordBase::zmin = -0.50 +CoordBase::xmax = +0.50 +CoordBase::ymax = +0.50 +CoordBase::zmax = +0.50 +CoordBase::dx = 0.05 +CoordBase::dy = 0.05 +CoordBase::dz = 0.05 + +CoordBase::boundary_staggered_x_lower = no +CoordBase::boundary_staggered_y_lower = no +CoordBase::boundary_staggered_z_lower = no +CoordBase::boundary_staggered_x_upper = no +CoordBase::boundary_staggered_y_upper = no +CoordBase::boundary_staggered_z_upper = no + +CartGrid3D::type = "coordbase" + + + +ActiveThorns = "CarpetRegrid2" + +CarpetRegrid2::regrid_every = 0 + +CarpetRegrid2::snap_to_coarse = yes + +CarpetRegrid2::num_centres = 1 + +CarpetRegrid2::num_levels_1 = 3 +CarpetRegrid2::radius_1[1] = 0.25 +CarpetRegrid2::radius_1[2] = 0.125 + + + +ActiveThorns = "WavetoyC" + +WaveToy::bound = "radiation" + + + +ActiveThorns = "IDScalarWaveC" + +IDScalarWave::initial_data = "gaussian" +IDScalarWave::radius = 0.0 +IDScalarWave::sigma = 0.1 +IDScalarWave::amplitude = 1.0 + + + +ActiveThorns = "Time" + +Time::dtfac = 0.5 + + + +ActiveThorns = "CarpetIOBasic" + +IOBasic::outInfo_every = 1 +IOBasic::outInfo_reductions = "norm2 minimum maximum" +IOBasic::outInfo_vars = " + WaveToy::phi +" + + + +ActiveThorns = "CarpetIOScalar" + +IOScalar::one_file_per_group = yes + +IOScalar::outScalar_every = 1 +IOScalar::outScalar_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +ActiveThorns = "CarpetIOASCII" + +IOASCII::one_file_per_group = yes + +#IOASCII::output_symmetry_points = no +#IOASCII::out3D_ghosts = no + +IOASCII::out0D_every = 1 +IOASCII::out0D_vars = " + Carpet::timing + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out1D_every = 1 +IOASCII::out1D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out2D_every = 10 +IOASCII::out2D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOASCII::out3D_every = 100 +IOASCII::out3D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + + +Activethorns = "CarpetIOHDF5" + +IOHDF5::out_every = 100 +IOHDF5::compression_level = 1 +IOHDF5::out_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +IOHDF5::checkpoint = yes +IO::checkpoint_dir = $parfile +IO::checkpoint_ID = yes +IO::checkpoint_every = 100 +IO::checkpoint_on_terminate = yes + +Carpet::regrid_during_recovery = no +IOHDF5::use_grid_structure_from_checkpoint = yes + +IO::recover = "autoprobe" +IO::recover_dir = $parfile + + + +ActiveThorns = "Formaline" + +Formaline::send_as_rdf = yes +Formaline::rdf_hostname[0] = "buran.aei.mpg.de" +Formaline::rdf_port [0] = 24997 +Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu" +Formaline::rdf_port [1] = 8080 + + + +ActiveThorns = "Twitter" + + + +ActiveThorns = "TimerReport" + +TimerReport::out_every = 100 +TimerReport::out_filename = "TimerReport" diff --git a/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par b/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par new file mode 100644 index 000000000..e2339f342 --- /dev/null +++ b/CarpetDev/CarpetIOF5/par/wavetoy-vc-uni.par @@ -0,0 +1,245 @@ +Cactus::cctk_run_title = "WaveToy vertex-centred unigrid" + +Cactus::cctk_full_warnings = yes +Cactus::highlight_warning_messages = no + +Cactus::terminate = "time" +Cactus::cctk_final_time = 1.0 + + + +ActiveThorns = "IOUtil" + +IO::out_dir = $parfile + + + +ActiveThorns = "InitBase" + +InitBase::initial_data_setup_method = "init_single_level" + + + +ActiveThorns = "AEILocalInterp" + +ActiveThorns = "Fortran" + +ActiveThorns = "HDF5" + +ActiveThorns = "LocalInterp" + +ActiveThorns = "LoopControl" + +ActiveThorns = "NaNCatcher" + +ActiveThorns = "Slab" + + + +ActiveThorns = "Carpet CarpetLib CarpetInterp CarpetReduce CarpetSlab" + +Carpet::verbose = no +Carpet::schedule_barriers = no +Carpet::veryverbose = no +CarpetLib::output_bboxes = no + +Carpet::domain_from_coordbase = yes +Carpet::max_refinement_levels = 3 + +driver::ghost_size = 2 + +Carpet::refinement_centering = "vertex" +Carpet::prolongation_order_space = 3 +Carpet::prolongation_order_time = 2 + +Carpet::convergence_level = 0 + +Carpet::init_each_timelevel = yes +Carpet::regrid_during_initialisation = no +Carpet::enable_all_storage = no + +Carpet::poison_new_timelevels = yes +CarpetLib::poison_new_memory = yes + +Carpet::grid_structure_filename = "carpet-grid-structure" +Carpet::output_timers_every = 100 +CarpetLib::print_timestats_every = 100 +CarpetLib::print_memstats_every = 100 + + + +ActiveThorns = "NaNChecker" + +NaNChecker::check_every = 100 +NaNChecker::action_if_found = "terminate" +NaNChecker::check_vars = " + WaveToy::scalarevolve +" + + + +ActiveThorns = "Boundary CartGrid3D CoordBase SymBase" + +CoordBase::domainsize = "minmax" + +CoordBase::xmin = -0.50 +CoordBase::ymin = -0.50 +CoordBase::zmin = -0.50 +CoordBase::xmax = +0.50 +CoordBase::ymax = +0.50 +CoordBase::zmax = +0.50 +CoordBase::dx = 0.0125 +CoordBase::dy = 0.0125 +CoordBase::dz = 0.0125 + +CoordBase::boundary_staggered_x_lower = no +CoordBase::boundary_staggered_y_lower = no +CoordBase::boundary_staggered_z_lower = no +CoordBase::boundary_staggered_x_upper = no +CoordBase::boundary_staggered_y_upper = no +CoordBase::boundary_staggered_z_upper = no + +CartGrid3D::type = "coordbase" + + + +ActiveThorns = "WavetoyC" + +WaveToy::bound = "radiation" + + + +ActiveThorns = "IDScalarWaveC" + +IDScalarWave::initial_data = "gaussian" +IDScalarWave::radius = 0.0 +IDScalarWave::sigma = 0.1 +IDScalarWave::amplitude = 1.0 + + + +ActiveThorns = "Time" + +Time::dtfac = 0.5 + + + +ActiveThorns = "CarpetIOBasic" + +IOBasic::outInfo_every = 1 +IOBasic::outInfo_reductions = "norm2 minimum maximum" +IOBasic::outInfo_vars = " + WaveToy::phi +" + + + +#ActiveThorns = "CarpetIOScalar" +# +#IOScalar::one_file_per_group = yes +# +#IOScalar::outScalar_every = 1 +#IOScalar::outScalar_vars = " +# grid::coordinates +# CarpetReduce::weight +# WaveToy::scalarevolve +#" + + + +ActiveThorns = "CarpetIOASCII" + +IOASCII::one_file_per_group = yes + +IOASCII::output_symmetry_points = yes # no +IOASCII::out3D_ghosts = yes # no + +#IOASCII::out0D_every = 1 +#IOASCII::out0D_vars = " +# Carpet::timing +# grid::coordinates +# CarpetReduce::weight +# WaveToy::scalarevolve +#" + +IOASCII::out1D_every = 1 +IOASCII::out1D_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +#IOASCII::out2D_every = 10 +#IOASCII::out2D_vars = " +# grid::coordinates +# CarpetReduce::weight +# WaveToy::scalarevolve +#" +# +#IOASCII::out3D_every = 100 +#IOASCII::out3D_vars = " +# grid::coordinates +# CarpetReduce::weight +# WaveToy::scalarevolve +#" + + + +Activethorns = "CarpetIOHDF5" + +IOHDF5::out_every = 64 +IOHDF5::compression_level = 1 +IOHDF5::out_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + +#IOHDF5::checkpoint = yes +#IO::checkpoint_dir = $parfile +#IO::checkpoint_ID = yes +#IO::checkpoint_every = 100 +#IO::checkpoint_on_terminate = yes +# +#Carpet::regrid_during_recovery = no +#IOHDF5::use_grid_structure_from_checkpoint = yes +# +#IO::recover = "autoprobe" +#IO::recover_dir = $parfile + + + +Activethorns = "CarpetIOF5" + +IOF5::out_every = 64 +IOF5::chunk_size_x = 8 +IOF5::chunk_size_y = 8 +IOF5::chunk_size_z = 8 +IOF5::compression_level = 1 +IOF5::write_checksum = yes +IOF5::out_vars = " + grid::coordinates + CarpetReduce::weight + WaveToy::scalarevolve +" + + +ActiveThorns = "Formaline" + +Formaline::send_as_rdf = yes +Formaline::rdf_hostname[0] = "buran.aei.mpg.de" +Formaline::rdf_port [0] = 24997 +Formaline::rdf_hostname[1] = "devportal.cct.lsu.edu" +Formaline::rdf_port [1] = 8080 + + + +ActiveThorns = "Twitter" + + + +ActiveThorns = "TimerReport" + +TimerReport::out_every = 100 +TimerReport::output_all_timers_together = yes +TimerReport::out_filename = "TimerReport" diff --git a/CarpetDev/CarpetIOF5/param.ccl b/CarpetDev/CarpetIOF5/param.ccl index 45b0eeca1..f9d1249bd 100644 --- a/CarpetDev/CarpetIOF5/param.ccl +++ b/CarpetDev/CarpetIOF5/param.ccl @@ -62,6 +62,15 @@ STRING out_extension "File name extension" STEERABLE=always "" :: "File extension (including a leading dot, if desired)" } ".f5" +BOOLEAN create_subdirs "Create subdirectories for the output files to reduce the number of files per directory" STEERABLE=always +{ +} "no" + +BOOLEAN one_dir_per_file "Create one subdirectory per output file to reduce locking overhead" STEERABLE=always +{ +} "no" + + STRING out_vars "Variables and groups which should be output in the F5 file format" STEERABLE=always @@ -101,17 +110,44 @@ REAL dt_fudge "Fudge factor for time comparisons" STEERABLE=always +INT chunk_size_x "Chunk size for chunked datasets" STEERABLE=always +{ + -1 :: "no chunking (use actual dataset size)" + 1:* :: "desired chunk size" +} -1 + +INT chunk_size_y "Chunk size for chunked datasets" STEERABLE=always +{ + -1 :: "no chunking (use actual dataset size)" + 1:* :: "desired chunk size" +} -1 + +INT chunk_size_z "Chunk size for chunked datasets" STEERABLE=always +{ + -1 :: "no chunking (use actual dataset size)" + 1:* :: "desired chunk size" +} -1 + INT compression_level "Compression level to use for writing HDF5 data" STEERABLE=always { - 0:9 :: "Higher numbers compress better, a value of zero disables compression" + 0 :: "no compression" + 1:9 :: "higher numbers compress better" } 0 +BOOLEAN write_checksum "Add a checksum when writing HDF5 files" STEERABLE=always +{ +} "no" + +BOOLEAN check_checksum "Check checksum (if present) when reading HDF5 files" STEERABLE=always +{ +} "no" + KEYWORD output_regions "Regions which should be output" STEERABLE=always { "exterior" :: "whole region (including boundaries)" - "communicated" :: "exterior without outer boundary" + #"communicated" :: "exterior without outer boundary" "owned" :: "evolved in time" # "active" :: "owned minus buffers" "interior" :: "interior (without ghost zones)" diff --git a/CarpetDev/CarpetIOF5/src/IOF5.cc b/CarpetDev/CarpetIOF5/src/IOF5.cc index 94e1089f3..1173f5228 100644 --- a/CarpetDev/CarpetIOF5/src/IOF5.cc +++ b/CarpetDev/CarpetIOF5/src/IOF5.cc @@ -9,12 +9,22 @@ #include #include +// force HDF5 1.8.x installations to use the new API +#define H5Dcreate_vers 2 + +#include + #include "cctk.h" #include "cctk_Arguments.h" #include "cctk_Parameters.h" +#include "CactusBase/IOUtil/src/ioGH.h" +#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h" + #include "carpet.hh" +#include "defs.hh" + #include "extending.hh" #include "file.hh" #include "utils.hh" @@ -377,36 +387,93 @@ namespace CarpetIOF5 { extending_t extending (cctkGH); bool const use_IO_out_dir = strcmp (out_dir, "") == 0; - string const basename - = (use_IO_out_dir ? IO_out_dir : out_dir) + string ("/") + alias; + string const path = use_IO_out_dir ? IO_out_dir : out_dir; + string const basename = alias; bool const did_truncate = extending.get_did_truncate (basename); bool const do_truncate = not did_truncate and IO_TruncateOutputFiles (cctkGH); extending.set_did_truncate (basename); - F5::file_t file (cctkGH, basename, string (out_extension), do_truncate); + int const proc = CCTK_MyProc (cctkGH); + bool have_metafile; // whether there is a metadata file + int metadata_processor; // the processor which outputs the metadata file + int output_processor; // the processor which outputs our data + if (CCTK_EQUALS (out_mode, "proc")) + { + have_metafile = true; + metadata_processor = 0; + output_processor = proc; + } + else if (CCTK_EQUALS (out_mode, "np")) + { + have_metafile = true; + metadata_processor = 0; + output_processor = proc / out_proc_every * out_proc_every; + } + else if (CCTK_EQUALS (out_mode, "onefile")) + { + have_metafile = false; + metadata_processor = 0; + output_processor = 0; + } + else + { + assert (0); + } + + F5::file_t * metafile = NULL; + if (have_metafile and proc == metadata_processor) + { + metafile + = new F5::file_t (cctkGH, path, basename, string (out_extension), + do_truncate, true, false); + } + + F5::file_t * file = NULL; + if (proc == output_processor) + { + file = new F5::file_t (cctkGH, path, basename, string (out_extension), + do_truncate, not have_metafile, true); + } if (do_truncate) { // Output parameters once after the output file has been created - if (CCTK_EQUALS (out_save_parameters, "all") or - CCTK_EQUALS (out_save_parameters, "only set")) - { - WriteParameters (file); - } - else if (CCTK_EQUALS (out_save_parameters, "no")) + if (proc == metadata_processor) { - // do nothing - } - else - { - assert (0); + if (CCTK_EQUALS (out_save_parameters, "all") or + CCTK_EQUALS (out_save_parameters, "only set")) + { + WriteParameters (have_metafile ? * metafile : * file); + } + else if (CCTK_EQUALS (out_save_parameters, "no")) + { + // do nothing + } + else + { + assert (0); + } } } - writer_t writer (cctkGH, variable); - writer.write (file); + if (metafile) + { + writer_t writer (cctkGH, variable); + writer.write (* metafile); + delete metafile; + metafile = NULL; + } + { + writer_t writer (cctkGH, variable); +#warning "TODO: handle the case where not all processors are writing to their own file" + assert (proc == output_processor); + assert (file); + writer.write (* file); + delete file; + file = NULL; + } return Error_none; } @@ -416,12 +483,84 @@ namespace CarpetIOF5 { void WriteParameters (F5::file_t & file) { - bool const use_metafile = file.get_have_metafile(); - hid_t const hdf5_file - = use_metafile ? file.get_hdf5_metafile() : file.get_hdf5_file(); + DECLARE_CCTK_PARAMETERS; + + cGH const * const cctkGH = file.get_cctkGH (); + + hid_t const hdf5_file = file.get_hdf5_file(); + + hid_t const attribute_group + = F5::open_or_create_group (hdf5_file, + "Parameters and Global Attributes"); + assert (attribute_group >= 0); + + // unique configuration identifier + if (CCTK_IsFunctionAliased ("UniqueConfigID")) { + F5::write_or_check_attribute + (attribute_group, "config id", + static_cast (UniqueConfigID (cctkGH))); + } + + // unique build identifier + if (CCTK_IsFunctionAliased ("UniqueBuildID")) { + F5::write_or_check_attribute + (attribute_group, "build id", + static_cast (UniqueBuildID (cctkGH))); + } + + // unique simulation identifier + if (CCTK_IsFunctionAliased ("UniqueSimulationID")) { + F5::write_or_check_attribute + (attribute_group, "simulation id", + static_cast (UniqueSimulationID (cctkGH))); + } + + // unique run identifier + if (CCTK_IsFunctionAliased ("UniqueRunID")) { + F5::write_or_check_attribute + (attribute_group, "run id", + static_cast (UniqueRunID (cctkGH))); + } + + // Output Cactus parameters as single string + { + char * const parameters = IOUtil_GetAllParameters (cctkGH, true); + assert (parameters); + // Create a dataset, since the data may not fit into an attribute + hsize_t const size = strlen (parameters) + 1; + hid_t const dataspace = H5Screate_simple (1, & size, NULL); + assert (dataspace >= 0); + hid_t properties = H5Pcreate (H5P_DATASET_CREATE); + assert (properties >= 0); + check (not H5Pset_chunk (properties, 1, & size)); + if (compression_level > 0) + { + check (not H5Pset_deflate (properties, compression_level)); + } + if (write_checksum) + { + check (not H5Pset_fletcher32 (properties)); + } + hid_t const dataset + = H5Dcreate (attribute_group, "All Parameters", H5T_NATIVE_CHAR, + dataspace, H5P_DEFAULT, + properties, H5P_DEFAULT); + assert (dataset >= 0); + check (not H5Dwrite (dataset, H5T_NATIVE_CHAR, H5S_ALL, H5S_ALL, + H5P_DEFAULT, parameters)); + check (not H5Dclose (dataset)); + check (not H5Pclose (properties)); + check (not H5Sclose (dataspace)); + free (parameters); + } + + check (not H5Gclose (attribute_group)); + // This is far too slow to be useful +#if 0 hid_t const parameter_group = F5::open_or_create_group (hdf5_file, "Cactus parameters"); + assert (parameter_group >= 0); int first = 1; for (;;) @@ -479,8 +618,8 @@ namespace CarpetIOF5 { first = 0; } - herr_t const herr = H5Gclose (parameter_group); - assert (not herr); + check (not H5Gclose (parameter_group)); +#endif } diff --git a/CarpetDev/CarpetIOF5/src/coordinate_system.cc b/CarpetDev/CarpetIOF5/src/coordinate_system.cc index bb5b6808f..e030c08a9 100644 --- a/CarpetDev/CarpetIOF5/src/coordinate_system.cc +++ b/CarpetDev/CarpetIOF5/src/coordinate_system.cc @@ -102,8 +102,7 @@ namespace CarpetIOF5 { Cartesian_coordinate_system_t:: ~ Cartesian_coordinate_system_t () { - herr_t const herr = H5Gclose (m_hdf5_coordinate_system); - assert (not herr); + check (not H5Gclose (m_hdf5_coordinate_system)); } @@ -114,8 +113,10 @@ namespace CarpetIOF5 { assert (all (m_level_delta > (CCTK_REAL) 0)); ostringstream namebuf; - namebuf << "Cartesian 3D, x0=" << m_level_origin - << ", dx=" << m_level_delta; + namebuf << "Cartesian3D-" + << name_from_ivect (m_level_origin) + << "-" + << name_from_ivect (m_level_delta); string const namestr = namebuf.str(); m_name = namestr; char const * const name = namestr.c_str(); @@ -124,6 +125,7 @@ namespace CarpetIOF5 { = open_or_create_group (m_topology.get_hdf5_topology(), name); assert (m_hdf5_coordinate_system >= 0); +#warning "TODO: don't output coordinates as attributes" write_or_check_attribute (m_hdf5_coordinate_system, "origin", m_level_origin); write_or_check_attribute @@ -133,21 +135,20 @@ namespace CarpetIOF5 { void coordinate_system_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) + get_topology().get_link_destination (proc, filename, objectname); + if (objectname.empty()) { - initialised = true; - get_topology().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; + objectname = m_name; + } + else + { + objectname += string ("/") + m_name; } - filename = l_filename; - objectname = l_objectname; } diff --git a/CarpetDev/CarpetIOF5/src/coordinate_system.hh b/CarpetDev/CarpetIOF5/src/coordinate_system.hh index 3a36a458a..2b67b4561 100644 --- a/CarpetDev/CarpetIOF5/src/coordinate_system.hh +++ b/CarpetDev/CarpetIOF5/src/coordinate_system.hh @@ -46,7 +46,8 @@ namespace CarpetIOF5 { const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; diff --git a/CarpetDev/CarpetIOF5/src/data_region.cc b/CarpetDev/CarpetIOF5/src/data_region.cc index 21728c2e6..99b188323 100644 --- a/CarpetDev/CarpetIOF5/src/data_region.cc +++ b/CarpetDev/CarpetIOF5/src/data_region.cc @@ -3,12 +3,14 @@ #include #include +#include + #include "cctk.h" -#include "cctk_Parameters.h" #include "carpet.hh" #include "data_region.hh" +#include "utils.hh" @@ -21,54 +23,19 @@ namespace CarpetIOF5 { data_region_t:: - data_region_t (tensor_component_t & tensor_component, + data_region_t (physical_quantity_t & physical_quantity, bbox const & region) - : m_tensor_component (tensor_component), - m_region (region) + : m_physical_quantity (physical_quantity), + m_region (region), + m_name (string ("region-") + F5::name_from_ibbox (region)) { - DECLARE_CCTK_PARAMETERS; - assert (not region.empty()); - ostringstream namebuf; -#if 0 - namebuf << "map=" << Carpet::map << " " - << "region=" << m_region; -#else - namebuf << "region=" << m_region; -#endif - string const namestr = namebuf.str(); - char const * const name = namestr.c_str(); - assert (name != 0); - - int const vartype = CCTK_VarTypeI (m_tensor_component.get_variable()); - assert (vartype >= 0); - hid_t const hdf5_datatype = hdf5_datatype_from_cactus_datatype (vartype); - assert (hdf5_datatype >= 0); - - vect const dims - = (region.shape() / region.stride()).reverse(); - m_dataspace = H5Screate_simple (dim, & dims [0], 0); - assert (m_dataspace >= 0); - - m_properties = H5Pcreate (H5P_DATASET_CREATE); - assert (m_properties >= 0); - if (compression_level > 0) - { - herr_t const herr1 = H5Pset_chunk (m_properties, dim, & dims [0]); - assert (not herr1); - herr_t const herr2 = H5Pset_deflate (m_properties, compression_level); - assert (not herr2); - } - - m_dataset - = H5Dcreate (m_tensor_component.get_hdf5_tensor_component(), name, - hdf5_datatype, m_dataspace, - H5P_DEFAULT, m_properties, H5P_DEFAULT); - assert (m_dataset >= 0); - - write_or_check_attribute - (m_dataset, "iorigin", region.lower() / region.stride()); + m_hdf5_data_region + = open_or_create_group (m_physical_quantity + .get_hdf5_physical_quantity(), + m_name.c_str()); + assert (m_hdf5_data_region >= 0); assert (invariant()); } @@ -78,73 +45,35 @@ namespace CarpetIOF5 { data_region_t:: ~ data_region_t () { - herr_t herr; - - herr = H5Dclose (m_dataset); - assert (not herr); - - herr = H5Sclose (m_dataspace); - assert (not herr); - - herr = H5Pclose (m_properties); + herr_t const herr = H5Gclose (m_hdf5_data_region); assert (not herr); } - string data_region_t:: - name_from_region (bbox const & region) + physical_quantity_t & data_region_t:: + get_physical_quantity () + const { - ostringstream namebuf; -#if 0 - namebuf << "map=" << Carpet::map << " " - << "region=" << region; -#else - namebuf << "region=" << region; -#endif - return namebuf.str(); + return m_physical_quantity; } - tensor_component_t & data_region_t:: - get_tensor_component () + bbox const & data_region_t:: + get_region () const { - return m_tensor_component; + return m_region; } - void data_region_t:: - write (void const * const data, - int const cactus_datatype) + hid_t data_region_t:: + get_hdf5_data_region () const { - hid_t const memory_hdf5_datatype - = hdf5_datatype_from_cactus_datatype (cactus_datatype); - assert (memory_hdf5_datatype >= 0); - - vect const dims - = (m_region.shape() / m_region.stride()).reverse(); - hid_t const memory_dataspace - = H5Screate_simple (dim, & dims [0], & dims [0]); - assert (memory_dataspace >= 0); - - hid_t const transfer_properties = H5Pcreate (H5P_DATASET_XFER); - assert (transfer_properties >= 0); - - herr_t herr; - herr - = H5Dwrite (m_dataset, memory_hdf5_datatype, memory_dataspace, - m_dataspace, transfer_properties, data); - assert (not herr); - - herr = H5Pclose (transfer_properties); - assert (not herr); - - herr = H5Sclose (memory_dataspace); - assert (not herr); + return m_hdf5_data_region; } @@ -154,9 +83,7 @@ namespace CarpetIOF5 { const { return (not m_region.empty() - and m_properties >= 0 - and m_dataset >= 0 - and m_dataspace >= 0); + and m_hdf5_data_region >= 0); } } // namespace F5 diff --git a/CarpetDev/CarpetIOF5/src/data_region.hh b/CarpetDev/CarpetIOF5/src/data_region.hh index 29d5504c7..35e5ecc79 100644 --- a/CarpetDev/CarpetIOF5/src/data_region.hh +++ b/CarpetDev/CarpetIOF5/src/data_region.hh @@ -1,18 +1,14 @@ #ifndef DATA_REGION_HH #define DATA_REGION_HH -// force HDF5 1.8.x installations to use the new API -#ifdef H5Dcreate_vers -#undef H5Dcreate_vers -#endif -#define H5Dcreate_vers 2 +#include #include #include "bbox.hh" #include "defs.hh" -#include "tensor_component.hh" +#include "physical_quantity.hh" @@ -20,15 +16,16 @@ namespace CarpetIOF5 { namespace F5 { + using std::string; + class data_region_t { - tensor_component_t & m_tensor_component; + physical_quantity_t & m_physical_quantity; bbox const m_region; + string const m_name; - hid_t m_properties; - hid_t m_dataset; - hid_t m_dataspace; + hid_t m_hdf5_data_region; data_region_t (); data_region_t (data_region_t const &); @@ -36,22 +33,22 @@ namespace CarpetIOF5 { public: - data_region_t (tensor_component_t & tensor_component, + data_region_t (physical_quantity_t & physical_quantity, bbox const & region); virtual ~ data_region_t (); - static string - name_from_region (bbox const & region); + physical_quantity_t & + get_physical_quantity () + const; - tensor_component_t & - get_tensor_component () + bbox const & + get_region () const; - void - write (void const * data, - int cactus_datatype) + hid_t + get_hdf5_data_region () const; virtual bool diff --git a/CarpetDev/CarpetIOF5/src/extending.cc b/CarpetDev/CarpetIOF5/src/extending.cc index bff256190..26d63cb60 100644 --- a/CarpetDev/CarpetIOF5/src/extending.cc +++ b/CarpetDev/CarpetIOF5/src/extending.cc @@ -1,6 +1,5 @@ #include #include -#include #include "cctk.h" diff --git a/CarpetDev/CarpetIOF5/src/f5writer.cc b/CarpetDev/CarpetIOF5/src/f5writer.cc new file mode 100644 index 000000000..b351c7391 --- /dev/null +++ b/CarpetDev/CarpetIOF5/src/f5writer.cc @@ -0,0 +1,445 @@ +#include +#include + +#include + +#include "cctk.h" +#include "cctk_Arguments.h" +#include "cctk_Parameters.h" + +#include "carpet.hh" + +#include "data_region.hh" +#include "f5writer.hh" +#include "file.hh" +#include "meta_data_region.hh" +#include "physical_quantity.hh" +#include "simulation.hh" +#include "tensor_component.hh" +#include "timestep.hh" +#include "topology.hh" + + + +namespace CarpetIOF5 { + + f5writer_t:: + f5writer_t (cGH const * const cctkGH, + int const variable) + : m_cctkGH (cctkGH), + m_variable (variable) + { + } + + + + void f5writer_t:: + write (F5::file_t & file) + const + { + write_meta (file, file.get_have_metafile()); + } + + + + void f5writer_t:: + write_meta (F5::file_t & file, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_meta"); + } + + if (Carpet::is_meta_mode()) + { + for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH); + not mglevel_iter.done(); + mglevel_iter.step()) + { + write_one_mglevel (file, have_metafile); + } + } + else + { + write_one_mglevel (file, have_metafile); + } + } + + + + void f5writer_t:: + write_one_mglevel (F5::file_t & file, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_mglevel mglevel=%d", + Carpet::mglevel); + } + + // ostringstream namebuf; + // namebuf << "convlevel=" << m_cctkGH->cctk_convlevel; + // string const namestr = namebuf.str(); + // char const * const name = namestr.c_str(); + + int const grouptype = CCTK_GroupTypeFromVarI (m_variable); + assert (grouptype >= 0); + switch (grouptype) + { + case CCTK_ARRAY: + case CCTK_SCALAR: + { + if (Carpet::do_global_mode) + { + write_global (file, have_metafile); + } + } + break; + case CCTK_GF: + { + if (Carpet::is_global_mode()) + { + for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH); + not reflevel_iter.done(); + reflevel_iter.step()) + { + write_one_reflevel (file, have_metafile); + } + } + else + { + write_one_reflevel (simulation, have_metafile); + } + } + break; + default: + assert (0); + } + } + + + + void f5writer_t:: + write_global (F5::simulation_t & simulation, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_INFO ("OutputVarAs/write_global"); + } + + F5::unigrid_topology_t topology (simulation); + + int const grouptype = CCTK_GroupTypeFromVarI (m_variable); + assert (grouptype >= 0); + assert (grouptype == CCTK_SCALAR or grouptype == CCTK_ARRAY); + + vect level_origin, level_delta; + for (int d=0; d= 0 and group < CCTK_NumGroups()); + F5::physical_quantity_t physical_quantity (coordinate_system, group); + + int const map = 0; + int const reflevel = 0; + int const myproc = CCTK_MyProc (m_cctkGH); + dh * const dd = Carpet::arrdata.at(group).at(map).dd; + dh::dboxes const & boxes + = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc); + bbox const & region = determine_region (boxes); + + if (have_metafile) + { + F5::meta_data_region_t meta_data_region (physical_quantity, region); + gh * const hh = Carpet::vhh.at(Carpet::map); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + + + + void f5writer_t:: + write_one_reflevel (F5::file_t & file, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_reflevel reflevel=%d", + Carpet::reflevel); + } + + // int const grouptype = CCTK_GroupTypeFromVarI (m_variable); + // assert (grouptype >= 0); + // assert (grouptype == CCTK_GF); + + // A name for the simulation + string sim_id; + if (CCTK_IsFunctionAliased ("UniqueSimulationID")) + { + sim_id = static_cast (UniqueSimulationID (m_cctkGH)); + } + else { + char parfilename[10000]; + int const len = CCTK_ParameterFilename (sizeof parfilename, parfilename); + assert (len >= 0 and len < sizeof parfilename); + sim_id = parfilename; + if (sim_id.length() >= 4 + and sim_id.substr (sim_id.length() - 4) == ".par") + { + sim_id = sim_id.substr (0, sim_id.length() - 4); + } + } + + // Refinement factor of current refinement level + vect const current_refinement + = vect ::ref (m_cctkGH->cctk_levfac); + + // Refinement factor of root refinement level + vect const rootlevel_refinement (1); + + // Remember time when the root level was output the last time + static CCTK_REAL rootlevel_time = 0.0; + if (all (current_refinement == 1)) + { + rootlevel_time = m_cctkGH->cctk_time; + } + + F5Path * refinement_path = NULL; + if (any (current_refinement != 1)) + { + refinement_path + = F5Rcreate_relative_vertex_Qrefinement3D (file, + m_cctkGH->cctk_time, + sim_id.c_str(), + current_refinement, + rootlevel_time, + rootlevel_refinement); + } + +#warning "TODO: what is this doing?" + F5T_REFINEMENT3D_POINT = refinement_path->myChart->Point_hid_t; + + if (Carpet::is_level_mode()) + { + for (Carpet::map_iterator map_iter (m_cctkGH, grouptype); + not map_iter.done(); + map_iter.step()) + { + write_one_map (simulation, have_metafile); + } + } + else + { + write_one_map (simulation, have_metafile); + } + } + + + + void f5writer_t:: + write_one_map (F5::simulation_t & simulation, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_map map=%d", Carpet::map); + } + + // F5::mesh_refinement_topology_t topology + // (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels, + // Carpet::spacereflevelfact, Carpet::maxspacereflevelfact); + + // vect level_origin, level_delta; + // for (int d=0; d= 0 and group < CCTK_NumGroups()); + // F5::physical_quantity_t physical_quantity (coordinate_system, group); + +#warning "TODO: this should depend on the patch number" + char const * const coordinate_system = NULL; + + // Depends on the refinement level + string const gridname = sim_id + "-" + current_refinement; + + F5Path * const myPath + = F5Rcreate_vertex_refinement3D (file, + m_cctkGH->cctk_time, + sim_id.c_str(), + refinement, + coordinate_system); + + if (not refinement_path) + { + // Make root level appear as unigrid topology as well, just to + // be nice to tools that do not understand AMR + F5Rlink_default_vertex_topology (myPath, current_refinement); + } + + // Global information for this refinement level + vect const level_dims + = vect ::ref (m_cctkGH->cctk_gsh); +#warning "TODO: switch to double precision" + vect const level_min + (CCTK_ORIGIN_SPACE(0), CCTK_ORIGIN_SPACE(1), CCTK_ORIGIN_SPACE(2)); + vect const level_spacing + (CCTK_DELTA_SPACE(0), CCTK_DELTA_SPACE(1), CCTK_DELTA_SPACE(2)); + vect const level_max + = level_min + level_spacing * (level_dims - 1); + + // Output geometric information for an entire level; it is + // uniformely covered in coordinate space + assert (dim == 3); + F5Fwrite_linear (myPath, FIBER_HDF5_POSITIONS_STRING, + dim, & level_dims [0], + F5T_COORD3_FLOAT, & level_min [0], & level_spacing [0]); + + F5Fset_range (myPath, & level_min [0], & level_max [0]); + +#error "CONTINUE HERE" + + if (Carpet::is_singlemap_mode()) + { + int const grouptype = CCTK_GroupTypeI (group); + assert (grouptype >= 0); + + for (Carpet::component_iterator component_iter (m_cctkGH, grouptype); + not component_iter.done(); + component_iter.step()) + { + write_one_component (physical_quantity, have_metafile); + } + } + else + { + write_one_component (physical_quantity, have_metafile); + } + } + + + + void f5writer_t:: + write_one_component (F5::physical_quantity_t & physical_quantity, + bool const have_metafile) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_component component=%d", + Carpet::component); + } + + gh * const hh = Carpet::vhh.at(Carpet::map); + bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component); + if (have_metafile or is_local) + { + dh * const dd = Carpet::vdd.at(Carpet::map); + bbox const & region + = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel) + .at(Carpet::component).exterior); + + if (have_metafile) + { + F5::meta_data_region_t meta_data_region (physical_quantity, region); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + + if (is_local) + { + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + } + } + + + + bbox const & f5writer_t:: + determine_region (dh::dboxes const & boxes) + const + { + DECLARE_CCTK_PARAMETERS; + +#warning "TODO: use superregions instead of regions (? only if the regions are on the same processor?); use HDF5 chunks as well" + + bbox dh::dboxes::* boxptr; + if (CCTK_EQUALS (output_regions, "exterior")) + { + boxptr = & dh::dboxes::exterior; + } + else if (CCTK_EQUALS (output_regions, "owned")) + { + boxptr = & dh::dboxes::owned; + } + else if (CCTK_EQUALS (output_regions, "interior")) + { + boxptr = & dh::dboxes::interior; + } + else + { + assert (0); + } + + return boxes.*boxptr; + } + +} // namespace CarpetIOF5 diff --git a/CarpetDev/CarpetIOF5/src/f5writer.hh b/CarpetDev/CarpetIOF5/src/f5writer.hh new file mode 100644 index 000000000..ae016721e --- /dev/null +++ b/CarpetDev/CarpetIOF5/src/f5writer.hh @@ -0,0 +1,74 @@ +#ifndef F5WRITER_HH +#define F5WRITER_HH + +#include "cctk.h" + +#include "bbox.hh" +#include "defs.hh" +#include "dh.hh" + +#include "file.hh" +#include "physical_quantity.hh" +#include "simulation.hh" +#include "timestep.hh" +#include "topology.hh" + + + +namespace CarpetIOF5 { + + class f5writer_t { + + cGH const * const m_cctkGH; + int const m_variable; + + public: + + f5writer_t (cGH const * cctkGH, + int variable); + + void + write (F5::file_t & file) + const; + + private: + + void + write_meta (F5::file_t & file, + bool have_metafile) + const; + + void + write_one_mglevel (F5::timestep_t & timestep, + bool have_metafile) + const; + + void + write_global (F5::simulation_t & simulation, + bool have_metafile) + const; + + void + write_one_reflevel (F5::simulation_t & simulation, + bool have_metafile) + const; + + void + write_one_map (F5::simulation_t & simulation, + bool have_metafile) + const; + + void + write_one_component (F5::physical_quantity_t & physical_quantity, + bool have_metafile) + const; + + bbox const & + determine_region (dh::dboxes const & boxes) + const; + + }; + +} // namespace CarpetIOF5 + +#endif // #ifndef F5WRITER_HH diff --git a/CarpetDev/CarpetIOF5/src/file.cc b/CarpetDev/CarpetIOF5/src/file.cc index f200277b5..d78dcb001 100644 --- a/CarpetDev/CarpetIOF5/src/file.cc +++ b/CarpetDev/CarpetIOF5/src/file.cc @@ -1,16 +1,19 @@ #include #include #include -#include #include #include +#include #include #include "cctk.h" #include "cctk_Parameters.h" +#include "defs.hh" + #include "file.hh" +#include "utils.hh" @@ -22,58 +25,86 @@ namespace CarpetIOF5 { file_t:: file_t (cGH const * const cctkGH, + string const path, string const basename, string const extension, - bool const do_truncate) + bool const do_truncate, + bool const is_metafile, + bool const is_datafile) : m_cctkGH (cctkGH), + m_path (path), m_basename (basename), - m_extension (extension) + m_extension (extension), + m_is_metafile (is_metafile), + m_is_datafile (is_datafile) { - assert (cctkGH); - - int const proc = CCTK_MyProc (cctkGH); - m_have_metafile = determine_want_metafile (proc); - m_output_processor = determine_output_processor (proc); + DECLARE_CCTK_PARAMETERS; - m_metafilename = make_metafilename (); - m_filename = make_filename (proc); + assert (cctkGH); - char const * const metafilenameptr = m_metafilename.c_str(); - char const * const filenameptr = m_filename.c_str(); + // Create file names, creating subdirectories to avoid placing + // too many files into the same directory + check (CCTK_CreateDirectory (mode, path.c_str()) >= 0); + if (is_metafile) + { + m_filename = basename + extension; + } + else + { + int const myproc = CCTK_MyProc (cctkGH); + m_filename = create_filename (myproc, true); + } + string const filepath = m_path + "/" + m_filename; htri_t is_hdf5; H5E_BEGIN_TRY { - is_hdf5 = H5Fis_hdf5 (filenameptr); + is_hdf5 = H5Fis_hdf5 (filepath.c_str()); } H5E_END_TRY; bool const file_exists = is_hdf5 > 0; - // Ignore the metafile when determining whether the file already - // exists if (do_truncate or not file_exists) { - m_hdf5_metafile = -1; - if (m_have_metafile) + if (veryverbose) { - m_hdf5_metafile - = H5Fcreate (metafilenameptr, - H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CCTK_VInfo (CCTK_THORNSTRING, + "H5Fcreate (name=\"%s\")", filepath.c_str()); } m_hdf5_file - = H5Fcreate (filenameptr, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + = H5Fcreate (filepath.c_str(), + H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); } else { - m_hdf5_metafile = -1; - if (m_have_metafile) + if (veryverbose) { - m_hdf5_metafile - = H5Fopen (metafilenameptr, H5F_ACC_RDWR, H5P_DEFAULT); + CCTK_VInfo (CCTK_THORNSTRING, + "H5Fopen (name=\"%s\")", filepath.c_str()); } - m_hdf5_file = H5Fopen (filenameptr, H5F_ACC_RDWR, H5P_DEFAULT); + m_hdf5_file = H5Fopen (filepath.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); } + assert (m_hdf5_file >= 0); - m_filenames.resize (CCTK_nProcs (cctkGH)); - m_filenames.at(proc) = m_filename; + m_hdf5_fiber_contents = -1; + m_hdf5_fiber_global_charts = -1; + m_hdf5_fiber_parameter_space = -1; + if (m_is_metafile) + { + create_or_check_version (m_hdf5_file); + m_hdf5_fiber_contents + = open_or_create_group (m_hdf5_file, "TableOfContents"); + assert (m_hdf5_fiber_contents >= 0); + m_hdf5_fiber_global_charts + = open_or_create_group (m_hdf5_file, "Charts"); + assert (m_hdf5_fiber_global_charts >= 0); + m_hdf5_fiber_parameter_space + = open_or_create_group (m_hdf5_fiber_contents, "Parameters"); + assert (m_hdf5_fiber_parameter_space >= 0); + + hid_t const hdf5_time_group + = open_or_create_group (m_hdf5_fiber_parameter_space, "Time"); + assert (hdf5_time_group >= 0); + check (not H5Gclose (hdf5_time_group)); + } assert (invariant()); } @@ -83,102 +114,91 @@ namespace CarpetIOF5 { file_t:: ~ file_t() { - if (m_have_metafile) - { - herr_t const herr = H5Fclose (m_hdf5_metafile); - assert (not herr); - } - herr_t const herr = H5Fclose (m_hdf5_file); - assert (not herr); - } - - - - bool file_t:: - determine_want_metafile (int const proc) - const - { - DECLARE_CCTK_PARAMETERS; - - if (CCTK_EQUALS (out_mode, "proc") or - CCTK_EQUALS (out_mode, "np")) - { - return proc == 0; - } - else if (CCTK_EQUALS (out_mode, "onefile")) - { - return false; - } - else + if (m_is_metafile) { - assert (0); + check (not H5Gclose (m_hdf5_fiber_parameter_space)); + check (not H5Gclose (m_hdf5_fiber_contents)); + check (not H5Gclose (m_hdf5_fiber_global_charts)); } + check (not H5Fclose (m_hdf5_file)); } - int file_t:: - determine_output_processor (int const proc) + string file_t:: + create_filename (int const proc, bool const create_directories) const { DECLARE_CCTK_PARAMETERS; - if (CCTK_EQUALS (out_mode, "proc")) - { - return proc; - } - else if (CCTK_EQUALS (out_mode, "np")) - { - return proc / out_proc_every * out_proc_every; - } - else if (CCTK_EQUALS (out_mode, "onefile")) + int const nprocs = CCTK_nProcs (m_cctkGH); + string extrapath; + if (create_subdirs) { - return 0; + if (nprocs >= 10000) + { + ostringstream buf; + buf << extrapath + << m_basename + << ".p" << setw (max (0, processor_digits - 4)) << setfill ('0') + << proc / 10000 + << "nnnn/"; + extrapath = buf.str(); + if (create_directories) + { + string const filepath = m_path + "/" + extrapath; + check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0); + } + } + if (nprocs >= 100) + { + ostringstream buf; + buf << extrapath + << m_basename + << ".p" << setw (max (0, processor_digits - 2)) << setfill ('0') + << proc / 100 + << "nn/"; + extrapath = buf.str(); + if (create_directories) + { + string const filepath = m_path + "/" + extrapath; + check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0); + } + } } - else + if (one_dir_per_file and nprocs >= 1) { - assert (0); + ostringstream buf; + buf << extrapath + << m_basename + << ".p" << setw (processor_digits) << setfill ('0') + << proc + << "/"; + extrapath = buf.str(); + if (create_directories) + { + string const filepath = m_path + "/" + extrapath; + check (CCTK_CreateDirectory (mode, filepath.c_str()) >= 0); + } } + ostringstream buf; + buf << extrapath + << m_basename + << ".p" << setw (processor_digits) << setfill ('0') << proc + << m_extension; + return buf.str(); } - - - string file_t:: - make_metafilename () - const + + + void file_t:: + create_or_check_version (hid_t const hdf5_file) { - return m_basename + m_extension; - } - - - - string file_t:: - make_filename (int const proc) - const - { - DECLARE_CCTK_PARAMETERS; - - ostringstream filenamebuf; - - filenamebuf << m_basename; - - if (CCTK_EQUALS (out_mode, "proc") or - CCTK_EQUALS (out_mode, "np")) - { - filenamebuf << ".p" << setw (processor_digits) << setfill ('0') << proc; - } - else if (CCTK_EQUALS (out_mode, "onefile")) - { - // do nothing - } - else - { - assert (0); - } - - filenamebuf << m_extension; - - return filenamebuf.str(); + hid_t const hdf5_group = open_or_create_group (hdf5_file, "version"); + assert (hdf5_group >= 0); + string const version ("http://www.zib.de/visual/F5-0.1.2/"); + write_or_check_attribute (hdf5_group, "version", version.c_str()); + check (not H5Gclose (hdf5_group)); } @@ -192,72 +212,50 @@ namespace CarpetIOF5 { - bool file_t:: - get_have_metafile () - const - { - return m_have_metafile; - } - - - - int file_t:: - get_output_processor () + hid_t file_t:: + get_hdf5_file () const { - return m_output_processor; + return m_hdf5_file; } - string file_t:: - get_filename (int const proc) + hid_t file_t:: + get_hdf5_fiber_parameter_space () const { - assert (proc >= 0 and proc < CCTK_nProcs (m_cctkGH)); - if (m_filenames.at(proc).empty()) - { - m_filenames.at(proc) = make_filename (proc); - } - return m_filenames.at(proc); + return m_hdf5_fiber_parameter_space; } - hid_t file_t:: - get_hdf5_metafile() + bool file_t:: + get_is_metafile () const { - return m_hdf5_metafile; + return m_is_metafile; } - hid_t file_t:: - get_hdf5_file() + bool file_t:: + get_is_datafile () const { - return m_hdf5_file; + return m_is_datafile; } void file_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) - { - initialised = true; - l_filename = m_filename; - l_objectname = string (""); - } - filename = l_filename; - objectname = l_objectname; + filename = create_filename (proc); + objectname = ""; } @@ -266,9 +264,7 @@ namespace CarpetIOF5 { invariant() const { - return (m_cctkGH != 0 - and (not m_have_metafile or m_hdf5_metafile >= 0) - and m_hdf5_file >= 0); + return m_cctkGH != 0 and m_hdf5_file >= 0; } } // namespace F5 diff --git a/CarpetDev/CarpetIOF5/src/file.hh b/CarpetDev/CarpetIOF5/src/file.hh index 19add249e..44d8ce00c 100644 --- a/CarpetDev/CarpetIOF5/src/file.hh +++ b/CarpetDev/CarpetIOF5/src/file.hh @@ -23,48 +23,46 @@ namespace CarpetIOF5 { class file_t { - cGH const * const m_cctkGH; + // File mode for creating directories + static int const mode = 0755; - bool m_have_metafile; - int m_output_processor; + cGH const * const m_cctkGH; + string const m_path; string const m_basename; string const m_extension; - - string m_metafilename; string m_filename; - vector mutable m_filenames; + bool const m_is_metafile; + bool const m_is_datafile; - hid_t m_hdf5_metafile; hid_t m_hdf5_file; + hid_t m_hdf5_fiber_contents; + hid_t m_hdf5_fiber_global_charts; + hid_t m_hdf5_fiber_parameter_space; + file_t (); file_t (file_t const &); file_t operator= (file_t const &); - bool - determine_want_metafile (int proc) - const; - - int - determine_output_processor (int proc) - const; - string - make_metafilename () + create_filename (int proc, + bool create_directories = false) const; - string - make_filename (int proc) - const; + static void + create_or_check_version (hid_t const hdf5_file); public: file_t (cGH const * cctkGH, - string filename, + string path, + string basename, string extension, - bool do_truncate); + bool do_truncate, + bool is_metafile, + bool is_datafile); virtual ~ file_t (); @@ -73,28 +71,23 @@ namespace CarpetIOF5 { get_cctkGH () const; - bool - get_have_metafile () - const; - - int - get_output_processor () + hid_t + get_hdf5_file () const; - string - get_filename (int proc) + hid_t + get_hdf5_fiber_parameter_space () const; - hid_t - get_hdf5_metafile () + bool get_is_metafile () const; - hid_t - get_hdf5_file () + bool get_is_datafile () const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; diff --git a/CarpetDev/CarpetIOF5/src/make.code.defn b/CarpetDev/CarpetIOF5/src/make.code.defn index 0641b4f9d..d0350d235 100644 --- a/CarpetDev/CarpetIOF5/src/make.code.defn +++ b/CarpetDev/CarpetIOF5/src/make.code.defn @@ -4,16 +4,16 @@ SRCS = IOF5.cc \ coordinate_system.cc \ data_region.cc \ - extending.cc \ - file.cc \ + extending.cc \ + file.cc \ meta_data_region.cc \ - physical_quantity.cc \ - simulation.cc \ - tensor_component.cc \ - timestep.cc \ - topology.cc \ - utils.cc \ - writer.cc + physical_quantity.cc \ + simulation.cc \ + tensor_component.cc \ + timestep.cc \ + topology.cc \ + utils.cc \ + writer.cc # Subdirectories containing source files SUBDIRS = diff --git a/CarpetDev/CarpetIOF5/src/meta_data_region.cc b/CarpetDev/CarpetIOF5/src/meta_data_region.cc index 505508eb9..88963ec34 100644 --- a/CarpetDev/CarpetIOF5/src/meta_data_region.cc +++ b/CarpetDev/CarpetIOF5/src/meta_data_region.cc @@ -3,11 +3,14 @@ #include #include +#include + #include "cctk.h" #include "carpet.hh" -#include "data_region.hh" +#include "defs.hh" + #include "meta_data_region.hh" #include "utils.hh" @@ -22,17 +25,14 @@ namespace CarpetIOF5 { meta_data_region_t:: - meta_data_region_t (tensor_component_t & tensor_component, + meta_data_region_t (physical_quantity_t & physical_quantity, bbox const & region) - : m_tensor_component (tensor_component), - m_region (region) + : m_physical_quantity (physical_quantity), + m_region (region), + m_name (string ("region-") + F5::name_from_ibbox (region)) { assert (not region.empty()); - string const namestr = data_region_t::name_from_region (region); - char const * const name = namestr.c_str(); - assert (name != 0); - assert (invariant()); } @@ -45,11 +45,11 @@ namespace CarpetIOF5 { - tensor_component_t & meta_data_region_t:: - get_tensor_component () + physical_quantity_t & meta_data_region_t:: + get_physical_quantity () const { - return m_tensor_component; + return m_physical_quantity; } @@ -58,22 +58,48 @@ namespace CarpetIOF5 { write (int const proc) const { + DECLARE_CCTK_PARAMETERS; + string filename; string objectname; - get_tensor_component().get_link_destination (filename, objectname); + get_physical_quantity().get_link_destination (proc, filename, objectname); - string const name = data_region_t::name_from_region (m_region); + hid_t const hdf5_physical_quantity + = m_physical_quantity.get_hdf5_physical_quantity(); - hid_t const hdf5_tensor_component - = m_tensor_component.get_hdf5_tensor_component(); + bool link_exists; + H5E_BEGIN_TRY { + H5L_info_t linkbuf; + herr_t const herr + = H5Lget_info (hdf5_physical_quantity, m_name.c_str(), + & linkbuf, H5P_DEFAULT); + link_exists = herr == 0; + } H5E_END_TRY; + if (veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "Link \"%s\" %s", + m_name.c_str(), + link_exists ? "exists" : "does not exist"); + } - herr_t const herr - = H5Lcreate_external (filename.c_str(), - objectname.c_str(), - hdf5_tensor_component, - name.c_str(), - H5P_DEFAULT, H5P_DEFAULT); - assert (not herr); + if (not link_exists) + { + if (veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "H5Lcreate_external " + "(filename=\"%s\", objectname=\"%s\", linkname=\"%s\")", + filename.c_str(), + objectname.c_str(), + m_name.c_str()); + } + check (not H5Lcreate_external (filename.c_str(), + objectname.c_str(), + hdf5_physical_quantity, + m_name.c_str(), + H5P_DEFAULT, H5P_DEFAULT)); + } } diff --git a/CarpetDev/CarpetIOF5/src/meta_data_region.hh b/CarpetDev/CarpetIOF5/src/meta_data_region.hh index b109c5da4..ac59c18e2 100644 --- a/CarpetDev/CarpetIOF5/src/meta_data_region.hh +++ b/CarpetDev/CarpetIOF5/src/meta_data_region.hh @@ -1,12 +1,14 @@ #ifndef META_DATA_REGION_HH #define META_DATA_REGION_HH +#include + #include #include "bbox.hh" #include "defs.hh" -#include "tensor_component.hh" +#include "physical_quantity.hh" @@ -14,11 +16,14 @@ namespace CarpetIOF5 { namespace F5 { + using std::string; + class meta_data_region_t { - tensor_component_t & m_tensor_component; + physical_quantity_t & m_physical_quantity; bbox const m_region; + string const m_name; hid_t m_properties; hid_t m_dataset; @@ -30,14 +35,14 @@ namespace CarpetIOF5 { public: - meta_data_region_t (tensor_component_t & tensor_component, + meta_data_region_t (physical_quantity_t & physical_quantity, bbox const & region); virtual ~ meta_data_region_t (); - tensor_component_t & - get_tensor_component () + physical_quantity_t & + get_physical_quantity () const; void diff --git a/CarpetDev/CarpetIOF5/src/physical_quantity.cc b/CarpetDev/CarpetIOF5/src/physical_quantity.cc index 0b6720ece..d0cb78dcd 100644 --- a/CarpetDev/CarpetIOF5/src/physical_quantity.cc +++ b/CarpetDev/CarpetIOF5/src/physical_quantity.cc @@ -1,6 +1,8 @@ #include #include +#include + #include "cctk.h" #include "physical_quantity.hh" @@ -67,6 +69,15 @@ namespace CarpetIOF5 { + string physical_quantity_t:: + get_name () + const + { + return m_name; + } + + + hid_t physical_quantity_t:: get_hdf5_physical_quantity () const @@ -77,21 +88,20 @@ namespace CarpetIOF5 { void physical_quantity_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) + get_coordinate_system().get_link_destination (proc, filename, objectname); + if (objectname.empty()) + { + objectname = m_name; + } + else { - initialised = true; - get_coordinate_system().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; + objectname += string ("/") + m_name; } - filename = l_filename; - objectname = l_objectname; } diff --git a/CarpetDev/CarpetIOF5/src/physical_quantity.hh b/CarpetDev/CarpetIOF5/src/physical_quantity.hh index 38b670cb2..e2b025b81 100644 --- a/CarpetDev/CarpetIOF5/src/physical_quantity.hh +++ b/CarpetDev/CarpetIOF5/src/physical_quantity.hh @@ -40,12 +40,17 @@ namespace CarpetIOF5 { get_group () const; + string + get_name () + const; + hid_t get_hdf5_physical_quantity () const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; diff --git a/CarpetDev/CarpetIOF5/src/simulation.cc b/CarpetDev/CarpetIOF5/src/simulation.cc index 98a0a3888..1adde5e5a 100644 --- a/CarpetDev/CarpetIOF5/src/simulation.cc +++ b/CarpetDev/CarpetIOF5/src/simulation.cc @@ -4,6 +4,8 @@ #include "cctk.h" +#include "defs.hh" + #include "simulation.hh" #include "utils.hh" @@ -28,6 +30,10 @@ namespace CarpetIOF5 { m_name.c_str()); assert (m_hdf5_simulation >= 0); + cGH const * const cctkGH = m_timestep.get_file().get_cctkGH(); + write_or_check_attribute + (m_hdf5_simulation, "TimeStep", cctkGH->cctk_iteration); + assert (invariant()); } @@ -36,8 +42,7 @@ namespace CarpetIOF5 { simulation_t:: ~ simulation_t() { - herr_t const herr = H5Gclose (m_hdf5_simulation); - assert (not herr); + check (not H5Gclose (m_hdf5_simulation)); } @@ -61,21 +66,20 @@ namespace CarpetIOF5 { void simulation_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) + get_timestep().get_link_destination (proc, filename, objectname); + if (objectname.empty()) + { + objectname = m_name; + } + else { - initialised = true; - get_timestep().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; + objectname += string ("/") + m_name; } - filename = l_filename; - objectname = l_objectname; } diff --git a/CarpetDev/CarpetIOF5/src/simulation.hh b/CarpetDev/CarpetIOF5/src/simulation.hh index 272b3385c..4c4639958 100644 --- a/CarpetDev/CarpetIOF5/src/simulation.hh +++ b/CarpetDev/CarpetIOF5/src/simulation.hh @@ -47,7 +47,8 @@ namespace CarpetIOF5 { const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; diff --git a/CarpetDev/CarpetIOF5/src/tensor_component.cc b/CarpetDev/CarpetIOF5/src/tensor_component.cc index 5d4b281fc..6cd2b6486 100644 --- a/CarpetDev/CarpetIOF5/src/tensor_component.cc +++ b/CarpetDev/CarpetIOF5/src/tensor_component.cc @@ -1,7 +1,15 @@ #include #include +// force HDF5 1.8.x installations to use the new API +#define H5Dcreate_vers 2 + +#include + #include "cctk.h" +#include "cctk_Parameters.h" + +#include "vect.hh" #include "tensor_component.hh" @@ -12,22 +20,65 @@ namespace CarpetIOF5 { namespace F5 { tensor_component_t:: - tensor_component_t (physical_quantity_t & physical_quantity, + tensor_component_t (data_region_t & data_region, int const variable) - : m_physical_quantity (physical_quantity), + : m_data_region (data_region), m_variable (variable) { + DECLARE_CCTK_PARAMETERS; + assert (variable >= 0 and variable < CCTK_NumVars()); char const * const name = CCTK_VarName (variable); assert (name != 0); m_name = string (name); - m_hdf5_tensor_component - = open_or_create_group (m_physical_quantity - .get_hdf5_physical_quantity(), - name); - assert (m_hdf5_tensor_component >= 0); + int const vartype = CCTK_VarTypeI (variable); + assert (vartype >= 0); + hid_t const hdf5_datatype = hdf5_datatype_from_cactus_datatype (vartype); + assert (hdf5_datatype >= 0); + + bbox const & region = m_data_region.get_region(); + + vect const dims + = (region.shape() / region.stride()).reverse(); + m_dataspace = H5Screate_simple (dim, & dims [0], 0); + assert (m_dataspace >= 0); + + m_properties = H5Pcreate (H5P_DATASET_CREATE); + assert (m_properties >= 0); + vect const user_chunk_size + (chunk_size_x, chunk_size_y, chunk_size_z); + bool const need_chunks + = any (user_chunk_size > 0) or compression_level > 0 or write_checksum; + if (need_chunks) + { + vect const chunk_size + = either (user_chunk_size > 0, + vect (user_chunk_size), + dims); + herr_t const herr = H5Pset_chunk (m_properties, dim, & chunk_size [0]); + assert (not herr); + } + if (compression_level > 0) + { + herr_t const herr = H5Pset_deflate (m_properties, compression_level); + assert (not herr); + } + if (write_checksum) + { + herr_t const herr = H5Pset_fletcher32 (m_properties); + assert (not herr); + } + + m_dataset + = H5Dcreate (m_data_region.get_hdf5_data_region(), m_name.c_str(), + hdf5_datatype, m_dataspace, + H5P_DEFAULT, m_properties, H5P_DEFAULT); + assert (m_dataset >= 0); + + write_or_check_attribute + (m_dataset, "iorigin", region.lower() / region.stride()); assert (invariant()); } @@ -37,17 +88,25 @@ namespace CarpetIOF5 { tensor_component_t:: ~ tensor_component_t () { - herr_t const herr = H5Gclose (m_hdf5_tensor_component); + herr_t herr; + + herr = H5Dclose (m_dataset); + assert (not herr); + + herr = H5Sclose (m_dataspace); + assert (not herr); + + herr = H5Pclose (m_properties); assert (not herr); } - physical_quantity_t & tensor_component_t:: - get_physical_quantity () + data_region_t & tensor_component_t:: + get_data_region () const { - return m_physical_quantity; + return m_data_region; } @@ -61,31 +120,38 @@ namespace CarpetIOF5 { - hid_t tensor_component_t:: - get_hdf5_tensor_component () - const - { - return m_hdf5_tensor_component; - } - - - +#warning "TODO: This assumes that the shape of data is the same as the shape of the region; this may not be so if not all of the data are written out" void tensor_component_t:: - get_link_destination (string & filename, - string & objectname) + write (void const * const data, + int const cactus_datatype) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) - { - initialised = true; - get_physical_quantity().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; - } - filename = l_filename; - objectname = l_objectname; + hid_t const memory_hdf5_datatype + = hdf5_datatype_from_cactus_datatype (cactus_datatype); + assert (memory_hdf5_datatype >= 0); + + bbox const & region = m_data_region.get_region(); + + vect const dims + = (region.shape() / region.stride()).reverse(); + hid_t const memory_dataspace + = H5Screate_simple (dim, & dims [0], & dims [0]); + assert (memory_dataspace >= 0); + + hid_t const transfer_properties = H5Pcreate (H5P_DATASET_XFER); + assert (transfer_properties >= 0); + + herr_t herr; + herr + = H5Dwrite (m_dataset, memory_hdf5_datatype, memory_dataspace, + m_dataspace, transfer_properties, data); + assert (not herr); + + herr = H5Pclose (transfer_properties); + assert (not herr); + + herr = H5Sclose (memory_dataspace); + assert (not herr); } @@ -95,7 +161,9 @@ namespace CarpetIOF5 { const { return (m_variable >= 0 and m_variable < CCTK_NumVars() - and m_hdf5_tensor_component >= 0); + and m_properties >= 0 + and m_dataset >= 0 + and m_dataspace >= 0); } } // namespace F5 diff --git a/CarpetDev/CarpetIOF5/src/tensor_component.hh b/CarpetDev/CarpetIOF5/src/tensor_component.hh index f659c0076..8a95d6faa 100644 --- a/CarpetDev/CarpetIOF5/src/tensor_component.hh +++ b/CarpetDev/CarpetIOF5/src/tensor_component.hh @@ -1,10 +1,11 @@ #ifndef TENSOR_COMPONENT_HH #define TENSOR_COMPONENT_HH +#include + #include -#include "physical_quantity.hh" -#include "utils.hh" +#include "data_region.hh" @@ -14,12 +15,14 @@ namespace CarpetIOF5 { class tensor_component_t { - physical_quantity_t & m_physical_quantity; + data_region_t & m_data_region; int const m_variable; string m_name; - hid_t m_hdf5_tensor_component; + hid_t m_dataspace; + hid_t m_properties; + hid_t m_dataset; tensor_component_t (); tensor_component_t (tensor_component_t const &); @@ -27,27 +30,23 @@ namespace CarpetIOF5 { public: - tensor_component_t (physical_quantity_t & physical_quantity, + tensor_component_t (data_region_t & data_region, int variable); virtual ~ tensor_component_t (); - physical_quantity_t & - get_physical_quantity () + data_region_t & + get_data_region () const; hid_t get_variable () const; - hid_t - get_hdf5_tensor_component () - const; - void - get_link_destination (string & filename, - string & objectname) + write (void const * data, + int cactus_datatype) const; virtual bool diff --git a/CarpetDev/CarpetIOF5/src/timestep.cc b/CarpetDev/CarpetIOF5/src/timestep.cc index 73d106a92..409f47221 100644 --- a/CarpetDev/CarpetIOF5/src/timestep.cc +++ b/CarpetDev/CarpetIOF5/src/timestep.cc @@ -9,6 +9,8 @@ #include "cctk.h" #include "cctk_Functions.h" +#include "defs.hh" + #include "timestep.hh" #include "utils.hh" @@ -42,21 +44,14 @@ namespace CarpetIOF5 { { // Create a string from the time without losing information int const precision = numeric_limits::digits10 + 2; - buf << setprecision (precision) << "t=" << time; + buf << setprecision (precision) << "T=" << time; } m_name = buf.str(); m_hdf5_timestep = open_or_create_group (m_file.get_hdf5_file(), m_name.c_str()); assert (m_hdf5_timestep >= 0); - write_or_check_attribute (m_hdf5_timestep, "time", time); - - if (CCTK_IsFunctionAliased ("UniqueSimulationID")) { - cGH const * const cctkGH = get_file().get_cctkGH(); - char const * const job_id - = static_cast (UniqueSimulationID (cctkGH)); - write_or_check_attribute (m_hdf5_timestep, "simulation id", job_id); - } + write_or_check_attribute (m_hdf5_timestep, "Time", time); assert (invariant()); } @@ -66,8 +61,7 @@ namespace CarpetIOF5 { timestep_t:: ~ timestep_t() { - herr_t const herr = H5Gclose (m_hdf5_timestep); - assert (not herr); + check (not H5Gclose (m_hdf5_timestep)); } @@ -100,21 +94,20 @@ namespace CarpetIOF5 { void timestep_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) + get_file().get_link_destination (proc, filename, objectname); + if (objectname.empty()) + { + objectname = m_name; + } + else { - initialised = true; - get_file().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; + objectname += string ("/") + m_name; } - filename = l_filename; - objectname = l_objectname; } diff --git a/CarpetDev/CarpetIOF5/src/timestep.hh b/CarpetDev/CarpetIOF5/src/timestep.hh index 3a44ec29b..d70a00024 100644 --- a/CarpetDev/CarpetIOF5/src/timestep.hh +++ b/CarpetDev/CarpetIOF5/src/timestep.hh @@ -53,7 +53,8 @@ namespace CarpetIOF5 { const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; diff --git a/CarpetDev/CarpetIOF5/src/topology.cc b/CarpetDev/CarpetIOF5/src/topology.cc index 401f65271..bc05c73a7 100644 --- a/CarpetDev/CarpetIOF5/src/topology.cc +++ b/CarpetDev/CarpetIOF5/src/topology.cc @@ -62,7 +62,7 @@ namespace CarpetIOF5 { unigrid_topology_t (simulation_t & simulation) : topology_t (simulation) { - char const * const name = "Vertices"; + char const * const name = "uniform"; m_name = string (name); m_hdf5_topology = open_or_create_group (m_simulation.get_hdf5_simulation(), name); @@ -94,6 +94,7 @@ namespace CarpetIOF5 { mesh_refinement_topology_t:: mesh_refinement_topology_t (simulation_t & simulation, int const map, + int const maps, int const refinement_level, int const max_refinement_levels, vect const & level_refinement_factor, @@ -110,7 +111,15 @@ namespace CarpetIOF5 { assert (all (level_refinement_factor <= max_refinement_factor)); ostringstream namebuf; - namebuf << "Vertices map=" << map << " level=" << refinement_level; + namebuf << "Vertices"; + if (maps > 1) + { + namebuf << "-map" << map; + } + if (max_refinement_levels > 1) + { + namebuf << "-level" << refinement_level; + } string const namestr = namebuf.str(); m_name = namestr; char const * const name = namestr.c_str(); @@ -157,21 +166,20 @@ namespace CarpetIOF5 { void topology_t:: - get_link_destination (string & filename, + get_link_destination (int const proc, + string & filename, string & objectname) const { - static bool initialised = false; - static string l_filename; - static string l_objectname; - if (not initialised) + get_simulation().get_link_destination (proc, filename, objectname); + if (objectname.empty()) + { + objectname = m_name; + } + else { - initialised = true; - get_simulation().get_link_destination (l_filename, l_objectname); - l_objectname += string ("/") + m_name; + objectname += string ("/") + m_name; } - filename = l_filename; - objectname = l_objectname; } diff --git a/CarpetDev/CarpetIOF5/src/topology.hh b/CarpetDev/CarpetIOF5/src/topology.hh index df841107b..e179e71db 100644 --- a/CarpetDev/CarpetIOF5/src/topology.hh +++ b/CarpetDev/CarpetIOF5/src/topology.hh @@ -45,7 +45,8 @@ namespace CarpetIOF5 { const; void - get_link_destination (string & filename, + get_link_destination (int proc, + string & filename, string & objectname) const; @@ -97,6 +98,7 @@ namespace CarpetIOF5 { mesh_refinement_topology_t (simulation_t & simulation, int map, + int maps, int refinement_level, int max_refinement_levels, vect const & level_refinement_factors, diff --git a/CarpetDev/CarpetIOF5/src/utils.cc b/CarpetDev/CarpetIOF5/src/utils.cc index b57e06a9e..d311f0992 100644 --- a/CarpetDev/CarpetIOF5/src/utils.cc +++ b/CarpetDev/CarpetIOF5/src/utils.cc @@ -1,11 +1,23 @@ #include #include #include +#include #include +// force HDF5 1.8.x installations to use the new API +#define H5Acreate_vers 2 +#define H5Gcreate_vers 2 +#define H5Gopen_vers 2 +#define H5Tarray_create_vers 2 + +#include + #include "cctk.h" +#include "cctk_Parameters.h" +#include "bbox.hh" #include "defs.hh" +#include "vect.hh" #include "utils.hh" @@ -175,28 +187,96 @@ namespace CarpetIOF5 { + template + string + name_from_ivect (vect const & ivect) + { + ostringstream buf; + for (int d = 0; d < dim; ++ d) + { + if (d > 0) + { + buf << ":"; + } + buf << ivect[d]; + } + return buf.str (); + } + + template + string + name_from_ivect (vect const & ivect); + + template + string + name_from_ivect (vect const & ivect); + + + + template + string + name_from_ibbox (bbox const & ibbox) + { + ostringstream buf; + buf << name_from_ivect (ibbox.lower ()) + << "-" + << name_from_ivect (ibbox.upper ()) + << "-" + << name_from_ivect (ibbox.stride ()); + return buf.str (); + } + + template + string + name_from_ibbox (bbox const & ivect); + + + hid_t open_or_create_group (hid_t const where, char const * const name) { + DECLARE_CCTK_PARAMETERS; + assert (where >= 0); assert (name != 0); + // bool group_exists; + // H5E_BEGIN_TRY { + // H5G_stat_t statbuf; + // herr_t const herr = H5Gget_objinfo (where, name, true, & statbuf); + // group_exists = herr == 0; + // } H5E_END_TRY; bool group_exists; H5E_BEGIN_TRY { - H5G_stat_t statbuf; + H5G_info_t groupinfo; herr_t const herr - = H5Gget_objinfo (where, name, true, & statbuf); + = H5Gget_info_by_name (where, name, & groupinfo, H5P_DEFAULT); group_exists = herr == 0; } H5E_END_TRY; + if (veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "Group \"%s\" %s", + name, + group_exists ? "exists" : "does not exist"); + } hid_t group; if (group_exists) { + if (veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "H5Gopen (name=\"%s\")", name); + } group = H5Gopen (where, name, H5P_DEFAULT); } else { + if (veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, "H5Gcreate (name=\"%s\")", name); + } group = H5Gcreate (where, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } diff --git a/CarpetDev/CarpetIOF5/src/utils.hh b/CarpetDev/CarpetIOF5/src/utils.hh index ae7cb65a1..8d9d0a925 100644 --- a/CarpetDev/CarpetIOF5/src/utils.hh +++ b/CarpetDev/CarpetIOF5/src/utils.hh @@ -1,24 +1,6 @@ #ifndef UTILS_HH #define UTILS_HH -// force HDF5 1.8.x installations to use the new API -#ifdef H5Gopen_vers -#undef H5Gopen_vers -#endif -#ifdef H5Gcreate_vers -#undef H5Gcreate_vers -#endif -#ifdef H5Acreate_vers -#undef H5Acreate_vers -#endif -#ifdef H5Tarray_create_vers -#undef H5Tarray_create_vers -#endif -#define H5Gopen_vers 2 -#define H5Gcreate_vers 2 -#define H5Acreate_vers 2 -#define H5Tarray_create_vers 2 - #include #include "cctk.h" @@ -69,6 +51,16 @@ namespace CarpetIOF5 { + template + string + name_from_ivect (vect const & ivect); + + template + string + name_from_ibbox (bbox const & ibbox); + + + hid_t open_or_create_group (hid_t where, char const * name); diff --git a/CarpetDev/CarpetIOF5/src/writer-old.cc b/CarpetDev/CarpetIOF5/src/writer-old.cc new file mode 100644 index 000000000..4c4f639a8 --- /dev/null +++ b/CarpetDev/CarpetIOF5/src/writer-old.cc @@ -0,0 +1,511 @@ +#include +#include +#include + +#include + +#include "cctk.h" +#include "cctk_Arguments.h" +#include "cctk_Parameters.h" + +#include "carpet.hh" + +#include "data_region.hh" +#include "file.hh" +#include "meta_data_region.hh" +#include "physical_quantity.hh" +#include "simulation.hh" +#include "tensor_component.hh" +#include "timestep.hh" +#include "topology.hh" +#include "writer.hh" + + + +namespace CarpetIOF5 { + + writer_t:: + writer_t (cGH const * const cctkGH, + int const variable) + : m_cctkGH (cctkGH), + m_variable (variable) + { + } + + + + void writer_t:: + write (F5::file_t & file) + const + { + write_meta (file); + } + + + + void writer_t:: + write_meta (F5::file_t & file) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_meta"); + } + + F5::timestep_t timestep (file, m_cctkGH->cctk_time); + + if (Carpet::is_meta_mode()) + { + for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH); + not mglevel_iter.done(); + mglevel_iter.step()) + { + write_one_mglevel (timestep); + } + } + else + { + write_one_mglevel (timestep); + } + } + + + + void writer_t:: + write_one_mglevel (F5::timestep_t & timestep) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_mglevel mglevel=%d", + Carpet::mglevel); + } + + // ostringstream namebuf; + // namebuf << sim_id; + // if (m_cctkGH->cctk_convlevel != 0) + // { + // namebuf << "-convlevel" << m_cctkGH->cctk_convlevel; + // } + // string const name = namebuf.str(); + // F5::simulation_t simulation (timestep, name.c_str()); + + int const grouptype = CCTK_GroupTypeFromVarI (m_variable); + assert (grouptype >= 0); + switch (grouptype) + { + case CCTK_ARRAY: + case CCTK_SCALAR: + { + if (Carpet::is_global_mode()) + { + int const rl = 0; + enter_level_mode (m_cctkGH, rl); + write_global_reflevel (timestep); + leave_level_mode (m_cctkGH); + } + else + { + if (Carpet::do_global_mode) + { + write_global_reflevel (timestep); + } + } + } + break; + case CCTK_GF: + { + if (Carpet::is_global_mode()) + { + for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH); + not reflevel_iter.done(); + reflevel_iter.step()) + { + write_one_reflevel (timestep); + } + } + else + { + write_one_reflevel (simulation); + } + } + break; + default: + assert (0); + } + } + + + + void writer_t:: + write_global_reflevel (F5::timestep_t & timestep) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_INFO ("OutputVarAs/write_global_reflevel"); + } + + if (Carpet::is_level_mode()) + { + for (Carpet::map_iterator map_iter (m_cctkGH, grouptype); + not map_iter.done(); + map_iter.step()) + { + write_global_map (timestep); + } + } + else + { + write_global_map (timestep); + } + } + + + + void writer_t:: + write_global_map (F5::timestep_t & timestep) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_INFO ("OutputVarAs/write_global_map"); + } + + write_global_component (timestep); + } + + + + void writer_t:: + write_global_component (F5::timestep_t & timestep) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_INFO ("OutputVarAs/write_global_component"); + } + + int const group = CCTK_GroupIndexFromVarI (m_variable); + assert (group >= 0 and group < CCTK_NumGroups()); + cGroup groupdata; + check (not CCTK_GropuData (group, & groupdata)); + assert (groupdata.grouptype == CCTK_SCALAR + or groupdata.grouptype == CCTK_ARRAY); + + for (Carpet::component_iterator component_iter (m_cctkGH, grouptype); + not component_iter.done(); + component_iter.step()) + + + + int const myproc = CCTK_MyProc (m_cctkGH); + if (groupdata.disttype == CCTK_DISTRIB_CONSTANT and myproc != 0) + { + // Output DISTRIB=constant groups only on the root processor + return; + } + + // Name the grid after the variable group + char * const c_name = CCTK_GroupNameFromVarI (m_variable); + string const name = c_name; + free (c_name); + F5::simulation_t simulation (timestep, name.c_str()); + + F5::unigrid_topology_t topology (simulation); + + vect const level_origin (0.0), level_delta (1.0); + F5::Cartesian_coordinate_system_t coordinate_system + (topology, level_origin, level_delta); + + F5::physical_quantity_t physical_quantity (coordinate_system, group); + + int const map = 0; + int const reflevel = 0; + dh * const dd = Carpet::arrdata.at(group).at(map).dd; + dh::dboxes const & boxes + = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc); + bbox const & region = determine_region (boxes); + + F5::file_t & file = timestep.get_file(); + bool const have_metafile + = file.get_is_metafile() and not file.get_is_datafile(); + if (have_metafile) + { + F5::meta_data_region_t meta_data_region (physical_quantity, region); + gh * const hh = Carpet::vhh.at(Carpet::map); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + + + + void writer_t:: + write_global (F5::timestep_t & timestep) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_INFO ("OutputVarAs/write_global"); + } + + int const group = CCTK_GroupIndexFromVarI (m_variable); + assert (group >= 0 and group < CCTK_NumGroups()); + cGroup groupdata; + check (not CCTK_GropuData (group, & groupdata)); + assert (groupdata.grouptype == CCTK_SCALAR + or groupdata.grouptype == CCTK_ARRAY); + + for (Carpet::component_iterator component_iter (m_cctkGH, grouptype); + not component_iter.done(); + component_iter.step()) + + + + int const myproc = CCTK_MyProc (m_cctkGH); + if (groupdata.disttype == CCTK_DISTRIB_CONSTANT and myproc != 0) + { + // Output DISTRIB=constant groups only on the root processor + return; + } + + // Name the grid after the variable group + char * const c_name = CCTK_GroupNameFromVarI (m_variable); + string const name = c_name; + free (c_name); + F5::simulation_t simulation (timestep, name.c_str()); + + F5::unigrid_topology_t topology (simulation); + + vect const level_origin (0.0), level_delta (1.0); + F5::Cartesian_coordinate_system_t coordinate_system + (topology, level_origin, level_delta); + + F5::physical_quantity_t physical_quantity (coordinate_system, group); + + int const map = 0; + int const reflevel = 0; + dh * const dd = Carpet::arrdata.at(group).at(map).dd; + dh::dboxes const & boxes + = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc); + bbox const & region = determine_region (boxes); + + F5::file_t & file = timestep.get_file(); + bool const have_metafile + = file.get_is_metafile() and not file.get_is_datafile(); + if (have_metafile) + { + F5::meta_data_region_t meta_data_region (physical_quantity, region); + gh * const hh = Carpet::vhh.at(Carpet::map); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + + + + void writer_t:: + write_one_map (F5::simulation_t & simulation) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_map map=%d", Carpet::map); + } + + F5::mesh_refinement_topology_t topology + (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels, + Carpet::spacereflevelfact, Carpet::maxspacereflevelfact); + + vect level_origin, level_delta; + for (int d=0; d= 0 and group < CCTK_NumGroups()); + F5::physical_quantity_t physical_quantity (coordinate_system, group); + + if (Carpet::is_singlemap_mode()) + { + int const grouptype = CCTK_GroupTypeI (group); + assert (grouptype >= 0); + + for (Carpet::component_iterator component_iter (m_cctkGH, grouptype); + not component_iter.done(); + component_iter.step()) + { + write_one_component (physical_quantity); + } + } + else + { + write_one_component (physical_quantity); + } + } + + + + void writer_t:: + write_one_reflevel (F5::simulation_t & simulation) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_reflevel reflevel=%d", + Carpet::reflevel); + } + + int const grouptype = CCTK_GroupTypeFromVarI (m_variable); + assert (grouptype >= 0); + assert (grouptype == CCTK_GF); + + if (Carpet::is_level_mode()) + { + for (Carpet::map_iterator map_iter (m_cctkGH, grouptype); + not map_iter.done(); + map_iter.step()) + { + write_one_map (simulation); + } + } + else + { + write_one_map (simulation); + } + } + + + + void writer_t:: + write_one_component (F5::physical_quantity_t & physical_quantity) + const + { + DECLARE_CCTK_PARAMETERS; + + if (verbose or veryverbose) + { + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_one_component component=%d", + Carpet::component); + } + + F5::file_t & file + = (physical_quantity.get_coordinate_system().get_topology() + .get_simulation().get_timestep().get_file()); + bool const have_metafile + = file.get_is_metafile() and not file.get_is_datafile(); + gh * const hh = Carpet::vhh.at(Carpet::map); + bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component); + if (have_metafile or is_local) + { + dh * const dd = Carpet::vdd.at(Carpet::map); + bbox const & region + = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel) + .at(Carpet::component).exterior); + + if (have_metafile) + { + F5::meta_data_region_t meta_data_region (physical_quantity, region); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + + if (is_local) + { + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + } + } + + + + bbox const & writer_t:: + determine_region (dh::dboxes const & boxes) + const + { + DECLARE_CCTK_PARAMETERS; + + // TODO: use superregions instead of regions (? only if the + // regions are on the same processor?) + + bbox dh::dboxes::* boxptr; + if (CCTK_EQUALS (output_regions, "exterior")) + { + boxptr = & dh::dboxes::exterior; + } + else if (CCTK_EQUALS (output_regions, "owned")) + { + boxptr = & dh::dboxes::owned; + } + else if (CCTK_EQUALS (output_regions, "interior")) + { + boxptr = & dh::dboxes::interior; + } + else + { + assert (0); + } + + return boxes.*boxptr; + } + +} // namespace CarpetIOF5 diff --git a/CarpetDev/CarpetIOF5/src/writer.cc b/CarpetDev/CarpetIOF5/src/writer.cc index df1949369..cf00271a0 100644 --- a/CarpetDev/CarpetIOF5/src/writer.cc +++ b/CarpetDev/CarpetIOF5/src/writer.cc @@ -1,22 +1,29 @@ +#include #include +#include + +#include #include "cctk.h" #include "cctk_Arguments.h" #include "cctk_Parameters.h" #include "carpet.hh" +#include "modes.hh" #include "data_region.hh" #include "file.hh" #include "meta_data_region.hh" +#include "physical_quantity.hh" #include "simulation.hh" #include "tensor_component.hh" #include "timestep.hh" #include "topology.hh" +#include "utils.hh" #include "writer.hh" - + namespace CarpetIOF5 { writer_t:: @@ -33,20 +40,18 @@ namespace CarpetIOF5 { write (F5::file_t & file) const { - write_meta (file, file.get_have_metafile()); + write_meta (file); } void writer_t:: - write_meta (F5::file_t & file, - bool const have_metafile) + write_meta (F5::file_t & file) const { DECLARE_CCTK_PARAMETERS; - if (verbose or veryverbose) - { + if (verbose or veryverbose) { CCTK_VInfo (CCTK_THORNSTRING, "OutputVarAs/write_meta"); } @@ -55,24 +60,22 @@ namespace CarpetIOF5 { if (Carpet::is_meta_mode()) { - for (Carpet::mglevel_iterator mglevel_iter (m_cctkGH); - not mglevel_iter.done(); - mglevel_iter.step()) + BEGIN_MGLEVEL_LOOP (m_cctkGH) { - write_one_mglevel (timestep, have_metafile); + write_one_mglevel (timestep); } + END_MGLEVEL_LOOP; } else { - write_one_mglevel (timestep, have_metafile); + write_one_mglevel (timestep); } } void writer_t:: - write_one_mglevel (F5::timestep_t & timestep, - bool const have_metafile) + write_one_mglevel (F5::timestep_t & timestep) const { DECLARE_CCTK_PARAMETERS; @@ -84,12 +87,6 @@ namespace CarpetIOF5 { Carpet::mglevel); } - ostringstream namebuf; - namebuf << "convlevel=" << m_cctkGH->cctk_convlevel; - string const namestr = namebuf.str(); - char const * const name = namestr.c_str(); - F5::simulation_t simulation (timestep, name); - int const grouptype = CCTK_GroupTypeFromVarI (m_variable); assert (grouptype >= 0); switch (grouptype) @@ -97,9 +94,20 @@ namespace CarpetIOF5 { case CCTK_ARRAY: case CCTK_SCALAR: { - if (Carpet::do_global_mode) + if (Carpet::is_global_mode()) { - write_global (simulation, have_metafile); + ENTER_LEVEL_MODE (m_cctkGH, 0) + { + write_global (timestep); + } + LEAVE_LEVEL_MODE; + } + else + { + if (Carpet::do_global_mode) + { + write_global (timestep); + } } } break; @@ -107,16 +115,15 @@ namespace CarpetIOF5 { { if (Carpet::is_global_mode()) { - for (Carpet::reflevel_iterator reflevel_iter (m_cctkGH); - not reflevel_iter.done(); - reflevel_iter.step()) + BEGIN_REFLEVEL_LOOP (m_cctkGH) { - write_one_reflevel (simulation, have_metafile); + write_one_reflevel (timestep); } + END_REFLEVEL_LOOP; } else { - write_one_reflevel (simulation, have_metafile); + write_one_reflevel (timestep); } } break; @@ -128,69 +135,96 @@ namespace CarpetIOF5 { void writer_t:: - write_global (F5::simulation_t & simulation, - bool const have_metafile) + write_global (F5::timestep_t & timestep) const { DECLARE_CCTK_PARAMETERS; if (verbose or veryverbose) { - CCTK_INFO ("OutputVarAs/write_global"); + CCTK_VInfo (CCTK_THORNSTRING, + "OutputVarAs/write_global"); } - F5::unigrid_topology_t topology (simulation); - int const grouptype = CCTK_GroupTypeFromVarI (m_variable); - assert (grouptype >= 0); assert (grouptype == CCTK_SCALAR or grouptype == CCTK_ARRAY); - vect level_origin, level_delta; - for (int d=0; d= 0 and group < CCTK_NumGroups()); - F5::physical_quantity_t physical_quantity (coordinate_system, group); - - F5::tensor_component_t tensor_component (physical_quantity, m_variable); - - int const map = 0; - int const reflevel = 0; - int const myproc = CCTK_MyProc (m_cctkGH); - dh * const dd = Carpet::arrdata.at(group).at(map).dd; - dh::dboxes const & boxes - = dd->boxes.at(Carpet::mglevel).at(reflevel).at(myproc); - bbox const & region = determine_region (boxes); - - if (have_metafile) + BEGIN_MAP_LOOP (m_cctkGH, grouptype) { - F5::meta_data_region_t meta_data_region (tensor_component, region); - gh * const hh = Carpet::vhh.at(Carpet::map); - int const proc = hh->processor (Carpet::reflevel, Carpet::component); - meta_data_region.write (proc); + + // Name the grid after the variable group + ostringstream namebuf; + char * const c_name = CCTK_GroupNameFromVarI (m_variable); + namebuf << "Cactus-" << c_name; + free (c_name); + string const name = namebuf.str(); + F5::simulation_t simulation (timestep, name.c_str()); + + F5::unigrid_topology_t topology (simulation); + + vect const level_origin (0.0), level_delta (1.0); + F5::Cartesian_coordinate_system_t coordinate_system + (topology, level_origin, level_delta); + + int const group = CCTK_GroupIndexFromVarI (m_variable); + assert (group >= 0 and group < CCTK_NumGroups()); + F5::physical_quantity_t physical_quantity (coordinate_system, group); + + int const myproc = CCTK_MyProc (m_cctkGH); + + F5::file_t & file = timestep.get_file(); + bool const write_metafile + = file.get_is_metafile() and not file.get_is_datafile(); + if (write_metafile) + { + + BEGIN_COMPONENT_LOOP (m_cctkGH, grouptype) + { + dh * const dd = Carpet::arrdata.at(group).at(Carpet::map).dd; + dh::dboxes const & boxes + = dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel).at(myproc); + bbox const & region = determine_region (boxes); + F5::meta_data_region_t meta_data_region (physical_quantity, region); + + gh * const hh = Carpet::vhh.at(Carpet::map); + int const proc = hh->processor (Carpet::reflevel, Carpet::component); + meta_data_region.write (proc); + } + END_COMPONENT_LOOP; + + } + else // if not write_metafile + { + + BEGIN_LOCAL_COMPONENT_LOOP (m_cctkGH, grouptype) + { + dh * const dd = Carpet::arrdata.at(group).at(Carpet::map).dd; + dh::dboxes const & boxes + = dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel).at(myproc); + bbox const & region = determine_region (boxes); + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); + int const timelevel = 0; + void const * const varptr + = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); + assert (varptr != 0); + int const vartype = CCTK_VarTypeI (m_variable); + assert (vartype >= 0); + tensor_component.write (varptr, vartype); + } + END_LOCAL_COMPONENT_LOOP; + + } // if not write_metafile + } - - F5::data_region_t data_region (tensor_component, region); - int const timelevel = 0; - void const * const varptr - = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); - assert (varptr != 0); - int const vartype = CCTK_VarTypeI (m_variable); - assert (vartype >= 0); - data_region.write (varptr, vartype); + END_MAP_LOOP; } void writer_t:: - write_one_reflevel (F5::simulation_t & simulation, - bool const have_metafile) + write_one_reflevel (F5::timestep_t & timestep) const { DECLARE_CCTK_PARAMETERS; @@ -203,29 +237,26 @@ namespace CarpetIOF5 { } int const grouptype = CCTK_GroupTypeFromVarI (m_variable); - assert (grouptype >= 0); assert (grouptype == CCTK_GF); if (Carpet::is_level_mode()) { - for (Carpet::map_iterator map_iter (m_cctkGH, grouptype); - not map_iter.done(); - map_iter.step()) + BEGIN_MAP_LOOP (m_cctkGH, grouptype) { - write_one_map (simulation, have_metafile); + write_one_map (timestep); } + END_MAP_LOOP; } else { - write_one_map (simulation, have_metafile); + write_one_map (timestep); } } void writer_t:: - write_one_map (F5::simulation_t & simulation, - bool const have_metafile) + write_one_map (F5::timestep_t & timestep) const { DECLARE_CCTK_PARAMETERS; @@ -236,8 +267,19 @@ namespace CarpetIOF5 { "OutputVarAs/write_one_map map=%d", Carpet::map); } + // Name the grid after the map number + ostringstream namebuf; + namebuf << "Carpet"; + if (Carpet::maps > 1) + { + namebuf << "-map" << Carpet::map; + } + string const name = namebuf.str(); + F5::simulation_t simulation (timestep, name.c_str()); + F5::mesh_refinement_topology_t topology - (simulation, Carpet::map, Carpet::reflevel, Carpet::maxreflevels, + (simulation, + Carpet::map, Carpet::maps, Carpet::reflevel, Carpet::maxreflevels, Carpet::spacereflevelfact, Carpet::maxspacereflevelfact); vect level_origin, level_delta; @@ -255,31 +297,27 @@ namespace CarpetIOF5 { assert (group >= 0 and group < CCTK_NumGroups()); F5::physical_quantity_t physical_quantity (coordinate_system, group); - F5::tensor_component_t tensor_component (physical_quantity, m_variable); - if (Carpet::is_singlemap_mode()) { int const grouptype = CCTK_GroupTypeI (group); assert (grouptype >= 0); - for (Carpet::component_iterator component_iter (m_cctkGH, grouptype); - not component_iter.done(); - component_iter.step()) + BEGIN_COMPONENT_LOOP (m_cctkGH, grouptype) { - write_one_component (tensor_component, have_metafile); + write_one_component (physical_quantity); } + END_COMPONENT_LOOP; } else { - write_one_component (tensor_component, have_metafile); + write_one_component (physical_quantity); } } void writer_t:: - write_one_component (F5::tensor_component_t & tensor_component, - bool const have_metafile) + write_one_component (F5::physical_quantity_t & physical_quantity) const { DECLARE_CCTK_PARAMETERS; @@ -293,30 +331,37 @@ namespace CarpetIOF5 { gh * const hh = Carpet::vhh.at(Carpet::map); bool const is_local = hh->is_local (Carpet::reflevel, Carpet::component); - if (have_metafile or is_local) + + F5::file_t & file + = (physical_quantity.get_coordinate_system().get_topology() + .get_simulation().get_timestep().get_file()); + bool const write_metafile + = file.get_is_metafile() and not file.get_is_datafile(); + if (write_metafile or is_local) { dh * const dd = Carpet::vdd.at(Carpet::map); bbox const & region = (dd->boxes.at(Carpet::mglevel).at(Carpet::reflevel) .at(Carpet::component).exterior); - if (have_metafile) + if (write_metafile) { - F5::meta_data_region_t meta_data_region (tensor_component, region); + F5::meta_data_region_t meta_data_region (physical_quantity, region); int const proc = hh->processor (Carpet::reflevel, Carpet::component); meta_data_region.write (proc); } - - if (is_local) + else if (is_local) { - F5::data_region_t data_region (tensor_component, region); + F5::data_region_t data_region (physical_quantity, region); + + F5::tensor_component_t tensor_component (data_region, m_variable); int const timelevel = 0; void const * const varptr = CCTK_VarDataPtrI (m_cctkGH, timelevel, m_variable); assert (varptr != 0); int const vartype = CCTK_VarTypeI (m_variable); assert (vartype >= 0); - data_region.write (varptr, vartype); + tensor_component.write (varptr, vartype); } } } @@ -325,19 +370,17 @@ namespace CarpetIOF5 { bbox const & writer_t:: determine_region (dh::dboxes const & boxes) - const { DECLARE_CCTK_PARAMETERS; + // TODO: use superregions instead of regions (? only if the + // regions are on the same processor?) + bbox dh::dboxes::* boxptr; if (CCTK_EQUALS (output_regions, "exterior")) { boxptr = & dh::dboxes::exterior; } - else if (CCTK_EQUALS (output_regions, "communicated")) - { - boxptr = & dh::dboxes::communicated; - } else if (CCTK_EQUALS (output_regions, "owned")) { boxptr = & dh::dboxes::owned; diff --git a/CarpetDev/CarpetIOF5/src/writer.hh b/CarpetDev/CarpetIOF5/src/writer.hh index 94b31d3cd..77bad3915 100644 --- a/CarpetDev/CarpetIOF5/src/writer.hh +++ b/CarpetDev/CarpetIOF5/src/writer.hh @@ -6,10 +6,11 @@ #include "bbox.hh" #include "defs.hh" #include "dh.hh" +#include "vect.hh" #include "file.hh" +#include "physical_quantity.hh" #include "simulation.hh" -#include "tensor_component.hh" #include "timestep.hh" #include "topology.hh" @@ -34,38 +35,33 @@ namespace CarpetIOF5 { private: void - write_meta (F5::file_t & file, - bool have_metafile) + write_meta (F5::file_t & file) const; void - write_one_mglevel (F5::timestep_t & timestep, - bool have_metafile) + write_one_mglevel (F5::timestep_t & timestep) const; void - write_global (F5::simulation_t & simulation, - bool have_metafile) + write_global (F5::timestep_t & timestep) const; void - write_one_reflevel (F5::simulation_t & simulation, - bool have_metafile) + write_one_reflevel (F5::timestep_t & timestep) const; void - write_one_map (F5::simulation_t & simulation, - bool have_metafile) + write_one_map (F5::timestep_t & timestep) const; void - write_one_component (F5::tensor_component_t & tensor_component, - bool have_metafile) + write_one_component (F5::physical_quantity_t & physical_quantity) const; - bbox const & - determine_region (dh::dboxes const & boxes) - const; + + + static bbox const & + determine_region (dh::dboxes const & boxes); }; -- cgit v1.2.3