From 1d0636369dd1030d0c1e1908f14b3fa55838d364 Mon Sep 17 00:00:00 2001 From: tradke Date: Mon, 14 Jan 2002 14:29:06 +0000 Subject: Added new INTEGER parameter IOHDF5Util::compression_level (ranging from 0..9) to set the compression rate of HDF5 datasets. 0 (which is the default) disables compression, 1..9 set the level the same way as for gzip (higher levels compress better but are more expensive to compute). git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5Util/trunk@48 7842ec3a-9562-4be5-9c5b-06ba18f2b668 --- param.ccl | 9 +++++++++ src/DumpVar.c | 55 +++++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/param.ccl b/param.ccl index 0d2e70b..b9e3052 100644 --- a/param.ccl +++ b/param.ccl @@ -1,6 +1,15 @@ # Parameter definitions for thorn IOHDF5Util # $Header$ +##################### +# Compression level +##################### +INT compression_level "Compression level to use for writing HDF5 data" +{ + 0:9 :: "A value of zero disables compression" +} 0 + + ###################### # Hyperslab parameters ###################### diff --git a/src/DumpVar.c b/src/DumpVar.c index a2f9a21..b955599 100644 --- a/src/DumpVar.c +++ b/src/DumpVar.c @@ -709,7 +709,7 @@ static void IOHDF5Util_procDump (const cGH *GH, int myproc; ioGH *ioUtilGH; ioHDF5UtilGH *myGH; - hid_t group, dataset, memspace, filespace, xfer_plist; + hid_t group, dataset, memspace, filespace, plist; char *fullname, *datasetname, *chunkname; hssize_t *chunk_origin; hsize_t *chunk_dims, *file_dims; @@ -776,8 +776,15 @@ static void IOHDF5Util_procDump (const cGH *GH, when writing its own data, otherwise the dataset is reopened */ if (proc == myproc) { + IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + /* enable compression for chunked dataset if compression was requested */ + if (compression_level) + { + IOHDF5_ERROR (H5Pset_chunk (plist, sdim, chunk_dims)); + IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); + } IOHDF5_ERROR (dataset = H5Dcreate (file, datasetname, - iohdf5_type, filespace, H5P_DEFAULT)); + iohdf5_type, filespace, plist)); IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, &geom[2*sdim], request, dataset); } @@ -787,19 +794,19 @@ static void IOHDF5Util_procDump (const cGH *GH, } /* increase the buffer size if the default isn't sufficient */ - IOHDF5_ERROR (xfer_plist = H5Pcreate (H5P_DATASET_XFER)); + IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); buffersize = H5Dget_storage_size (dataset); - if (buffersize > H5Pget_buffer (xfer_plist, NULL, NULL)) + if (buffersize > H5Pget_buffer (plist, NULL, NULL)) { - IOHDF5_ERROR (H5Pset_buffer (xfer_plist, buffersize, NULL, NULL)); + IOHDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); } /* write the data */ IOHDF5_ERROR (H5Dwrite (dataset, iohdf5_type, memspace, filespace, - xfer_plist, outme)); + plist, outme)); /* and close the transfer property list and the file dataspace */ - IOHDF5_ERROR (H5Pclose (xfer_plist)); + IOHDF5_ERROR (H5Pclose (plist)); IOHDF5_ERROR (H5Sclose (filespace)); } else @@ -816,9 +823,17 @@ static void IOHDF5Util_procDump (const cGH *GH, /* now the chunk dataset for each processor is created within the group */ chunkname = (char *) malloc (strlen (datasetname) + 20); sprintf (chunkname, "%s/chunk%d", datasetname, proc - myproc); + IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + /* enable compression for chunked dataset if compression was requested */ + if (compression_level) + { + IOHDF5_ERROR (H5Pset_chunk (plist, sdim, chunk_dims)); + IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); + } /* create the chunk dataset and dump the chunk data */ IOHDF5_ERROR (dataset = H5Dcreate (file, chunkname, - iohdf5_type, memspace, H5P_DEFAULT)); + iohdf5_type, memspace, plist)); + IOHDF5_ERROR (H5Pclose (plist)); IOHDF5_ERROR (H5Dwrite (dataset, iohdf5_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, outme)); @@ -916,7 +931,7 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, DECLARE_CCTK_PARAMETERS int i, dim; ioHDF5GH *myGH; - hid_t dataset, memspace, filespace, xfer_plist; + hid_t dataset, memspace, filespace, plist; char *fullname, *datasetname; hssize_t *chunk_origin; hsize_t *chunk_dims, *file_dims; @@ -964,8 +979,16 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, H5Gunlink (file, datasetname); IOHDF5_ERROR (H5Eset_auto (myGH->print_error_fn, myGH->print_error_fn_arg)); } + /* enable compression for chunked dataset if compression was requested */ + IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + if (compression_level) + { + IOHDF5_ERROR (H5Pset_chunk (plist, sdim, chunk_dims)); + IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); + } IOHDF5_ERROR (dataset = H5Dcreate (file, datasetname, hdf5io_type, filespace, - H5P_DEFAULT)); + plist)); + IOHDF5_ERROR (H5Pclose (plist)); if (CCTK_MyProc (GH) == 0) { IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, &geom[2*sdim], @@ -973,19 +996,19 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, } /* increase the buffer size if the default isn't sufficient */ - IOHDF5_ERROR (xfer_plist = H5Pcreate (H5P_DATASET_XFER)); + IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); buffersize = H5Dget_storage_size (dataset); - if (buffersize > H5Pget_buffer (xfer_plist, NULL, NULL)) + if (buffersize > H5Pget_buffer (plist, NULL, NULL)) { - IOHDF5_ERROR (H5Pset_buffer (xfer_plist, buffersize, NULL, NULL)); + IOHDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); } /* write the data */ - IOHDF5_ERROR (H5Dwrite (dataset, hdf5io_type, memspace, - filespace, xfer_plist, outme)); + IOHDF5_ERROR (H5Dwrite (dataset, hdf5io_type, memspace, filespace, plist, + outme)); /* close resources */ - IOHDF5_ERROR (H5Pclose (xfer_plist)); + IOHDF5_ERROR (H5Pclose (plist)); IOHDF5_ERROR (H5Sclose (filespace)); IOHDF5_ERROR (H5Dclose (dataset)); IOHDF5_ERROR (H5Sclose (memspace)); -- cgit v1.2.3