diff options
Diffstat (limited to 'src/DumpVar.c')
-rw-r--r-- | src/DumpVar.c | 926 |
1 files changed, 311 insertions, 615 deletions
diff --git a/src/DumpVar.c b/src/DumpVar.c index 109df75..96148d9 100644 --- a/src/DumpVar.c +++ b/src/DumpVar.c @@ -2,9 +2,9 @@ @file DumpVar.c @date Fri May 21 1999 @author Thomas Radke - @desc + @desc Routines for writing variables into HDF5 data or checkpoint files. - These routines are used by other HDF5 IO methods. + These routines are used by other HDF5 I/O methods. @enddesc @version $Id$ @@*/ @@ -15,77 +15,40 @@ #include "cctk.h" #include "cctk_Parameters.h" #include "CactusPUGH/PUGH/src/include/pugh.h" -#include "CactusPUGH/PUGHSlab/src/NewPUGHSlab.h" +#include "CactusPUGH/PUGHSlab/src/NewPUGHSlab.h" #include "CactusBase/IOUtil/src/ioGH.h" #include "ioHDF5UtilGH.h" - /* the rcs ID and its dummy function to use it */ static const char *rcsid = "$Header$"; CCTK_FILEVERSION(BetaThorns_IOHDF5Util_DumpVar_c) -/* #define DEBUG_ME 1 */ - -#define IOTAGBASE 20000 /* This may break on more than 2000 processors */ - -/* info structure describing how to dump a given CCTK variable type */ -typedef struct -{ - int iohdf5_type; /* the HDF5 type to use */ - int element_size; /* size of a single data element */ -#ifdef CCTK_MPI - MPI_Datatype mpi_type; /* the data type for MPI communication */ +/******************************************************************** + ******************** Macro Definitions ************************ + ********************************************************************/ +/* uncomment the following line to enable some debugging output */ +/* #define IOHDF5UTIL_DEBUG 1 */ + +/* tag base for MPI messages (this may break on more than 2000 processors) */ +#define MPITAGBASE 20000 + + +/******************************************************************** + ******************** Internal Routines ************************ + ********************************************************************/ +static int WriteGS (const cGH *GH, const ioSlab *slab, const char *name, + hid_t file); +static int WriteGA (const cGH *GH, const ioSlab *slab, const char *name, + hid_t file); +static int GetLocalHyperslab (const cGH *GH, const ioSlab *slab, void **data, + int *free_data); +static void WriteData (const cGH *GH, const ioSlab *slab, const char *name, + const void *data, int proc, hid_t file); +#if defined(CCTK_MPI) && defined(H5_HAVE_PARALLEL) +static void WriteDataCollective (const cGH *GH, const ioSlab *slab, + const char *name, const void *data,hid_t file); #endif -} dumpInfo; - - -/* prototypes of routines defined in this source file */ -static int IOHDF5Util_DumpGS (const cGH *GH, - int vindex, - int timelevel, - int iohdf5_type, - const ioHDF5Geo_t *request, - int check_exisiting_objects, - hid_t file); -static int IOHDF5Util_DumpGA (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const dumpInfo *info, - int check_exisiting_objects, - hid_t file); -static int IOHDF5Util_getDumpData (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - void **outme, - int *free_outme, - CCTK_INT *geom); -static void IOHDF5Util_procDump (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const void *outme, - const CCTK_INT *geom, - int proc, - int iohdf5_type, - int check_exisiting_objects, - hid_t file); -#ifdef CCTK_MPI -#ifdef H5_HAVE_PARALLEL -static void IOHDF5Util_collectiveDump (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const void *outme, - const CCTK_INT *geom, - int hdf5io_type, - int check_exisiting_objects, - hid_t file); -#endif /* H5_HAVE_PARALLEL */ -#endif /* MPI */ - /*@@ @@ -96,27 +59,17 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, Generic dump routine, just calls the type-specific dump routine. @enddesc - @calls IOHDF5Util_DumpGS - IOHDF5Util_DumpGA + @calls WriteGS + WriteGA @var GH - @vdesc Pointer to CCTK grid hierarchy + @vdesc pointer to CCTK grid hierarchy @vtype const cGH * @vio in @endvar - @var vindex - @vdesc index of the variable to be dumped - @vtype int - @vio in - @endvar - @var timelevel - @vdesc the timelevel to dump - @vtype int - @vio in - @endvar - @var request - @vdesc pointer to the IO request structure - @vtype const ioHDF5Geo_t * + @var slab + @vdesc reference to the I/O hyperslab description + @vtype const ioSlab * @vio in @endvar @var file @@ -124,141 +77,83 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, @vtype hid_t @vio in @endvar - @var check_exisiting_objects - @vdesc flag indicating if we should check for already existing objects - before these are created anew - This might happen for existing files reopened after recovery. - @vtype int - @vio in - @endvar @returntype int @returndesc - -1 if variable has unknown type - or return code of either - @seeroutine IOHDF5Util_DumpGS or - @seeroutine IOHDF5Util_DumpGA + -1 if variable has unknown group type + or return code of either @seeroutine WriteGS or + @seeroutine WriteGA @endreturndesc @@*/ -int IOHDF5Util_DumpVar (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - hid_t file, - int check_exisiting_objects) +int IOHDF5Util_DumpVar (const cGH *GH, const ioSlab *slab, hid_t file) { - int vtype, gtype; - pGH *pughGH; - ioHDF5UtilGH *myGH; - ioGH *ioUtilGH; - dumpInfo info; - int retval; + int gtype, retval; + char *fullname, *objectname; - /* Get the handle for PUGH, IOUtil, and IOHDF5Util extensions */ - pughGH = PUGH_pGH (GH); - ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); - myGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util"); + /* build the unique name for the file object to write */ + fullname = CCTK_FullName (slab->vindex); + objectname = (char *) malloc (strlen (fullname) + 80); + sprintf (objectname, "%s timelevel %d at iteration %d", + fullname, slab->timelevel, GH->cctk_iteration); - vtype = CCTK_VarTypeI (vindex); - /* downgrade the output precision if requested */ - if (ioUtilGH->out_single) + /* check whether the object already exists */ + if (slab->check_exist && file >= 0) { - /* Do we only want to downgrade generic CCTK types ? */ -#ifdef CCTK_REAL4 - if (vtype == CCTK_VARIABLE_REAL) - { - vtype = CCTK_VARIABLE_REAL4; - } - else if (vtype == CCTK_VARIABLE_COMPLEX) - { - vtype = CCTK_VARIABLE_COMPLEX8; - } -#endif -#ifdef CCTK_INT4 - if (vtype == CCTK_VARIABLE_INT) + H5E_BEGIN_TRY { - vtype = CCTK_VARIABLE_INT4; - } -#endif - } - info.element_size = CCTK_VarTypeSize (vtype); -#ifdef CCTK_MPI - info.mpi_type = PUGH_MPIDataType (pughGH, vtype); -#endif - info.iohdf5_type = IOHDF5Util_DataType (myGH, vtype); - - if (info.element_size <= 0 || -#ifdef CCTK_MPI - info.mpi_type == MPI_DATATYPE_NULL || -#endif - info.iohdf5_type < 0) - { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Unsupported variable type %d", vtype); - return (-1); + H5Gunlink (file, objectname); + } H5E_END_TRY; } - /* now branch to the appropriate dump routine */ - gtype = CCTK_GroupTypeFromVarI (vindex); + /* branch to the appropriate dump routine */ + gtype = CCTK_GroupTypeFromVarI (slab->vindex); if (gtype == CCTK_SCALAR) { - retval = IOHDF5Util_DumpGS (GH, vindex, timelevel, info.iohdf5_type, - request, check_exisiting_objects, file); + retval = WriteGS (GH, slab, objectname, file); } else if (gtype == CCTK_ARRAY || gtype == CCTK_GF) { - retval = IOHDF5Util_DumpGA (GH, vindex, timelevel, request, &info, - check_exisiting_objects, file); + retval = WriteGA (GH, slab, objectname, file); } else { CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Invalid group type %d", gtype); + "Invalid group type %d for variable '%s'", gtype, fullname); retval = -1; } + /* clean up */ + free (objectname); + free (fullname); + return (retval); } +/******************************************************************** + ******************** Internal Routines ************************ + ********************************************************************/ /*@@ - @routine IOHDF5Util_DumpGS + @routine WriteGS @date May 21 1999 @author Thomas Radke @desc - Dumps a CCTK_SCALAR type variable into a HDF5 file. - @enddesc + Writes a CCTK_SCALAR type variable into a HDF5 file. + @enddesc @var GH - @vdesc Pointer to CCTK grid hierarchy + @vdesc pointer to CCTK grid hierarchy @vtype const cGH * @vio in @endvar - @var vindex - @vdesc index of the variable to be dumped - @vtype int - @vio in - @endvar - @var timelevel - @vdesc the timelevel to dump - @vtype int - @vio in - @endvar - @var iohdf5_type - @vdesc the HDF5 datatype - @vtype int - @vio in - @endvar - @var request - @vdesc pointer to the IO request structure - @vtype const ioHDF5Geo_t * + @var slab + @vdesc reference to the I/O hyperslab description + @vtype const ioSlab * @vio in @endvar - @var check_exisiting_objects - @vdesc flag indicating if we should check for already existing objects - before these are created anew - This might happen for existing files reopened after recovery. - @vtype int + @var name + @vdesc name of the dataset to write + @vtype const char * @vio in @endvar @var file @@ -272,113 +167,68 @@ int IOHDF5Util_DumpVar (const cGH *GH, 0 for success, or -1 if file handle is invalid @endreturndesc @@*/ -static int IOHDF5Util_DumpGS (const cGH *GH, - int vindex, - int timelevel, - int iohdf5_type, - const ioHDF5Geo_t *request, - int check_exisiting_objects, - hid_t file) +static int WriteGS (const cGH *GH, const ioSlab *slab, const char *name, + hid_t file) { ioGH *ioUtilGH; ioHDF5UtilGH *myGH; - hid_t dataset; - char *fullname; - char *datasetname; - static CCTK_INT global_shape[1] = {0}; - + hid_t dataset, hdf5type; - /* immediately return if file handle is invalid */ - if (file < 0) - { - return (-1); - } - /* Get the handles for IOHDF5Util and IOUtil extensions */ ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); myGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util"); + /* only I/O processors write data */ if (CCTK_MyProc (GH) != ioUtilGH->ioproc) { return (0); } - fullname = CCTK_FullName (vindex); - datasetname = (char *) malloc (strlen (fullname) + 80); - sprintf (datasetname, "%s timelevel %d at iteration %d", - fullname, timelevel, GH->cctk_iteration); - - /* if restart from recovery delete an already existing dataset - so that it can be created as anew */ - if (check_exisiting_objects) + /* check if file handle is invalid */ + if (file < 0) { - IOHDF5_ERROR (H5Eset_auto (NULL, NULL)); - H5Gunlink (file, datasetname); - IOHDF5_ERROR (H5Eset_auto (myGH->print_error_fn, myGH->print_error_fn_arg)); + return (-1); } - IOHDF5_ERROR (dataset = H5Dcreate (file, datasetname, iohdf5_type, - myGH->scalar_dataspace, H5P_DEFAULT)); - IOHDF5_ERROR (H5Dwrite (dataset, iohdf5_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, - CCTK_VarDataPtrI (GH, timelevel, vindex))); - - IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, global_shape, - request, dataset); + hdf5type = IOHDF5Util_DataType (myGH, slab->hdatatype); + HDF5_ERROR (dataset = H5Dcreate (file, name, hdf5type, myGH->scalar_dataspace, + H5P_DEFAULT)); + HDF5_ERROR (H5Dwrite (dataset, hdf5type, H5S_ALL, H5S_ALL, H5P_DEFAULT, + CCTK_VarDataPtrI (GH, slab->timelevel, slab->vindex))); - IOHDF5_ERROR (H5Dclose (dataset)); + IOHDF5Util_DumpCommonAttributes (GH, slab, dataset); - free (datasetname); - free (fullname); + HDF5_ERROR (H5Dclose (dataset)); return (0); } /*@@ - @routine IOHDF5Util_DumpGA + @routine WriteGA @date May 21 1999 @author Paul Walker @desc - Dumps a CCTK_GF or CCTK_ARRAY type variable into a HDF5 file. - @enddesc + Writes a grid array into a HDF5 file. + @enddesc - @calls IOHDF5Util_getDumpData - IOHDF5Util_procDump - IOHDF5Util_collectiveDump + @calls GetLocalHyperslab + WriteData + WriteDataCollective @var GH - @vdesc Pointer to CCTK grid hierarchy + @vdesc pointer to CCTK grid hierarchy @vtype const cGH * @vio in @endvar - @var vindex - @vdesc index of the variable to be dumped - @vtype int - @vio in - @endvar - @var timelevel - @vdesc the timelevel to dump - @vtype int - @vio in - @endvar - @var request - @vdesc pointer to the IO request structure - @vtype const ioHDF5Geo_t * - @vio in - @endvar - @var info - @vdesc info structure describing - - the HDF5 datatype to store - - the size of an element of variable - - the MPI datatype to communicate - @vtype const dumpInfo * + @var slab + @vdesc reference to the I/O hyperslab description + @vtype const ioSlab * @vio in @endvar - @var check_exisiting_objects - @vdesc flag indicating if we should check for already existing objects - before these are created anew - This might happen for existing files reopened after recovery. - @vtype int + @var name + @vdesc name of the dataset to write + @vtype const char * @vio in @endvar @var file @@ -390,45 +240,36 @@ static int IOHDF5Util_DumpGS (const cGH *GH, @returntype int @returndesc 0 for success - or returncode of @seeroutine IOHDF5Util_getDumpData + or returncode of @seeroutine GetLocalHyperslab @endreturndesc @@*/ -static int IOHDF5Util_DumpGA (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const dumpInfo *info, - int check_exisiting_objects, - hid_t file) +static int WriteGA (const cGH *GH, const ioSlab *slab, const char *name, + hid_t file) { ioGH *ioUtilGH; - pGH *pughGH; - int myproc; - int nprocs; - CCTK_INT *geom; /* Lower bounds, sizes and global shape of data */ - void *outme; /* The data pointer to dump ... */ - int free_outme; /* and whether it needs freeing */ - int retval; + ioHDF5UtilGH *myGH; + int myproc, nprocs, retval; + void *data; /* The data pointer to dump ... */ + int free_data; /* and whether it needs freeing */ #ifdef CCTK_MPI void *tmpd; - int incoming; - int outgoing; - int i, j; + int i, j, incoming, outgoing; + MPI_Comm comm; MPI_Status ms; + MPI_Datatype mpitype; #endif DECLARE_CCTK_PARAMETERS - /* Get the handles for PUGH and IOUtil extensions */ - pughGH = PUGH_pGH (GH); + myGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util"); ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); - /* allocate the geometry buffer */ - geom = (CCTK_INT *) malloc (3 * request->sdim * sizeof (CCTK_INT)); - - /* Get the pointer to the data we want to output (includes downsampling) */ - retval = IOHDF5Util_getDumpData (GH, vindex, timelevel, request, &outme, - &free_outme, geom); +#ifdef CCTK_MPI + mpitype = PUGH_MPIDataType (PUGH_pGH (GH), slab->hdatatype); +#endif + + /* get the pointer to the data we want to output (includes downsampling) */ + retval = GetLocalHyperslab (GH, slab, &data, &free_data); if (retval < 0) { return (retval); @@ -441,87 +282,82 @@ static int IOHDF5Util_DumpGA (const cGH *GH, #ifdef H5_HAVE_PARALLEL if (ioUtilGH->unchunked) { - IOHDF5Util_collectiveDump (GH, vindex, timelevel, request, outme, geom, - info->iohdf5_type, check_exisiting_objects,file); - if (free_outme) + WriteDataCollective (GH, slab, name, data, file); + if (free_data) { - free (outme); + free (data); } - free (geom); return (0); } #endif #endif - /* Dump data held on IO processor */ + /* dump data held on I/O processor */ if (myproc == ioUtilGH->ioproc) { - IOHDF5Util_procDump (GH, vindex, timelevel, request, outme, geom, - myproc, info->iohdf5_type, check_exisiting_objects, - file); + WriteData (GH, slab, name, data, myproc, file); } #ifdef CCTK_MPI + comm = PUGH_pGH (GH)->PUGH_COMM_WORLD; if (myproc == ioUtilGH->ioproc) { - /* Dump data from all other processors */ + /* dump data from all other processors */ for (i = myproc+1; i < myproc+ioUtilGH->ioproc_every && i < nprocs; i++) { - /* receive geometry */ - CACTUS_MPI_ERROR (MPI_Recv (geom, 3*request->sdim, PUGH_MPI_INT, i, - 2*i + IOTAGBASE + 1, pughGH->PUGH_COMM_WORLD, - &ms)); + /* receive geometry (this assumes the geometry arrays + to be contiguous starting at hoffset */ + CACTUS_MPI_ERROR (MPI_Recv (slab->hoffset, 3*slab->hdim, PUGH_MPI_INT, i, + 2*i + MPITAGBASE + 1, comm, &ms)); - incoming = geom[request->sdim]; - for (j = 1; j < request->sdim; j++) + incoming = 1; + for (j = 0; j < slab->hdim; j++) { - incoming *= geom[request->sdim + j]; + incoming *= slab->hsize_chunk[slab->hdim + j]; } /* receive data */ + tmpd = NULL; if (incoming > 0) { - tmpd = malloc (incoming * info->element_size); - CACTUS_MPI_ERROR (MPI_Recv (tmpd, incoming, info->mpi_type, i, - 2*i + IOTAGBASE, pughGH->PUGH_COMM_WORLD, - &ms)); + tmpd = malloc (incoming * CCTK_VarTypeSize (slab->hdatatype)); + CACTUS_MPI_ERROR (MPI_Recv (tmpd, incoming, mpitype, i, + 2*i + MPITAGBASE, comm, &ms)); } - else - { - tmpd = NULL; - } - IOHDF5Util_procDump (GH, vindex, timelevel, request, tmpd, geom, i, - info->iohdf5_type, check_exisiting_objects, file); + + /* write data */ + WriteData (GH, slab, name, tmpd, i, file); + if (tmpd) { free (tmpd); } - } /* End loop over processors */ + } /* end loop over processors */ } else { - - /* Send the geometry and data from the non-IO processors - * to the IO processors + /* + * send the geometry and data from non-I/O processors to the I/O processors */ - outgoing = geom[request->sdim]; - for (i = 1; i < request->sdim; i++) + outgoing = 1; + for (i = 0; i < slab->hdim; i++) { - outgoing *= geom[request->sdim + i]; + outgoing *= slab->hsize_chunk[slab->hdim + i]; } - /* send geometry */ - CACTUS_MPI_ERROR (MPI_Send (geom, 3*request->sdim, PUGH_MPI_INT, - ioUtilGH->ioproc, 2*myproc + IOTAGBASE + 1, - pughGH->PUGH_COMM_WORLD)); + /* send geometry (this assumes the geometry arrays + to be contiguous starting at hoffset */ + CACTUS_MPI_ERROR (MPI_Send (slab->hoffset, 3*slab->hdim, PUGH_MPI_INT, + ioUtilGH->ioproc, 2*myproc + MPITAGBASE + 1, + comm)); + /* send data */ if (outgoing > 0) { - /* send data if outgoing>0 */ - CACTUS_MPI_ERROR (MPI_Send (outme, outgoing, info->mpi_type, - ioUtilGH->ioproc, 2*myproc + IOTAGBASE, - pughGH->PUGH_COMM_WORLD)); -#ifdef DEBUG_ME + CACTUS_MPI_ERROR (MPI_Send (data, outgoing, mpitype, + ioUtilGH->ioproc, 2*myproc + MPITAGBASE, + comm)); +#ifdef IOHDF5UTIL_DEBUG printf ("Processor %d sent %d data points\n", myproc, outgoing); #endif } @@ -532,118 +368,95 @@ static int IOHDF5Util_DumpGA (const cGH *GH, #endif /* free allocated resources */ - if (free_outme) + if (free_data) { - free (outme); + free (data); } - free (geom); return (retval); } -/**************************************************************************/ -/* local functions */ -/**************************************************************************/ -static int IOHDF5Util_getDumpData (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - void **outme, - int *free_outme, - CCTK_INT *geom) +static int GetLocalHyperslab (const cGH *GH, const ioSlab *slab, void **data, + int *free_data) { int i, retval; - int cctk_output_type; - ioGH *ioUtilGH; - int *hsizes, *hsizes_global, *hsizes_offset; /* geometry information */ char *fullname; - - - /* get GH extension for IOUtil */ - ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); - - /* determine the datatype for the hyperslab */ - cctk_output_type = CCTK_VarTypeI (vindex); - if (ioUtilGH->out_single) - { -#ifdef CCTK_INT4 - if (cctk_output_type == CCTK_VARIABLE_INT) - { - cctk_output_type = CCTK_VARIABLE_INT4; - } -#endif -#ifdef CCTK_REAL4 - if (cctk_output_type == CCTK_VARIABLE_REAL) - { - cctk_output_type = CCTK_VARIABLE_REAL4; - } - else if (cctk_output_type == CCTK_VARIABLE_COMPLEX) - { - cctk_output_type = CCTK_VARIABLE_COMPLEX8; - } -#endif - } - /* FIXME */ - /* allocate array to hold size information of request */ - hsizes = (int *) malloc (3 * request->sdim * sizeof (int)); - hsizes_global = hsizes + 1*request->sdim; - hsizes_offset = hsizes + 2*request->sdim; #if 0 - FIXME: what is this for ??? - /* Get the start indeces for the request from origin. */ - /* Slab_start from the geo structure is not a true start index, - it is currently used as the intersection point of three surfaces, - that's why two entries have to be set to zero in the case of a - surface (one entry for line) - FIXME */ - for (i = 0; i < vdim; i++) + mapping = Hyperslab_DefineLocalMappingByIndex (GH, vindex, slab->hdim, + slab->direction, slab->origin, + slab->extent, slab->downsample, + -1, NULL, slab->hsize_local, + slab->hsize_global, + slab->hoffset_global); + const cGH *GH, + CCTK_INT vindex, + CCTK_INT hdim, + const CCTK_INT *direction /* vdim*hdim */, + const CCTK_INT *origin /* vdim */, + const CCTK_INT *extent /* hdim */, + const CCTK_INT *downsample /* hdim */, + CCTK_INT table_handle, + t_hslabConversionFn conversion_fn, + CCTK_INT *hsize_local, /* hdim */ + CCTK_INT *hsize_global, /* hdim */ + CCTK_INT *hoffset_global /* hdim */); + + if (mapping < 0) { - slabstart[i] = request->origin[i]; + fullname = CCTK_FullName (vindex); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Failed to extract hyperslab for variable '%s'", fullname); + free (fullname); } - for (i = 0; i < request->sdim; i++) - { - slabstart[request->direction[i]] = 0; - } -#endif - retval = NewHyperslab_GetLocalHyperslab (GH, vindex, timelevel, request->sdim, - cctk_output_type, NULL, request->origin, - request->direction, request->length, - request->downsample, outme, free_outme, +#else +{ + int *hsizes, *hsizes_global, *hsizes_offset; /* geometry information */ + + /* allocate array to hold size information of request */ + hsizes = (int *) malloc (3 * slab->hdim * sizeof (int)); + hsizes_global = hsizes + 1*slab->hdim; + hsizes_offset = hsizes + 2*slab->hdim; + + retval = NewHyperslab_GetLocalHyperslab (GH, slab->vindex, slab->timelevel, slab->hdim, + slab->hdatatype, NULL, slab->origin, + slab->direction, slab->extent, + slab->downsample, data, free_data, hsizes, hsizes_global, hsizes_offset); if (retval == 0) { - for (i = 0; i < request->sdim; i++) + for (i = 0; i < slab->hdim; i++) { - geom[i + 0*request->sdim] = hsizes_offset[i]; - geom[i + 1*request->sdim] = hsizes[i]; - geom[i + 2*request->sdim] = hsizes_global[i]; - request->actlen[i] = hsizes_global[i]; + slab->hoffset[i] = hsizes_offset[i]; + slab->hsize[i] = hsizes_global[i]; + slab->hsize_chunk[i] = hsizes[i]; } } else { - fullname = CCTK_FullName (vindex); + fullname = CCTK_FullName (slab->vindex); CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "Failed to extract hyperslab for variable '%s'", fullname); free (fullname); } free (hsizes); +} +#endif return (retval); } /*@@ - @routine IOHDF5Util_procDump + @routine WriteData @author Thomas Radke @date May 21 1999 @desc - All IO processors dump the data of their group into the file, + All I/O processors dump the data of their group into the file, either as one chunk per processor or unchunked (depending on parameter "unchunked"). @enddesc @@ -651,78 +464,44 @@ static int IOHDF5Util_getDumpData (const cGH *GH, @calls IOHDF5Util_DumpCommonAttributes @var GH - @vdesc Pointer to CCTK grid hierarchy + @vdesc pointer to CCTK grid hierarchy @vtype const cGH * @vio in @endvar - @var vindex - @vdesc global index of the variable to be dumped - @vtype int - @vio in - @endvar - @var timelevel - @vdesc the timelevel to store - @vtype int + @var slab + @vdesc reference to the I/O hyperslab description + @vtype const ioSlab * @vio in @endvar - @var request - @vdesc pointer to the IO request structure - @vtype const ioHDF5Geo_t * + @var name + @vdesc name of the dataset to write + @vtype const char * @vio in @endvar - @var outme + @var data @vdesc pointer to the chunk to dump @vtype const void * @vio in @endvar - @var geom - @vdesc bounds and sizes of the output - @vtype const CCTK_INT[3*dim] - @vio in - @vcomment geom[0*dim..1*dim-1]: lower bounds - geom[1*dim..2*dim-1]: (local) data sizes - geom[2*dim..3*dim-1]: global space - @endvar @var proc @vdesc the processor which's chunk is to be dumped @vtype int @vio in @endvar - @var hdf5io_type - @vdesc the HDF5 datatype identifier - @vtype int - @vio in - @endvar - @var check_exisiting_objects - @vdesc flag indicating if we should check for already existing objects - before these are created anew - This might happen for existing files reopened after recovery. - @vtype int - @vio in - @endvar @var file @vdesc the HDF5 file handle @vtype hid_t @vio in @endvar @@*/ -static void IOHDF5Util_procDump (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const void *outme, - const CCTK_INT *geom, - int proc, - int iohdf5_type, - int check_exisiting_objects, - hid_t file) +static void WriteData (const cGH *GH, const ioSlab *slab, const char *name, + const void *data, int proc, hid_t file) { - int i, sdim; - int myproc; + int i, myproc; ioGH *ioUtilGH; ioHDF5UtilGH *myGH; - hid_t group, dataset, memspace, filespace, plist; - char *fullname, *datasetname, *chunkname; + hid_t hdf5type, group, dataset, memspace, filespace, plist; + char *chunkname; hssize_t *chunk_origin; hsize_t *chunk_dims, *file_dims; hsize_t buffersize; @@ -738,160 +517,132 @@ static void IOHDF5Util_procDump (const cGH *GH, ioUtilGH = (ioGH *) CCTK_GHExtension (GH, "IO"); myGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util"); - sdim = request->sdim; - myproc = CCTK_MyProc (GH); - - chunk_origin = (hssize_t *) malloc (sdim * sizeof (hssize_t)); - chunk_dims = (hsize_t *) malloc (2*sdim * sizeof (hsize_t)); - file_dims = chunk_dims + sdim; - - /* HDF5 needs it in reverse order */ - for (i = 0; i < sdim; i++) + /* copy the size arrays from CCTK_INT to appropriate types + note that HDF5 wants elements in reverse order */ + chunk_origin = (hssize_t *) malloc (slab->hdim * sizeof (hssize_t)); + chunk_dims = (hsize_t *) malloc (2*slab->hdim * sizeof (hsize_t)); + file_dims = chunk_dims + slab->hdim; + for (i = 0; i < slab->hdim; i++) { - chunk_origin[i] = geom[1*sdim - 1 - i]; - chunk_dims [i] = geom[2*sdim - 1 - i]; - file_dims [i] = geom[3*sdim - 1 - i]; + chunk_origin[i] = slab->hoffset[slab->hdim - 1 - i]; + file_dims [i] = slab->hsize[slab->hdim - 1 - i]; + chunk_dims [i] = slab->hsize_chunk[slab->hdim - 1 - i]; } - /* build the unique dataset name from the variable's full name, - the timelevel and the current iteration number */ - fullname = CCTK_FullName (vindex); - datasetname = (char *) malloc (strlen (fullname) + 80); - sprintf (datasetname, "%s timelevel %d at iteration %d", - fullname, timelevel, GH->cctk_iteration); - - /* if restart from recovery delete an already existing dataset - so that it can be created as anew */ - if (proc == myproc && check_exisiting_objects) - { - IOHDF5_ERROR (H5Eset_auto (NULL, NULL)); - H5Gunlink (file, datasetname); - IOHDF5_ERROR (H5Eset_auto (myGH->print_error_fn, myGH->print_error_fn_arg)); - } + myproc = CCTK_MyProc (GH); - if (outme) + memspace = -1; + if (data) { /* create the memspace according to chunk dims */ - IOHDF5_ERROR (memspace = H5Screate_simple (sdim, chunk_dims, NULL)); - } - else - { - memspace = -1; + HDF5_ERROR (memspace = H5Screate_simple (slab->hdim, chunk_dims, NULL)); } + hdf5type = IOHDF5Util_DataType (myGH, slab->hdatatype); if (ioUtilGH->unchunked) { /* create the (global) filespace and set the hyperslab for the chunk */ - IOHDF5_ERROR (filespace = H5Screate_simple (sdim, file_dims, NULL)); - IOHDF5_ERROR (H5Sselect_hyperslab (filespace, H5S_SELECT_SET, - chunk_origin, NULL, chunk_dims, NULL)); - - /* the IO processor creates the dataset and adds the common attributes + HDF5_ERROR (filespace = H5Screate_simple (slab->hdim, file_dims, NULL)); + HDF5_ERROR (H5Sselect_hyperslab (filespace, H5S_SELECT_SET, chunk_origin, + NULL, chunk_dims, NULL)); + + /* the I/O processor creates the dataset and adds the common attributes when writing its own data, otherwise the dataset is reopened */ if (proc == myproc) { - IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + HDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); /* enable compression for chunked dataset if compression was requested */ if (compression_level) { - IOHDF5_ERROR (H5Pset_chunk (plist, sdim, chunk_dims)); - IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); + HDF5_ERROR (H5Pset_chunk (plist, slab->hdim, chunk_dims)); + HDF5_ERROR (H5Pset_deflate (plist, compression_level)); } - IOHDF5_ERROR (dataset = H5Dcreate (file, datasetname, - iohdf5_type, filespace, plist)); - IOHDF5_ERROR (H5Pclose (plist)); - IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, &geom[2*sdim], - request, dataset); + HDF5_ERROR (dataset = H5Dcreate (file, name, hdf5type, filespace, plist)); + HDF5_ERROR (H5Pclose (plist)); + IOHDF5Util_DumpCommonAttributes (GH, slab, dataset); } else { - IOHDF5_ERROR (dataset = H5Dopen (file, datasetname)); + HDF5_ERROR (dataset = H5Dopen (file, name)); } if (memspace >= 0) { /* increase the buffer size if the default isn't sufficient */ - IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); + HDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); buffersize = H5Dget_storage_size (dataset); if (buffersize > H5Pget_buffer (plist, NULL, NULL)) { - IOHDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); + HDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); } /* write the data */ - IOHDF5_ERROR (H5Dwrite (dataset, iohdf5_type, memspace, filespace, - plist, outme)); + HDF5_ERROR (H5Dwrite (dataset, hdf5type, memspace, filespace, plist, + data)); /* close the transfer property list */ - IOHDF5_ERROR (H5Pclose (plist)); + HDF5_ERROR (H5Pclose (plist)); } /* close the file dataspace */ - IOHDF5_ERROR (H5Sclose (filespace)); + HDF5_ERROR (H5Sclose (filespace)); } else { - /* the IO processor creates the chunk group and adds common attributes */ + /* the I/O processor creates the chunk group and adds common attributes */ if (proc == myproc) { - IOHDF5_ERROR (group = H5Gcreate (file, datasetname, 0)); - IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, &geom[2*sdim], - request, group); - IOHDF5_ERROR (H5Gclose (group)); + HDF5_ERROR (group = H5Gcreate (file, name, 0)); + IOHDF5Util_DumpCommonAttributes (GH, slab, group); + HDF5_ERROR (H5Gclose (group)); } + dataset = -1; if (memspace >= 0) { /* now the chunk dataset for each processor is created within the group */ - chunkname = (char *) malloc (strlen (datasetname) + 20); - sprintf (chunkname, "%s/chunk%d", datasetname, proc - myproc); - IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + chunkname = (char *) malloc (strlen (name) + 20); + sprintf (chunkname, "%s/chunk%d", name, proc - myproc); + HDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); /* enable compression for chunked dataset if compression was requested */ if (compression_level) { - IOHDF5_ERROR (H5Pset_chunk (plist, sdim, chunk_dims)); - IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); - } + HDF5_ERROR (H5Pset_chunk (plist, slab->hdim, chunk_dims)); + HDF5_ERROR (H5Pset_deflate (plist, compression_level)); + } /* create the chunk dataset and dump the chunk data */ - IOHDF5_ERROR (dataset = H5Dcreate (file, chunkname, - iohdf5_type, memspace, plist)); - IOHDF5_ERROR (H5Pclose (plist)); - IOHDF5_ERROR (H5Dwrite (dataset, iohdf5_type, H5S_ALL, H5S_ALL, - H5P_DEFAULT, outme)); + HDF5_ERROR (dataset = H5Dcreate (file, chunkname, hdf5type, memspace, + plist)); + HDF5_ERROR (H5Pclose (plist)); + HDF5_ERROR (H5Dwrite (dataset, hdf5type, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data)); /* add the "origin" attribute for the chunk */ - WRITE_ATTRIBUTE ("chunk_origin", geom, dataset, myGH->array_dataspace, - sdim, IOHDF5_INT); + WRITE_ATTRIBUTE ("chunk_origin", slab->hoffset, dataset, myGH, slab->hdim, + HDF5_INT); free (chunkname); } - else - { - dataset = -1; - } } /* close the dataset and the memspace */ if (dataset >= 0) { - IOHDF5_ERROR (H5Dclose (dataset)); + HDF5_ERROR (H5Dclose (dataset)); } if (memspace >= 0) { - IOHDF5_ERROR (H5Sclose (memspace)); + HDF5_ERROR (H5Sclose (memspace)); } /* free allocated resources */ - free (datasetname); - free (fullname); free (chunk_origin); free (chunk_dims); } -#ifdef CCTK_MPI -#ifdef H5_HAVE_PARALLEL +#if defined(CCTK_MPI) && defined(H5_HAVE_PARALLEL) /*@@ - @routine IOHDF5Util_collectiveDump + @routine WriteDataCollective @author Thomas Radke @date May 21 1999 @desc @@ -901,70 +652,36 @@ static void IOHDF5Util_procDump (const cGH *GH, @calls IOHDF5Util_DumpCommonAttributes @var GH - @vdesc Pointer to CCTK grid hierarchy + @vdesc pointer to CCTK grid hierarchy @vtype const cGH * @vio in @endvar - @var vindex - @vdesc global index of the variable to be dumped - @vtype int + @var slab + @vdesc reference to the I/O hyperslab description + @vtype const ioSlab * @vio in @endvar - @var timelevel - @vdesc the timelevel to store - @vtype int + @var name + @vdesc name of the dataset to write + @vtype const char * @vio in @endvar - @var request - @vdesc pointer to the IO request structure - @vtype const ioHDF5Geo_t * - @vio in - @endvar - @var outme + @var data @vdesc pointer to the chunk to dump @vtype const void * @vio in @endvar - @var geom - @vdesc bounds and sizes of the output - @vtype const CCTK_INT[3*dim] - @vio in - @vcomment geom[0*dim..1*dim-1]: lower bounds - geom[1*dim..2*dim-1]: (local) data sizes - geom[2*dim..3*dim-1]: global space - @endvar - @var hdf5io_type - @vdesc the HDF5 datatype identifier - @vtype int - @vio in - @endvar - @var check_exisiting_objects - @vdesc flag indicating if we should check for already existing objects - before these are created anew - This might happen for existing files reopened after recovery. - @vtype int - @vio in - @endvar @var file @vdesc the HDF5 file handle @vtype hid_t @vio in @endvar @@*/ -static void IOHDF5Util_collectiveDump (const cGH *GH, - int vindex, - int timelevel, - const ioHDF5Geo_t *request, - const void *outme, - const CCTK_INT *geom, - int hdf5io_type, - int check_exisiting_objects, - hid_t file) +static void WriteDataCollective (const cGH *GH, const ioSlab *slab, + const char *name, const void *data, hid_t file) { int i, dim; - ioHDF5UtilGH *myGH; - hid_t dataset, memspace, filespace, plist; - char *fullname, *datasetname; + hid_t hdf5type, dataset, memspace, filespace, plist; hssize_t *chunk_origin; hsize_t *chunk_dims, *file_dims; hsize_t buffersize; @@ -977,84 +694,63 @@ static void IOHDF5Util_collectiveDump (const cGH *GH, return; } - myGH = (ioHDF5UtilGH *) CCTK_GHExtension (GH, "IOHDF5Util"); - - /* get the dimension of the variable */ - chunk_origin = (hssize_t *) malloc (request->sdim * sizeof (hssize_t)); - chunk_dims = (hsize_t *) malloc (2*request->sdim * sizeof (hsize_t)); - file_dims = chunk_dims + request->sdim; - - /* HDF5 needs it in reverse order */ - for (i = 0; i < request->sdim; i++) + /* copy the size arrays from CCTK_INT to appropriate types + note that HDF5 wants elements in reverse order */ + chunk_origin = (hssize_t *) malloc (slab->hdim * sizeof (hssize_t)); + chunk_dims = (hsize_t *) malloc (2*slab->hdim * sizeof (hsize_t)); + file_dims = chunk_dims + slab->hdim; + for (i = 0; i < slab->hdim; i++) { - chunk_origin[i] = geom[1*request->sdim - 1 - i]; - chunk_dims [i] = geom[2*request->sdim - 1 - i]; - file_dims [i] = geom[3*request->sdim - 1 - i]; + chunk_origin[i] = slab->hoffset[slab->hdim - 1 - i]; + chunk_dims [i] = slab->hsize_chunk[slab->hdim - 1 - i]; + file_dims [i] = slab->hsize_global[slab->hdim - 1 - i]; } - /* build the unique dataset name */ - fullname = CCTK_FullName (vindex); - datasetname = (char *) malloc (strlen (fullname) + 80); - sprintf (datasetname, "%s timelevel %d at iteration %d", - fullname, timelevel, GH->cctk_iteration); - /* create the memspace according to chunk dims */ - IOHDF5_ERROR (memspace = H5Screate_simple (request->sdim, chunk_dims, NULL)); + HDF5_ERROR (memspace = H5Screate_simple (slab->hdim, chunk_dims, NULL)); /* create the (global) filespace and set the hyperslab for the chunk */ - IOHDF5_ERROR (filespace = H5Screate_simple (request->sdim, file_dims, NULL)); - IOHDF5_ERROR (H5Sselect_hyperslab (filespace, H5S_SELECT_SET, - chunk_origin, NULL, chunk_dims, NULL)); + HDF5_ERROR (filespace = H5Screate_simple (slab->hdim, file_dims, NULL)); + HDF5_ERROR (H5Sselect_hyperslab (filespace, H5S_SELECT_SET, chunk_origin, + NULL, chunk_dims, NULL)); - /* the IO processor creates the dataset and adds the common attributes + /* the I/O processor creates the dataset and adds the common attributes when writing its own data, otherwise the dataset is reopened */ - /* if restart from recovery delete an already existing group - so that it can be created as anew */ - if (check_exisiting_objects) - { - IOHDF5_ERROR (H5Eset_auto (NULL, NULL)); - H5Gunlink (file, datasetname); - IOHDF5_ERROR (H5Eset_auto (myGH->print_error_fn, myGH->print_error_fn_arg)); - } + hdf5type = IOHDF5Util_DataType (myGH, slab->hdatatype); + /* enable compression for chunked dataset if compression was requested */ - IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); + HDF5_ERROR (plist = H5Pcreate (H5P_DATASET_CREATE)); if (compression_level) { - IOHDF5_ERROR (H5Pset_chunk (plist, request->sdim, chunk_dims)); - IOHDF5_ERROR (H5Pset_deflate (plist, compression_level)); + HDF5_ERROR (H5Pset_chunk (plist, slab->hdim, chunk_dims)); + HDF5_ERROR (H5Pset_deflate (plist, compression_level)); } - IOHDF5_ERROR (dataset = H5Dcreate (file, datasetname, hdf5io_type, filespace, - plist)); - IOHDF5_ERROR (H5Pclose (plist)); + HDF5_ERROR (dataset = H5Dcreate (file, name, hdf5type, filespace, plist)); + HDF5_ERROR (H5Pclose (plist)); if (CCTK_MyProc (GH) == 0) { - IOHDF5Util_DumpCommonAttributes (GH, vindex, timelevel, - &geom[2*request->sdim], request, dataset); + IOHDF5Util_DumpCommonAttributes (GH, slab, dataset); } /* increase the buffer size if the default isn't sufficient */ - IOHDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); + HDF5_ERROR (plist = H5Pcreate (H5P_DATASET_XFER)); buffersize = H5Dget_storage_size (dataset); if (buffersize > H5Pget_buffer (plist, NULL, NULL)) { - IOHDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); + HDF5_ERROR (H5Pset_buffer (plist, buffersize, NULL, NULL)); } /* write the data */ - IOHDF5_ERROR (H5Dwrite (dataset, hdf5io_type, memspace, filespace, plist, - outme)); + HDF5_ERROR (H5Dwrite (dataset, hdf5type, memspace, filespace, plist, data)); /* close resources */ - IOHDF5_ERROR (H5Pclose (plist)); - IOHDF5_ERROR (H5Sclose (filespace)); - IOHDF5_ERROR (H5Dclose (dataset)); - IOHDF5_ERROR (H5Sclose (memspace)); + HDF5_ERROR (H5Pclose (plist)); + HDF5_ERROR (H5Sclose (filespace)); + HDF5_ERROR (H5Dclose (dataset)); + HDF5_ERROR (H5Sclose (memspace)); /* free allocated resources */ - free (datasetname); - free (fullname); free (chunk_origin); free (chunk_dims); } -#endif /* H5_HAVE_PARALLEL */ -#endif /* MPI */ +#endif /* CCTK_MPI && H5_HAVE_PARALLEL */ |