From 0c0aac7197a5ee0d9815ebbbbcc900b3be32c817 Mon Sep 17 00:00:00 2001 From: tradke Date: Mon, 18 Feb 2002 13:01:01 +0000 Subject: Fix for recombining cropped datasets where some processors contribute with no data chunks at all. git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5/trunk@93 4825ed28-b72c-4eae-9704-e50c059e567d --- src/util/hdf5_recombiner.c | 108 ++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 64 deletions(-) (limited to 'src/util/hdf5_recombiner.c') diff --git a/src/util/hdf5_recombiner.c b/src/util/hdf5_recombiner.c index ffd7e13..14b420f 100644 --- a/src/util/hdf5_recombiner.c +++ b/src/util/hdf5_recombiner.c @@ -233,11 +233,11 @@ int main (int argc, char *argv[]) return (-1); } - /* check if the input file already contains only unchunked data */ + /* check if the input file already contains unchunked data */ if (unchunked) { fprintf (stderr, "WARNING: Nothing to do ! Input file '%s' already " - "contains only unchunked data.\n\n", argv[1]); + "contains unchunked data.\n\n", argv[1]); return (0); } @@ -615,9 +615,10 @@ static int RecombineGroupData (const char *groupname) CHECK_ERROR (group = H5Gopen (infiles[0], groupname)); /* a group with chunked data must have a 'global_size' attribute */ - CHECK_ERROR (H5Eset_auto (NULL, NULL)); - attr = H5Aopen_name (group, "global_size"); - CHECK_ERROR (H5Eset_auto ((H5E_auto_t) H5Eprint, stderr)); + H5E_BEGIN_TRY + { + attr = H5Aopen_name (group, "global_size"); + } H5E_END_TRY; if (attr < 0) { CHECK_ERROR (H5Gclose (group)); @@ -657,29 +658,8 @@ static int RecombineGroupData (const char *groupname) /* allocate string buffer holding the possible dataset chunk names */ chunkname = (char *) malloc (strlen (groupname) + sizeof ("chunk") + 20); - /* now try to open the first chunk of a dataset as '/chunk0' */ - sprintf (chunkname, "%s/chunk0", pathname); - CHECK_ERROR (H5Eset_auto (NULL, NULL)); - chunked_dataset = H5Dopen (infiles[0], chunkname); - CHECK_ERROR (H5Eset_auto ((H5E_auto_t) H5Eprint, stderr)); - if (chunked_dataset < 0) - { - free (chunkname); - free (global_size); - CHECK_ERROR (H5Gclose (group)); - return (0); - } - - /*** Okay, now we can be quite sure that this is a group containing - chunks of a dataset named 'groupname'. - So let's go on and create the unchunked dataset in the output file. - ***/ - - /* the unchunked dataset gets the same datatype as the first chunk */ - CHECK_ERROR (unchunked_datatype = H5Dget_type (chunked_dataset)); - - /* its dimensions are taken from the 'global_size' attribute - with the least changing dimension being the first element */ + /* the unchunked dataset's dimensions are taken from the 'global_size' + attribute, with the least changing dimension being the first element */ for (chunk_ndims = 0; chunk_ndims < ndims/2; chunk_ndims++) { tmp1 = global_size[chunk_ndims]; @@ -687,45 +667,26 @@ static int RecombineGroupData (const char *groupname) global_size[ndims - chunk_ndims - 1] = tmp1; } CHECK_ERROR (unchunked_dataspace = H5Screate_simple (ndims, global_size, - NULL)); - - /* create the unchunked dataset */ - CHECK_ERROR (unchunked_dataset = H5Dcreate (outfile, pathname, - unchunked_datatype, - unchunked_dataspace, - H5P_DEFAULT)); - - /* copy all group attributes to the new dataset */ - CHECK_ERROR (H5Aiterate (group, NULL, CopyAttribute, &unchunked_dataset)); - - /* don't need these anymore */ + NULL)); + /* don't need this anymore */ free (global_size); - CHECK_ERROR (H5Dclose (chunked_dataset)); - CHECK_ERROR (H5Gclose (group)); /* allocate buffers to read a 'chunk_origin' attribute and the chunk dims */ chunk_origin = (hssize_t *) calloc (ndims, sizeof (hssize_t)); chunk_dims = (hsize_t *) calloc (ndims, sizeof (hsize_t)); + unchunked_dataset = unchunked_datatype = -1; + /* now read all the chunks from all input files and write them into the unchunked output dataset as a hyperslab */ - printf (" recombining dataset '%s'\n", pathname); for (infile = 0; infile < ninfiles; infile++) { /* get the number of chunks in this file */ -#if 0 - nchunks = nprocs / ninfiles; - if (infile == ninfiles - 1) - { - nchunks += nprocs % ninfiles; - } -#else nchunks = ioproc_every; if (infile == ninfiles - 1 && nprocs % ioproc_every) { nchunks = nprocs % ninfiles; } -#endif /* re-open the file if it was closed before */ if (infile > max_filehandles) @@ -743,17 +704,20 @@ static int RecombineGroupData (const char *groupname) /* build the object name of this chunk */ sprintf (chunkname, "%s/chunk%d", pathname, chunk); - /* open the dataset chunk */ - CHECK_ERROR (chunked_dataset = H5Dopen (infiles[infile], chunkname)); + /* try to open the dataset chunk */ + H5E_BEGIN_TRY + { + chunked_dataset = H5Dopen (infiles[infile], chunkname); + } H5E_END_TRY; if (chunked_dataset < 0) { continue; } /* read the 'chunk_origin' attribute of this chunk */ - attr = H5Aopen_name (chunked_dataset, "chunk_origin"); - datatype = H5Aget_type (attr); - dataspace = H5Aget_space (attr); + CHECK_ERROR (attr = H5Aopen_name (chunked_dataset, "chunk_origin")); + CHECK_ERROR (datatype = H5Aget_type (attr)); + CHECK_ERROR (dataspace = H5Aget_space (attr)); class = H5Tget_class (datatype); chunk_ndims = H5Sget_simple_extent_npoints (dataspace); if (chunk_ndims == ndims) @@ -783,16 +747,26 @@ static int RecombineGroupData (const char *groupname) continue; } - /* check the datatype of this chunk to be consistent with the others - NOTE: for data files from a distributed heterogenous run - the data types might differ in their native representation - so we cannot use H5Tequal() here */ - chunked_datatype = H5Dget_type (chunked_dataset); + /* the first time through, create the unchunked dataset */ + if (unchunked_dataset < 0) + { + printf (" recombining dataset '%s'\n", pathname); + + /* the unchunked dataset gets the same datatype as the first chunk */ + CHECK_ERROR (unchunked_datatype = H5Dget_type (chunked_dataset)); + CHECK_ERROR (unchunked_dataset = H5Dcreate (outfile, pathname, + unchunked_datatype, + unchunked_dataspace, + H5P_DEFAULT)); + } + + /* check the datatype of this chunk to be consistent with previous ones */ + CHECK_ERROR (chunked_datatype = H5Dget_type (chunked_dataset)); class = H5Tget_class (chunked_datatype); CHECK_ERROR (H5Tclose (chunked_datatype)); if (class != H5Tget_class (unchunked_datatype)) { - fprintf (stderr, "WARNING: datatype clas of chunk '%s' differs from " + fprintf (stderr, "WARNING: datatype class of chunk '%s' differs from " "first chunk's datatype ! Chunk will be omitted.\n", chunkname); CHECK_ERROR (H5Dclose (chunked_dataset)); @@ -867,9 +841,15 @@ static int RecombineGroupData (const char *groupname) } /* end of loop over all input files */ /* close objects and free allocated resources */ - CHECK_ERROR (H5Dclose (unchunked_dataset)); - CHECK_ERROR (H5Tclose (unchunked_datatype)); + if (unchunked_dataset >= 0) + { + /* copy all group attributes to the new dataset */ + CHECK_ERROR (H5Aiterate (group, NULL, CopyAttribute, &unchunked_dataset)); + CHECK_ERROR (H5Dclose (unchunked_dataset)); + CHECK_ERROR (H5Tclose (unchunked_datatype)); + } CHECK_ERROR (H5Sclose (unchunked_dataspace)); + CHECK_ERROR (H5Gclose (group)); free (chunk_dims); free (chunk_origin); free (chunkname); -- cgit v1.2.3