Fix for recombining cropped datasets where some processors contribute

with no data chunks at all.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5/trunk@93 4825ed28-b72c-4eae-9704-e50c059e567d

1 files changed, 44 insertions, 64 deletions

diff --git a/src/util/hdf5_recombiner.c b/src/util/hdf5_recombiner.c
index ffd7e13..14b420f 100644
--- a/src/util/hdf5_recombiner.c
+++ b/src/util/hdf5_recombiner.c
@@ -233,11 +233,11 @@ int main (int argc, char *argv[])
     return (-1);
   }
 
-  /* check if the input file already contains only unchunked data */
+  /* check if the input file already contains unchunked data */
   if (unchunked)
   {
     fprintf (stderr, "WARNING: Nothing to do ! Input file '%s' already "
-                     "contains only unchunked data.\n\n", argv[1]);
+                     "contains unchunked data.\n\n", argv[1]);
     return (0);
   }
 
@@ -615,9 +615,10 @@ static int RecombineGroupData (const char *groupname)
   CHECK_ERROR (group = H5Gopen (infiles[0], groupname));
 
   /* a group with chunked data must have a 'global_size' attribute */
-  CHECK_ERROR (H5Eset_auto (NULL, NULL));
-  attr = H5Aopen_name (group, "global_size");
-  CHECK_ERROR (H5Eset_auto ((H5E_auto_t) H5Eprint, stderr));
+  H5E_BEGIN_TRY
+  {
+    attr = H5Aopen_name (group, "global_size");
+  } H5E_END_TRY;
   if (attr < 0)
   {
     CHECK_ERROR (H5Gclose (group));
@@ -657,29 +658,8 @@ static int RecombineGroupData (const char *groupname)
   /* allocate string buffer holding the possible dataset chunk names */
   chunkname = (char *) malloc (strlen (groupname) + sizeof ("chunk") + 20);
 
-  /* now try to open the first chunk of a dataset as '<groupname>/chunk0' */
-  sprintf (chunkname, "%s/chunk0", pathname);
-  CHECK_ERROR (H5Eset_auto (NULL, NULL));
-  chunked_dataset = H5Dopen (infiles[0], chunkname);
-  CHECK_ERROR (H5Eset_auto ((H5E_auto_t) H5Eprint, stderr));
-  if (chunked_dataset < 0)
-  {
-    free (chunkname);
-    free (global_size);
-    CHECK_ERROR (H5Gclose (group));
-    return (0);
-  }
-
-  /*** Okay, now we can be quite sure that this is a group containing
-       chunks of a dataset named 'groupname'.
-       So let's go on and create the unchunked dataset in the output file.
-   ***/
-
-  /* the unchunked dataset gets the same datatype as the first chunk */
-  CHECK_ERROR (unchunked_datatype = H5Dget_type (chunked_dataset));
-
-  /* its dimensions are taken from the 'global_size' attribute
-     with the least changing dimension being the first element */
+  /* the unchunked dataset's dimensions are taken from the 'global_size'
+     attribute, with the least changing dimension being the first element */
   for (chunk_ndims = 0; chunk_ndims < ndims/2; chunk_ndims++)
   {
     tmp1 = global_size[chunk_ndims];
@@ -687,45 +667,26 @@ static int RecombineGroupData (const char *groupname)
     global_size[ndims - chunk_ndims - 1] = tmp1;
   }
   CHECK_ERROR (unchunked_dataspace = H5Screate_simple (ndims, global_size,
-                                                        NULL));
-
-  /* create the unchunked dataset */
-  CHECK_ERROR (unchunked_dataset = H5Dcreate (outfile, pathname,
-                                              unchunked_datatype,
-                                              unchunked_dataspace,
-                                              H5P_DEFAULT));
-
-  /* copy all group attributes to the new dataset */
-  CHECK_ERROR (H5Aiterate (group, NULL, CopyAttribute, &unchunked_dataset));
-
-  /* don't need these anymore */
+                                                       NULL));
+  /* don't need this anymore */
   free (global_size);
-  CHECK_ERROR (H5Dclose (chunked_dataset));
-  CHECK_ERROR (H5Gclose (group));
 
   /* allocate buffers to read a 'chunk_origin' attribute and the chunk dims */
   chunk_origin = (hssize_t *) calloc (ndims, sizeof (hssize_t));
   chunk_dims   = (hsize_t  *) calloc (ndims, sizeof (hsize_t));
 
+  unchunked_dataset = unchunked_datatype = -1;
+
   /* now read all the chunks from all input files and write them into the
      unchunked output dataset as a hyperslab */
-  printf ("  recombining dataset '%s'\n", pathname);
   for (infile = 0; infile < ninfiles; infile++)
   {
     /* get the number of chunks in this file */
-#if 0
-    nchunks = nprocs / ninfiles;
-    if (infile == ninfiles - 1)
-    {
-      nchunks += nprocs % ninfiles;
-    }
-#else
     nchunks = ioproc_every;
     if (infile == ninfiles - 1 && nprocs % ioproc_every)
     {
       nchunks = nprocs % ninfiles;
     }
-#endif
 
     /* re-open the file if it was closed before */
     if (infile > max_filehandles)
@@ -743,17 +704,20 @@ static int RecombineGroupData (const char *groupname)
       /* build the object name of this chunk */
       sprintf (chunkname, "%s/chunk%d", pathname, chunk);
 
-      /* open the dataset chunk */
-      CHECK_ERROR (chunked_dataset = H5Dopen (infiles[infile], chunkname));
+      /* try to open the dataset chunk */
+      H5E_BEGIN_TRY
+      {
+        chunked_dataset = H5Dopen (infiles[infile], chunkname);
+      } H5E_END_TRY;
       if (chunked_dataset < 0)
       {
         continue;
       }
 
       /* read the 'chunk_origin' attribute of this chunk */
-      attr = H5Aopen_name (chunked_dataset, "chunk_origin");
-      datatype = H5Aget_type (attr);
-      dataspace = H5Aget_space (attr);
+      CHECK_ERROR (attr = H5Aopen_name (chunked_dataset, "chunk_origin"));
+      CHECK_ERROR (datatype = H5Aget_type (attr));
+      CHECK_ERROR (dataspace = H5Aget_space (attr));
       class = H5Tget_class (datatype);
       chunk_ndims = H5Sget_simple_extent_npoints (dataspace);
       if (chunk_ndims == ndims)
@@ -783,16 +747,26 @@ static int RecombineGroupData (const char *groupname)
         continue;
       }
 
-      /* check the datatype of this chunk to be consistent with the others
-         NOTE: for data files from a distributed heterogenous run
-               the data types might differ in their native representation
-               so we cannot use H5Tequal() here */
-      chunked_datatype = H5Dget_type (chunked_dataset);
+      /* the first time through, create the unchunked dataset */
+      if (unchunked_dataset < 0)
+      {
+        printf ("  recombining dataset '%s'\n", pathname);
+
+        /* the unchunked dataset gets the same datatype as the first chunk */
+        CHECK_ERROR (unchunked_datatype = H5Dget_type (chunked_dataset));
+        CHECK_ERROR (unchunked_dataset = H5Dcreate (outfile, pathname,
+                                                    unchunked_datatype,
+                                                    unchunked_dataspace,
+                                                    H5P_DEFAULT));
+      }
+
+      /* check the datatype of this chunk to be consistent with previous ones */
+      CHECK_ERROR (chunked_datatype = H5Dget_type (chunked_dataset));
       class = H5Tget_class (chunked_datatype);
       CHECK_ERROR (H5Tclose (chunked_datatype));
       if (class != H5Tget_class (unchunked_datatype))
       {
-        fprintf (stderr, "WARNING: datatype clas of chunk '%s' differs from "
+        fprintf (stderr, "WARNING: datatype class of chunk '%s' differs from "
                          "first chunk's datatype ! Chunk will be omitted.\n",
                          chunkname);
         CHECK_ERROR (H5Dclose (chunked_dataset));
@@ -867,9 +841,15 @@ static int RecombineGroupData (const char *groupname)
   } /* end of loop over all input files */
 
   /* close objects and free allocated resources */
-  CHECK_ERROR (H5Dclose (unchunked_dataset));
-  CHECK_ERROR (H5Tclose (unchunked_datatype));
+  if (unchunked_dataset >= 0)
+  {
+    /* copy all group attributes to the new dataset */
+    CHECK_ERROR (H5Aiterate (group, NULL, CopyAttribute, &unchunked_dataset));
+    CHECK_ERROR (H5Dclose (unchunked_dataset));
+    CHECK_ERROR (H5Tclose (unchunked_datatype));
+  }
   CHECK_ERROR (H5Sclose (unchunked_dataspace));
+  CHECK_ERROR (H5Gclose (group));
   free (chunk_dims);
   free (chunk_origin);
   free (chunkname);