Fixed a problem with recovering small distributed grid arrays

which have zero size on some processors.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5Util/trunk@99 7842ec3a-9562-4be5-9c5b-06ba18f2b668

1 files changed, 45 insertions, 36 deletions

diff --git a/src/RecoverVar.c b/src/RecoverVar.c
index 4a5b56f..1f80b5c 100644
--- a/src/RecoverVar.c
+++ b/src/RecoverVar.c
@@ -38,7 +38,7 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5Util_RecoverVar_c)
  ********************************************************************/
 typedef struct
 {
-  cGH *GH;
+  const cGH *GH;
   int ioproc;
   int ioproc_every;
   int unchunked;
@@ -167,9 +167,12 @@ int IOHDF5Util_RecoverVariables (cGH *GH, const fileinfo_t *fileinfo)
       }
 
       /* receive following data from my IO processor */
-      CACTUS_MPI_ERROR (MPI_Recv (GH->data[vindex][timelevel], npoints,
-                                  mpi_type, fileinfo->ioproc, MPITAGBASE,
-                                  pughGH->PUGH_COMM_WORLD, &ms));
+      if (npoints > 0)
+      {
+        CACTUS_MPI_ERROR (MPI_Recv (GH->data[vindex][timelevel], npoints,
+                                    mpi_type, fileinfo->ioproc, MPITAGBASE,
+                                    pughGH->PUGH_COMM_WORLD, &ms));
+      }
     }
   }
 #endif /* CCTK_MPI */
@@ -275,7 +278,7 @@ int IOHDF5Util_RecoverParameters (const fileinfo_t *fileinfo)
     {
       HDF5_ERROR (atype = H5Aget_type (attr));
       parameterSize = H5Tget_size (atype);
-      parameters = (char *) malloc (parameterSize + 1);
+      parameters = malloc (parameterSize + 1);
       HDF5_ERROR (H5Aread (attr, atype, parameters));
       parameters[parameterSize] = 0;
       HDF5_ERROR (H5Tclose (atype));
@@ -310,7 +313,7 @@ int IOHDF5Util_RecoverParameters (const fileinfo_t *fileinfo)
 #ifdef CCTK_MPI
     if (CCTK_MyProc (NULL) != 0)
     {
-      parameters = (char *) malloc (parameterSize + 1);
+      parameters = malloc (parameterSize + 1);
     }
 
     CACTUS_MPI_ERROR (MPI_Bcast (parameters, parameterSize + 1, PUGH_MPI_CHAR,
@@ -345,7 +348,7 @@ int IOHDF5Util_RecoverParameters (const fileinfo_t *fileinfo)
  If successful, the group type and the possibly corrected timelevel
  to restore are stored in {*gtype, *timelevel}, and 0 is returned.
 */
-static int GetCommonAttributes (cGH *GH,
+static int GetCommonAttributes (const cGH *GH,
                                 hid_t dataset,
                                 int vindex,
                                 const char *fullname,
@@ -370,12 +373,13 @@ static int GetCommonAttributes (cGH *GH,
   /* read and verify the group name */
   READ_ATTRIBUTE (dataset, "groupname", H5T_C_S1, groupname_stored);
   groupname = CCTK_GroupNameFromVarI (vindex);
-  if (! CCTK_Equals (groupname_stored, groupname))
+  i = CCTK_Equals (groupname_stored, groupname);
+  free (groupname);
+  if (! i)
   {
     CCTK_WARN (2, "Groupnames don't match");
     return (-1);
   }
-  free (groupname);
 
   /* verify group type, variable type, dims, sizes and ntimelevels */
   READ_ATTRIBUTE (dataset, "grouptype", H5T_NATIVE_INT, &grouptype_stored);
@@ -441,11 +445,10 @@ static int GetCommonAttributes (cGH *GH,
   /* verify the dims and sizes */
   HDF5_ERROR (dataspace = H5Dget_space (dataset));
   HDF5_ERROR (rank_stored = H5Sget_simple_extent_ndims (dataspace));
-  /* scalars are stored as rank 0 in HDF5 but have rank 1 in Cactus */
   dims_stored = NULL;
   if (rank_stored > 0)
   {
-    dims_stored = (hsize_t *) malloc (rank_stored * sizeof (hsize_t));
+    dims_stored = malloc (rank_stored * sizeof (hsize_t));
     HDF5_ERROR (H5Sget_simple_extent_dims (dataspace, dims_stored, NULL));
   }
   HDF5_ERROR (H5Sclose (dataspace));
@@ -531,7 +534,6 @@ static int IOHDF5Util_RestoreGS (hid_t dataset, int vindex, int timelevel,
 #ifdef CCTK_MPI
   int proc;
   CCTK_INT var_info[3];
-  MPI_Comm comm;
 #endif
 
 
@@ -544,8 +546,6 @@ static int IOHDF5Util_RestoreGS (hid_t dataset, int vindex, int timelevel,
 #ifdef CCTK_MPI
   /* ... and communicate it for the MPI case */
 
-  comm = PUGH_pGH (rec_info->it_info->GH)->PUGH_COMM_WORLD;
-
   /* set the variable's index and the timelevel */
   var_info[0] = vindex; var_info[1] = timelevel; var_info[2] = 1;
 
@@ -556,9 +556,9 @@ static int IOHDF5Util_RestoreGS (hid_t dataset, int vindex, int timelevel,
        proc++)
   {
     CACTUS_MPI_ERROR (MPI_Send (var_info, 3, PUGH_MPI_INT, proc, MPITAGBASE,
-                                comm));
+                            PUGH_pGH (rec_info->it_info->GH)->PUGH_COMM_WORLD));
     CACTUS_MPI_ERROR (MPI_Send (data, 1, rec_info->mpi_type, proc, MPITAGBASE,
-                                comm));
+                            PUGH_pGH (rec_info->it_info->GH)->PUGH_COMM_WORLD));
   }
 #endif /* CCTK_MPI */
 
@@ -599,6 +599,11 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
   /* Get the pGExtras pointer as a shortcut */
   extras = ((pGA ***) pughGH->variables)[vindex][timelevel]->extras;
 
+  /* get the dimension of the variable */
+  dim = CCTK_GroupDimFromVarI (vindex);
+  chunk_origin = malloc (dim * sizeof (hssize_t));
+  chunk_dims = malloc (dim * sizeof (hsize_t));
+
   /* allocate memory for the biggest chunk */
   npoints = 1;
   for (proc = rec_info->it_info->ioproc + 1;
@@ -611,7 +616,7 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
       npoints = extras->rnpoints[proc];
     }
   }
-  buffer = malloc (npoints * rec_info->element_size);
+  buffer = npoints > 0 ? malloc (npoints * rec_info->element_size) : NULL;
 
   /* set the variable's index and timelevel to restore */
   var_info[0] = vindex; var_info[1] = timelevel;
@@ -632,7 +637,11 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
       data = buffer;
     }
 
-    if (! rec_info->it_info->unchunked)
+    if (extras->rnpoints[proc] <= 0)
+    {
+      /* chunk contains no data - fall through */
+    }
+    else if (! rec_info->it_info->unchunked)
     {
       /* Chunked data is stored as separate chunk datasets within a group.
          So open, read and close the separate chunks here. */
@@ -645,11 +654,6 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
     }
     else
     {
-      /* get the dimension of the variable */
-      dim = CCTK_GroupDimI (CCTK_GroupIndexFromVarI (vindex));
-      chunk_origin = (hssize_t *) malloc (dim * sizeof (hssize_t));
-      chunk_dims = (hsize_t *) malloc (dim * sizeof (hsize_t));
-
       /* Unchunked data is read as one hyperslab per processor.
          So prepare the memspace and the filespace and read the hyperslab. */
       for (i = 0; i < dim; i++)
@@ -661,16 +665,13 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
       HDF5_ERROR (filespace = H5Dget_space (dataset));
       HDF5_ERROR (memspace = H5Screate_simple (dim, chunk_dims, NULL));
       HDF5_ERROR (H5Sselect_hyperslab (filespace, H5S_SELECT_SET,
-                           chunk_origin, NULL, chunk_dims, NULL));
+                                       chunk_origin, NULL, chunk_dims, NULL));
 
       HDF5_ERROR (H5Dread (dataset, rec_info->hdf5type, memspace, filespace,
                            H5P_DEFAULT, data));
 
       HDF5_ERROR (H5Sclose (memspace));
       HDF5_ERROR (H5Sclose (filespace));
-
-      free (chunk_dims);
-      free (chunk_origin);
     }
 
     /* send the index and the data to the non-IO processors */
@@ -679,13 +680,21 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
       var_info[2] = extras->rnpoints[proc];
       CACTUS_MPI_ERROR (MPI_Send (var_info, 3, PUGH_MPI_INT, proc,
                                   MPITAGBASE, pughGH->PUGH_COMM_WORLD));
-      CACTUS_MPI_ERROR (MPI_Send (data, extras->rnpoints[proc],
-                                  rec_info->mpi_type, proc, MPITAGBASE,
-                                  pughGH->PUGH_COMM_WORLD));
+      if (extras->rnpoints[proc] > 0)
+      {
+        CACTUS_MPI_ERROR (MPI_Send (data, extras->rnpoints[proc],
+                                    rec_info->mpi_type, proc, MPITAGBASE,
+                                    pughGH->PUGH_COMM_WORLD));
+      }
     }
   }
 
-  free (buffer);
+  if (buffer)
+  {
+    free (buffer);
+  }
+  free (chunk_dims);
+  free (chunk_origin);
 #endif /* CCTK_MPI */
 
   return (0);
@@ -694,10 +703,10 @@ static int IOHDF5Util_RestoreGA (hid_t dataset, int vindex, int timelevel,
 
 static herr_t processDataset (hid_t group, const char *datasetname, void *arg)
 {
-  ioGH *ioUtilGH;
-  ioHDF5UtilGH *myGH;
+  const ioGH *ioUtilGH;
+  const ioHDF5UtilGH *myGH;
   int vindex, vtype, gtype, timelevel, iteration, is_group, retval;
-  iterate_info_t *it_info = (iterate_info_t *) arg;
+  iterate_info_t *it_info = arg;
   recover_info_t rec_info;
   hid_t dataset;
   H5G_stat_t object_info;
@@ -741,8 +750,8 @@ static herr_t processDataset (hid_t group, const char *datasetname, void *arg)
     return (0);
   }
 
-  ioUtilGH = (ioGH *) CCTK_GHExtension (it_info->GH, "IO");
-  myGH = (ioHDF5UtilGH *) CCTK_GHExtension (it_info->GH, "IOHDF5Util");
+  ioUtilGH = CCTK_GHExtension (it_info->GH, "IO");
+  myGH = CCTK_GHExtension (it_info->GH, "IOHDF5Util");
 
   /* if we read in initial data via the file reader interface
      check whether the user wants to have this variable with a specific