From c6bf5a043d36c35ca60ad14651cb2b307d712578 Mon Sep 17 00:00:00 2001 From: tradke Date: Thu, 27 Feb 2003 11:44:05 +0000 Subject: Removed code which assumed that input files would need temporarily closed/reopened. HDF5 doesn't seem to have a limitation in the number of open files. git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5/trunk@160 4825ed28-b72c-4eae-9704-e50c059e567d --- src/util/hdf5_recombiner.c | 167 ++++++++++++--------------------------------- 1 file changed, 45 insertions(+), 122 deletions(-) diff --git a/src/util/hdf5_recombiner.c b/src/util/hdf5_recombiner.c index bdcdee8..fd305d3 100644 --- a/src/util/hdf5_recombiner.c +++ b/src/util/hdf5_recombiner.c @@ -9,17 +9,11 @@ @version $Id$ @@*/ -#include "cctk.h" - -#include -#include -#include +#include #include #include -#include -#ifdef HAVE_UNISTD_H -#include /* sysconf(3) */ -#endif + +#include "cctk.h" #include "CactusPUGHIO/IOHDF5Util/src/ioHDF5UtilGH.h" /* the rcs ID and its dummy function to use it */ @@ -30,13 +24,9 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_util_hdf5_recombiner_c) /*****************************************************************************/ /* macro definitions */ /*****************************************************************************/ -/* uncomment the following to get some debugging output */ -/* #define IOHDF5_DEBUG 1 */ - /* macro to do an HDF5 call, check its return code, and print a warning in case of an error */ #define CHECK_ERROR(hdf5_call) \ - do \ { \ int _error_code = hdf5_call; \ \ @@ -48,7 +38,7 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_util_hdf5_recombiner_c) __LINE__, #hdf5_call, _error_code); \ nerrors++; \ } \ - } while (0) + } /*****************************************************************************/ @@ -62,7 +52,6 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_util_hdf5_recombiner_c) static char *pathname = NULL; /* pathname of the current object */ static hid_t *infiles = NULL; /* list of input file handles */ static hid_t outfile = -1 ; /* output file handle */ -static int max_filehandles = 0; /* maximum number of open files */ static char **infilenames = NULL; /* list of input filenames */ static int nprocs = 0; /* total number of processors */ static int ioproc_every = 0; /* I/O was done on every N'th processor */ @@ -107,65 +96,38 @@ static int RecombineGroupData (const char *groupname); 0 for success, negative return values indicate an error @endreturndesc @@*/ -int main (int argc, char *argv[]) +int main (int argc, const char *const argv[]) { - int infile; int unchunked; - struct stat fileinfo; char *tmp, *template; - hid_t group, attr, dataspace; + hid_t infile, group, attr, dataspace; /* say hello to the user */ - printf ("\n"); - printf (" -----------------------------\n"); - printf (" Cactus 4 HDF5 File Recombiner\n"); - printf (" -----------------------------\n"); - printf ("\n"); + printf ("\n -----------------------------" + "\n Cactus 4 HDF5 File Recombiner" + "\n -----------------------------\n\n"); /* give some help if called with incorrect number of parameters */ if (argc != 3) { - fprintf (stderr, "Usage: %s \n", - argv[0]); - fprintf (stderr, " eg, %s alp.file_0.h5 alp.h5\n\n", argv[0]); + fprintf (stderr, "Usage: %s \n" + " eg, %s alp.file_0.h5 alp.h5\n\n", argv[0], argv[0]); return (0); } - /* determine maximum number of files opened simultaneously */ - max_filehandles = sysconf (_SC_OPEN_MAX); - if (max_filehandles < 0) - { - fprintf (stderr, "WARNING: Cannot determine filehandle limit ! " - "Assuming no limit...\n"); - } - /* subtract stdin, stdout, stderr, and output filehandle */ - max_filehandles -= 4; - /* open (first) input file */ - if (stat (argv[1], &fileinfo)) - { - fprintf (stderr, "ERROR: Cannot open input file '%s': %s\n\n", - argv[1], strerror (errno)); - return (-1); - } - if (! H5Fis_hdf5 (argv[1])) + H5E_BEGIN_TRY { - fprintf (stderr, "ERROR: Input file '%s' is not an HDF5 file !\n\n", - argv[1]); - return (-1); - } - infiles = (hid_t *) malloc (1 * sizeof (hid_t)); - infiles[0] = H5Fopen (argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); - if (infiles[0] < 0) + infile = H5Fopen (argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + } H5E_END_TRY + if (infile < 0) { - fprintf (stderr, "ERROR: Cannot open input file '%s' as an HDF5 file !\n\n", - argv[1]); + fprintf (stderr, "ERROR: Cannot open HDF5 input file '%s' !\n\n", argv[1]); return (-1); } - /* read the file set info from the GLOBAL_ATTRIBUTES_GROUP group */ - CHECK_ERROR (group = H5Gopen (infiles[0], GLOBAL_ATTRIBUTES_GROUP)); + CHECK_ERROR (group = H5Gopen (infile, GLOBAL_ATTRIBUTES_GROUP)); if (group < 0) { fprintf (stderr, "ERROR: Cannot open attribute group '" @@ -255,17 +217,14 @@ int main (int argc, char *argv[]) { ninfiles++; } - if (max_filehandles < 0) - { - max_filehandles = ninfiles; - } printf ("Recombining HDF5 data from %d processors, " "output was written to %d file(s)\n\n", nprocs, ninfiles); /* allocate arrays for input file descriptors and filenames */ - infiles = (hid_t *) realloc (infiles, ninfiles * sizeof (hid_t)); - infilenames = (char **) malloc (ninfiles * sizeof (char *)); + infiles = malloc (ninfiles * sizeof (hid_t)); + infiles[0] = infile; + infilenames = malloc (ninfiles * sizeof (char *)); /* now get all chunked input filenames and check that they can be opened */ if (ninfiles == 1) @@ -289,7 +248,7 @@ int main (int argc, char *argv[]) } /* build the input filename template */ - template = (char *) malloc (strlen (argv[1]) + 2); + template = malloc (strlen (argv[1]) + 2); strcpy (template, argv[1]); template[tmp - argv[1] + 6] = 0; strcat (template, "%d.h5"); @@ -298,7 +257,7 @@ int main (int argc, char *argv[]) for (infile = 0; infile < ninfiles; infile++) { /* build the input filename */ - infilenames[infile] = (char *) malloc (strlen (template) + 10); + infilenames[infile] = malloc (strlen (template) + 10); sprintf (infilenames[infile], template, infile); infiles[infile] = H5Fopen (infilenames[infile], H5F_ACC_RDONLY, H5P_DEFAULT); @@ -308,12 +267,6 @@ int main (int argc, char *argv[]) infilenames[infile]); return (-1); } - - /* close file if filehandle limit would be exceeded */ - if (infile > max_filehandles) - { - CHECK_ERROR (H5Fclose (infiles[infile])); - } } free (template); @@ -352,10 +305,6 @@ int main (int argc, char *argv[]) /* finally, close all open files */ for (infile = 0; infile < ninfiles; infile++) { - if (infile <= max_filehandles) - { - CHECK_ERROR (H5Fclose (infiles[infile])); - } free (infilenames[infile]); } free (infiles); @@ -423,13 +372,12 @@ static herr_t CopyObject (hid_t from, char *current_pathname; int retval; size_t objectsize; - void *value; + void *data; /* build the full pathname for the current to object to process */ current_pathname = pathname; - pathname = (char *) malloc (strlen (current_pathname) + - strlen (objectname) + 2); + pathname = malloc (strlen (current_pathname) + strlen (objectname) + 2); sprintf (pathname, "%s/%s", current_pathname, objectname); /* get the output object identifier */ @@ -466,19 +414,13 @@ static herr_t CopyObject (hid_t from, CHECK_ERROR (dataspace = H5Dget_space (from)); CHECK_ERROR (to = H5Dcreate (to, objectname, datatype, dataspace, H5P_DEFAULT)); - objectsize = H5Tget_size (datatype); - if (H5Sis_simple (dataspace) > 0) - { - objectsize *= H5Sget_simple_extent_npoints (dataspace); - } + objectsize = H5Dget_storage_size (from); if (objectsize > 0) { - value = malloc (objectsize); - CHECK_ERROR (H5Dread (from, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, - value)); - CHECK_ERROR (H5Dwrite (to, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, - value)); - free (value); + data = malloc (objectsize); + CHECK_ERROR (H5Dread (from, datatype, H5S_ALL, H5S_ALL,H5P_DEFAULT,data)); + CHECK_ERROR (H5Dwrite (to, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT,data)); + free (data); } CHECK_ERROR (H5Aiterate (from, NULL, CopyAttribute, &to)); CHECK_ERROR (H5Dclose (to)); @@ -516,7 +458,7 @@ static herr_t CopyObject (hid_t from, @vtype hid_t @vio in @endvar - @var attrname + @var name @vdesc name of the current attribute @vtype const char * @vio in @@ -534,7 +476,7 @@ static herr_t CopyObject (hid_t from, @endreturndesc @@*/ static herr_t CopyAttribute (hid_t from, - const char *attrname, + const char *name, void *_to) { hid_t attr, datatype, dataspace, to; @@ -547,7 +489,7 @@ static herr_t CopyAttribute (hid_t from, /* open the attribute given by its name, get type, dataspace, and value and just copy it */ - CHECK_ERROR (attr = H5Aopen_name (from, attrname)); + CHECK_ERROR (attr = H5Aopen_name (from, name)); CHECK_ERROR (datatype = H5Aget_type (attr)); CHECK_ERROR (dataspace = H5Aget_space (attr)); attrsize = H5Tget_size (datatype); @@ -560,8 +502,7 @@ static herr_t CopyAttribute (hid_t from, value = malloc (attrsize); CHECK_ERROR (H5Aread (attr, datatype, value)); CHECK_ERROR (H5Aclose (attr)); - CHECK_ERROR (attr = H5Acreate (to, attrname, datatype, dataspace, - H5P_DEFAULT)); + CHECK_ERROR (attr = H5Acreate (to, name, datatype, dataspace, H5P_DEFAULT)); CHECK_ERROR (H5Awrite (attr, datatype, value)); free (value); } @@ -635,15 +576,12 @@ static int RecombineGroupData (const char *groupname) "of type integer !\n", groupname); ndims = 0; } + global_size = NULL; if (ndims > 0) { - global_size = (hsize_t *) calloc (ndims, sizeof (hsize_t)); + global_size = calloc (ndims, sizeof (hsize_t)); CHECK_ERROR (H5Aread (attr, H5T_NATIVE_HSIZE, global_size)); } - else - { - global_size = NULL; - } CHECK_ERROR (H5Sclose (dataspace)); CHECK_ERROR (H5Tclose (datatype)); CHECK_ERROR (H5Aclose (attr)); @@ -656,7 +594,7 @@ static int RecombineGroupData (const char *groupname) } /* allocate string buffer holding the possible dataset chunk names */ - chunkname = (char *) malloc (strlen (groupname) + sizeof ("chunk") + 20); + chunkname = malloc (strlen (groupname) + sizeof ("chunk") + 20); /* the unchunked dataset's dimensions are taken from the 'global_size' attribute, with the least changing dimension being the first element */ @@ -672,8 +610,8 @@ static int RecombineGroupData (const char *groupname) free (global_size); /* allocate buffers to read a 'chunk_origin' attribute and the chunk dims */ - chunk_origin = (hssize_t *) calloc (ndims, sizeof (hssize_t)); - chunk_dims = (hsize_t *) calloc (ndims, sizeof (hsize_t)); + chunk_origin = calloc (ndims, sizeof (hssize_t)); + chunk_dims = calloc (ndims, sizeof (hsize_t)); unchunked_dataset = unchunked_datatype = -1; @@ -688,16 +626,6 @@ static int RecombineGroupData (const char *groupname) nchunks = nprocs % ninfiles; } - /* re-open the file if it was closed before */ - if (infile > max_filehandles) - { -#ifdef DEBUG - printf ("reopening input file '%s'\n", infilenames[infile]); -#endif - CHECK_ERROR (infiles[infile] = H5Fopen (infilenames[infile], - H5F_ACC_RDONLY, H5P_DEFAULT)); - } - /* loop over all chunks of this input file */ for (chunk = 0; chunk < nchunks; chunk++) { @@ -807,13 +735,17 @@ static int RecombineGroupData (const char *groupname) if (chunksize > 0) { /* give some info output */ - printf (" - file %d chunk %d\n", infile, chunk + ioproc_every*infile); + printf (" - file %d chunk %d\n", infile,chunk + ioproc_every*infile); printf (" chunk dimensions: [%d", (int) chunk_dims[ndims - 1]); for (chunk_ndims = 1; chunk_ndims < ndims; chunk_ndims++) - printf (", %d", (int) chunk_dims[ndims - chunk_ndims - 1]); + { + printf (", %d", (int) chunk_dims[ndims - chunk_ndims - 1]); + } printf ("] chunk origin: [%d", (int) chunk_origin[ndims - 1]); for (chunk_ndims = 1; chunk_ndims < ndims; chunk_ndims++) - printf (", %d", (int) chunk_origin[ndims - chunk_ndims - 1]); + { + printf (", %d", (int) chunk_origin[ndims - chunk_ndims - 1]); + } printf ("]\n"); /* read the chunk and write it to the unchunked dataset */ @@ -829,15 +761,6 @@ static int RecombineGroupData (const char *groupname) CHECK_ERROR (H5Sclose (chunked_dataspace)); } /* end of loop over all chunks in this input file */ - - /* close input file if filehandle limit would be exceeded */ - if (infile > max_filehandles) - { -#ifdef DEBUG - printf ("temporarily closing input file '%s'\n", infilenames[infile]); -#endif - CHECK_ERROR (H5Fclose (infiles[infile])); - } } /* end of loop over all input files */ /* close objects and free allocated resources */ -- cgit v1.2.3