There is doch a 'maximum number of open file descriptors' limitation

so I had undo the corresponding changes from my last commit.
Tested now with 1024 chunked files on a linux system.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusPUGHIO/IOHDF5/trunk@161 4825ed28-b72c-4eae-9704-e50c059e567d

1 files changed, 55 insertions, 1 deletions

diff --git a/src/util/hdf5_recombiner.c b/src/util/hdf5_recombiner.c
index fd305d3..9933942 100644
--- a/src/util/hdf5_recombiner.c
+++ b/src/util/hdf5_recombiner.c
@@ -9,11 +9,15 @@
    @version   $Id$
  @@*/
 
+#include "cctk.h"       /* HAVE_UNISTD_H */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>     /* sysconf(3) */
+#endif
 
-#include "cctk.h"
 #include "CactusPUGHIO/IOHDF5Util/src/ioHDF5UtilGH.h"
 
 /* the rcs ID and its dummy function to use it */
@@ -24,6 +28,11 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_util_hdf5_recombiner_c)
 /*****************************************************************************/
 /*                           macro definitions                               */
 /*****************************************************************************/
+/* number of file descriptors that should be reserved for system usage
+   (these are at least stdin, stdout, stderr; some MPI implementations
+    may need additional descriptors for internal use) */
+#define RESERVED_FILE_DESCRIPTORS  5
+
 /* macro to do an HDF5 call, check its return code, and print a warning
    in case of an error */
 #define CHECK_ERROR(hdf5_call)                                                \
@@ -52,6 +61,7 @@ CCTK_FILEVERSION(CactusPUGHIO_IOHDF5_util_hdf5_recombiner_c)
 static char *pathname = NULL;      /* pathname of the current object */
 static hid_t *infiles = NULL;      /* list of input file handles */
 static hid_t outfile = -1 ;        /* output file handle */
+static int max_filedescriptors = 0;/* maximum number of open files */
 static char **infilenames = NULL;  /* list of input filenames */
 static int nprocs = 0;             /* total number of processors */
 static int ioproc_every = 0;       /* I/O was done on every N'th processor */
@@ -218,6 +228,21 @@ int main (int argc, const char *const argv[])
     ninfiles++;
   }
 
+  /* determine maximum number of files opened simultaneously */
+  max_filedescriptors = sysconf (_SC_OPEN_MAX);
+  if (max_filedescriptors < 0)
+  {
+    fprintf (stderr, "WARNING: Cannot determine filehandle limit ! "
+                     "Assuming no limit...\n");
+  }
+  /* subtract by number of reserved file descriptors
+     and the one for the output file */
+  max_filedescriptors -= RESERVED_FILE_DESCRIPTORS + 1;
+  if (max_filedescriptors < 0)
+  {
+    max_filedescriptors = ninfiles;
+  }
+
   printf ("Recombining HDF5 data from %d processors, "
           "output was written to %d file(s)\n\n", nprocs, ninfiles);
 
@@ -267,6 +292,12 @@ int main (int argc, const char *const argv[])
                  infilenames[infile]);
         return (-1);
       }
+
+      /* close file if filehandle limit would be exceeded */
+      if (infile > max_filedescriptors)
+      {
+        CHECK_ERROR (H5Fclose (infiles[infile]));
+      }
     }
 
     free (template);
@@ -305,6 +336,10 @@ int main (int argc, const char *const argv[])
   /* finally, close all open files */
   for (infile = 0; infile < ninfiles; infile++)
   {
+    if (infile <= max_filedescriptors)
+    {
+      CHECK_ERROR (H5Fclose (infiles[infile]));
+    }
     free (infilenames[infile]);
   }
   free (infiles);
@@ -626,6 +661,16 @@ static int RecombineGroupData (const char *groupname)
       nchunks = nprocs % ninfiles;
     }
 
+    /* re-open the file if it was closed before */
+    if (infile > max_filedescriptors)
+    {
+#ifdef DEBUG
+      printf ("reopening input file '%s'\n", infilenames[infile]);
+#endif
+      CHECK_ERROR (infiles[infile] = H5Fopen (infilenames[infile],
+                                              H5F_ACC_RDONLY, H5P_DEFAULT));
+    }
+
     /* loop over all chunks of this input file */
     for (chunk = 0; chunk < nchunks; chunk++)
     {
@@ -761,6 +806,15 @@ static int RecombineGroupData (const char *groupname)
       CHECK_ERROR (H5Sclose (chunked_dataspace));
 
     } /* end of loop over all chunks in this input file */
+
+    /* close input file if filehandle limit would be exceeded */
+    if (infile > max_filedescriptors)
+    {
+#ifdef DEBUG
+      printf ("temporarily closing input file '%s'\n", infilenames[infile]);
+#endif
+      CHECK_ERROR (H5Fclose (infiles[infile]));
+    }
   } /* end of loop over all input files */
 
   /* close objects and free allocated resources */