From d2552f96b45a95001e024a05923373344e851f1a Mon Sep 17 00:00:00 2001 From: Thomas Radke Date: Mon, 27 Jun 2005 14:17:00 +0000 Subject: CarpetIOStreamedHDF5: new thorn which provides streamed HDF5 data output darcs-hash:20050627141701-776a0-cba926db30167a725a6dad6b584a31fa5476b5e0.gz --- Carpet/CarpetIOStreamedHDF5/README | 18 + Carpet/CarpetIOStreamedHDF5/configuration.ccl | 5 + Carpet/CarpetIOStreamedHDF5/doc/documentation.tex | 159 ++++++++ Carpet/CarpetIOStreamedHDF5/interface.ccl | 12 + .../par/CarpetIOStreamedHDF5.par | 59 +++ Carpet/CarpetIOStreamedHDF5/param.ccl | 51 +++ Carpet/CarpetIOStreamedHDF5/schedule.ccl | 19 + .../src/CarpetIOStreamedHDF5.cc | 442 +++++++++++++++++++++ .../src/CarpetIOStreamedHDF5.hh | 48 +++ Carpet/CarpetIOStreamedHDF5/src/make.code.defn | 4 + .../src/make.configuration.defn | 6 + 11 files changed, 823 insertions(+) create mode 100644 Carpet/CarpetIOStreamedHDF5/README create mode 100644 Carpet/CarpetIOStreamedHDF5/configuration.ccl create mode 100644 Carpet/CarpetIOStreamedHDF5/doc/documentation.tex create mode 100644 Carpet/CarpetIOStreamedHDF5/interface.ccl create mode 100644 Carpet/CarpetIOStreamedHDF5/par/CarpetIOStreamedHDF5.par create mode 100644 Carpet/CarpetIOStreamedHDF5/param.ccl create mode 100644 Carpet/CarpetIOStreamedHDF5/schedule.ccl create mode 100644 Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.cc create mode 100644 Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.hh create mode 100644 Carpet/CarpetIOStreamedHDF5/src/make.code.defn create mode 100644 Carpet/CarpetIOStreamedHDF5/src/make.configuration.defn (limited to 'Carpet/CarpetIOStreamedHDF5') diff --git a/Carpet/CarpetIOStreamedHDF5/README b/Carpet/CarpetIOStreamedHDF5/README new file mode 100644 index 000000000..57009b177 --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/README @@ -0,0 +1,18 @@ +Cactus Code Thorn CarpetIOStreamedHDF5 +Thorn Author(s) : Thomas Radke +Thorn Maintainer(s) : Thomas Radke +-------------------------------------------------------------------------- + +Purpose of the thorn: + +This thorn uses the HDF5 Stream Virtual File Driver to stream Carpet HDF5 data +files via a socket to any connected clients. +In combination with client programs, which are capable of receiving and +postprocessing streamed HDF5 data, this thorn can be used to perform online +remote visualisation of live data from running Carpet FMR/AMR simulations. + +Further information: + + * this thorn's doc/documentation.tex + * HDF5: http://hdf.ncsa.uiuc.edu/HDF5/ + * Cactus Visualisation Tools: http://www.cactuscode.org/VizTools.html diff --git a/Carpet/CarpetIOStreamedHDF5/configuration.ccl b/Carpet/CarpetIOStreamedHDF5/configuration.ccl new file mode 100644 index 000000000..da13af17d --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/configuration.ccl @@ -0,0 +1,5 @@ +# Configuration definitions for thorn CarpetIOStreamedHDF5 + +REQUIRES Socket CarpetLib + +REQUIRES THORNS: CarpetIOHDF5 diff --git a/Carpet/CarpetIOStreamedHDF5/doc/documentation.tex b/Carpet/CarpetIOStreamedHDF5/doc/documentation.tex new file mode 100644 index 000000000..419af19aa --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/doc/documentation.tex @@ -0,0 +1,159 @@ +\documentclass{article} + +\usepackage{../../../../doc/latex/cactus} + +\begin{document} + +\author{Thomas Radke \textless tradke@aei.mpg.de\textgreater} + +\title{CarpetIOStreamedHDF5} + +% the date your document was last changed, if your document is in CVS, +% please use: +% \date{$ $Date: 2004/01/07 20:12:39 $ $} +%\date{June 24 2005} +\date{} + +\maketitle + +% Do not delete next line +% START CACTUS THORNGUIDE + +\newcommand{\ThisThorn}{{\it CarpetIOStreamedHDF5}} + +\begin{abstract} +Thorn \ThisThorn\ provides an I/O method to stream Cactus grid +variables in HDF5 file format via a socket to any connected clients. +In combination with client programs, which are capable of receiving and +postprocessing streamed HDF5 data, this thorn can be used to perform online +remote visualisation of live data from running Carpet FMR/AMR simulations. +\end{abstract} + + +\section{Introduction} + +Thorn \ThisThorn\ uses the standard I/O library HDF5 (Hierarchical +Data Format version 5) to output any type of Cactus grid variables +(grid scalars, grid functions, and grid arrays of arbitrary dimension) +in the HDF5 file format. + +Streamed output is enabled by activating thorn \ThisThorn\ in your parameter +file. At simulation startup it registers its own I/O method with the +flesh's I/O interface and opens a server port on the root processor +(with processor ID 0) to which clients can connect to. +Like any Cactus I/O method, it will then check periodically after each iteration +whether output should be done for grid variables as chosen in the +parameter file. + +Data is streamed as a serialized HDF5 file to all clients which try to +connect to the server port at the time of output. If multiple variables are to +be output at the same time they will all be sent in a single streamed HDF5 file. + +It should be noticed here that, due to the implementation of data streaming +in the HDF5 library, streaming many variables (or a single variable with many +refinement levels and/or a large global grid size) can be a costly +operation in terms of memory requirements, as the resulting HDF5 file has +to be kept in main memory before it gets sent to a client. You should instead +switch on streamed HDF5 output only for those variables/refinement levels +which you are currently interested in visualising; the corresponding +I/O parameter is steerable so you can select other variables and/or levels +any time. + + +\section{\ThisThorn\ Parameters} + +Parameters to control the \ThisThorn\ I/O method are: +\begin{itemize} + \item {\tt IOStreamedHDF5::port}\\ + The initial port number which should be opened at startup for + client connections. If the given port is already in use by some other + program, {\tt IOStreamedHDF5} will search for the next available port + by advancing {\tt IOStreamedHDF5::port}.\\ + The actual port number used by \ThisThorn\ will be output in a INFO + message at startup, along with the hostname of the root processor. + + \item {\tt IOStreamedHDF5::out\_every} (steerable)\\ + How often to do periodic \ThisThorn\ output. If this parameter + is set in the parameter file, it will override the setting of the shared + {\tt IO::out\_every} parameter. The output frequency can also be set + for individual variables using the {\tt out\_every} option in an option + string appended to the {\tt IOStreamedHDF5::out\_vars} parameter. + + \item {\tt IOStreamedHDF5::out\_dt} (steerable)\\ + output in intervals of that much coordinate time (overwrites {\tt IO::ou +t\_dt}) + + \item {\tt IOStreamedHDF5::out\_criterion} (steerable)\\ + criterion to select output intervals (overwrites {\tt IO::out\_criterion +}) + + \item {\tt IOStreamedHDF5::out\_vars} (steerable)\\ + The list of variables to output using the \ThisThorn\ I/O method. + The variables must be given by their fully qualified variable or group + name. Multiple names must be separated by whitespaces.\\ + + Each group/variable name can have an option string attached in which you + can specify a different output frequency for that individual variable + or a set of individual refinement levels to be output, e.g. +\begin{verbatim} + IOStreamedHDF5::out_vars = "wavetoy::phi{ out_every = 4 refinement_levels = { 1 2 } }" +\end{verbatim} + + \item {\tt IO::out\_single\_precision (steerable)}\\ + whether to output double-precision data in single precision + + \item {\tt IO::out\_unchunked}\\ + whether to output the data from multiple processors in chunked or + unchunked format + +\end{itemize} + + +\section{A Practical Session of Remote Online Visualisation} + +The following steps illustrate how a practical session of visualising +live data from a Carpet simulation with the DataVault visualisation tool +could look like. +It is assumed that Cactus is running as a PBS job on a parallel cluster, +with the compute nodes behind a cluster firewall (only the cluster head node +can be directly accessed from outside). + +\begin{enumerate} + \item[0.] Your job has been submitted to PBS and shortly after begins its + execution.\\ + Let's assume that you want to run the HDF5 data streaming + demo parameter file {\tt CarpetIOStreamedHDF5.par} contained in the + {\tt par/} subdirectory of thorn \ThisThorn. + + \item Grep the stdout of your job's startup messages for a line + containing '{\tt CarpetIOStreamedHDF5}' and '{\tt data streaming + service started on}'. + This tells you the hostname and port number to connect to (eg. {\tt ic0010:10000}). + + \item On the head node, set the {\tt DVHOST} shell environment variable + to the hostname of the machine which runs the DV server (ideally + your laptop or local workstation).\\ + Then run the {\tt hdf5todv} client + with the URL of your Cactus simulation's data streaming service, eg. + '{\tt hdf5todv ic0010:10000}'. This should receive one timestep of + chosen \ThisThorn\ variables from the simulation and + send it on to DV. The new timestep should appear there as a new + register. + + \item Repeat the previous step as often as you like in order + to get a sequence of timesteps which you then can animate. + +\end{enumerate} + +\section{Further Information} + +More information on HDF5 can be found on the HDF5 home page +\url{http://hdf.ncsa.uiuc.edu/whatishdf5.html}. + +The list of tools for visualising Cactus and Carpet output data can be found +on the Cactus VisTools page \url{http://www.cactuscode.org/VizTools.html}. + +% Do not delete next line +% END CACTUS THORNGUIDE + +\end{document} diff --git a/Carpet/CarpetIOStreamedHDF5/interface.ccl b/Carpet/CarpetIOStreamedHDF5/interface.ccl new file mode 100644 index 000000000..34c52f458 --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/interface.ccl @@ -0,0 +1,12 @@ +# Interface definition for thorn CarpetIOStreamedHDF5 + +implements: IOStreamedHDF5 +inherits: IO + +uses include header: SocketUtils.h +uses include header: CarpetIOHDF5.hh + +private: +INT next_output_iteration TYPE = SCALAR +REAL next_output_time TYPE = SCALAR +INT this_iteration TYPE = SCALAR diff --git a/Carpet/CarpetIOStreamedHDF5/par/CarpetIOStreamedHDF5.par b/Carpet/CarpetIOStreamedHDF5/par/CarpetIOStreamedHDF5.par new file mode 100644 index 000000000..7818242dd --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/par/CarpetIOStreamedHDF5.par @@ -0,0 +1,59 @@ +# /*@@ +# @file CarpetIOStreamedHDF5.par +# @date Sunday 25 June 2005 +# @author Thomas Radke +# @desc +# WaveToy parameter file demonstrating streamed HDF5 output +# for a WaveBinarySource simulation +# @enddesc +# @@*/ + +ActiveThorns = "Boundary Time CartGrid3D CoordBase SymBase" +ActiveThorns = "IOUtil IOBasic" +ActiveThorns = "Carpet CarpetLib CarpetRegrid CarpetReduce CarpetSlab" +ActiveThorns = "CarpetIOHDF5 CarpetIOStreamedHDF5" +ActiveThorns = "IDScalarWaveC WaveToyC WaveBinarySource" + +#################### +# Driver parameters +#################### +Driver::global_nsize = 40 + +Carpet::max_refinement_levels = 4 +CarpetRegrid::refinement_levels = 4 +CarpetRegrid::refined_regions = "manual-gridpoint-list" +CarpetRegrid::gridpoints = "[ + [ ([ 60, 60, 60]:[260,260,260]:[4,4,4]) ], + [ ([ 80,110, 80]:[160,210,150]:[2,2,2]), + ([ 80,110,170]:[160,210,240]:[2,2,2]) ], + [ ([100,135, 90]:[140,185,140]:[1,1,1]), + ([135,135,180]:[140,185,230]:[1,1,1]) ] + ]" + +######################### +# Application parameters +######################### +Cactus::terminate = "never" + +Grid::type = "BySpacing" +Grid::domain = "full" +Grid::dxyz = 0.005 + +Time::dtfac = 0.5 + +WaveToy::bound = "radiation" + +WaveBinarySource::binary_omega = 26 +WaveBinarySource::binary_charge = 0.00001 +WaveBinarySource::binary_radius = 0.07 +WaveBinarySource::binary_size = 0.04 + +#################### +# output parameters +#################### +IO::out_every = 16 +IO::out_single_precision = "yes" +IO::parfile_write = "no" + +IOBasic::outInfo_vars = "wavetoy::phi" +IOStreamedHDF5::out_vars = "wavetoy::phi" diff --git a/Carpet/CarpetIOStreamedHDF5/param.ccl b/Carpet/CarpetIOStreamedHDF5/param.ccl new file mode 100644 index 000000000..6db82cc93 --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/param.ccl @@ -0,0 +1,51 @@ +# Parameter definitions for thorn CarpetIOStreamedHDF5 + +private: + +STRING out_vars "Variables to stream in HDF5 file format" STEERABLE = ALWAYS +{ + ".+" :: "Space-separated list of fully qualified variable/group names" + "^$" :: "An empty string to output nothing" +} "" + +KEYWORD out_criterion "Criterion to select CarpetIOHDF5 output intervals, overrides out_every" STEERABLE = ALWAYS +{ + "default" :: "Use 'IO::out_criterion'" + "never" :: "Never output" + "iteration" :: "Output every so many iterations" + "divisor" :: "Output if (iteration % out_every) == 0." + "time" :: "Output every that much coordinate time" +} "default" + +INT out_every "How often to do HDF5 streaming, overrides IO::out_every" STEERABLE = ALWAYS +{ + 1:* :: "Output every so many time steps" + -1:0 :: "No output" + -2 :: "Use 'IO::out_every'" +} -2 + +REAL out_dt "How often to do CarpetIOHDF5 output, overrides 'IO::out_dt'" STEERABLE = ALWAYS +{ + (0:* :: "In intervals of that much coordinate time" + 0 :: "As often as possible" + -1 :: "Disable output" + -2 :: "Default to 'IO::out_dt'" +} -2 + +INT port "Port number for clients to connect to" STEERABLE = RECOVER +{ + 1000:32000 :: "Ports below 1000 require root access" +} 10000 + + +############################################################################# +### import IOUtil parameters +############################################################################# +shares: IO + +USES INT out_every AS io_out_every +USES REAL out_dt AS io_out_dt +USES KEYWORD out_criterion AS io_out_criterion +USES KEYWORD verbose +USES BOOLEAN out_unchunked +USES BOOLEAN strict_io_parameter_check diff --git a/Carpet/CarpetIOStreamedHDF5/schedule.ccl b/Carpet/CarpetIOStreamedHDF5/schedule.ccl new file mode 100644 index 000000000..5b2edb9ae --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/schedule.ccl @@ -0,0 +1,19 @@ +# Schedule definitions for thorn CarpetIOStreamedHDF5 + +storage: next_output_iteration next_output_time this_iteration + +schedule CarpetIOStreamedHDF5_Startup at STARTUP +{ + LANG: C +} "Open the socket to be used for HDF5 data streaming" + +schedule CarpetIOStreamedHDF5_Init at BASEGRID +{ + LANG: C + OPTIONS: global +} "Initialise I/O scalars" + +schedule CarpetIOStreamedHDF5_Terminate at TERMINATE +{ + LANG:C +} "Close the socket used for HDF5 data streaming" diff --git a/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.cc b/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.cc new file mode 100644 index 000000000..65e8ccdbf --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.cc @@ -0,0 +1,442 @@ +#include + +#include "cctk.h" +#include "cctk_Arguments.h" +#include "cctk_Parameters.h" +#include "util_String.h" +#include "util_Network.h" + +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "CactusBase/IOUtil/src/ioGH.h" + +#include "CarpetIOStreamedHDF5.hh" + + +namespace CarpetIOStreamedHDF5 +{ + +using namespace std; +using namespace Carpet; +using namespace CarpetIOHDF5; + + +// Variable definitions +static vector > > last_output; // [ml][rl][var] + +// registered GH extension setup routine +static void* SetupGH (tFleshConfig* const fleshconfig, + const int convLevel, cGH* const cctkGH); + +// callbacks for CarpetIOStreamedHDF5's I/O method +static int OutputGH (const cGH* const cctkGH); +static int TimeToOutput (const cGH* const cctkGH, const int vindex); +static int OutputVar (const cGH* const cctkGH, const ioRequest* const request, + hid_t file); + +static void CheckSteerableParameters (const cGH *const cctkGH, + CarpetIOStreamedHDF5GH *myGH); + +////////////////////////////////////////////////////////////////////////////// +// public routines +////////////////////////////////////////////////////////////////////////////// + +void CarpetIOStreamedHDF5_Startup (void) +{ + CCTK_RegisterBanner ("AMR streamed HDF5 output " + "provided by CarpetIOStreamedHDF5"); + + const int GHExtension = CCTK_RegisterGHExtension (CCTK_THORNSTRING); + CCTK_RegisterGHExtensionSetupGH (GHExtension, SetupGH); +} + + +void CarpetIOStreamedHDF5_Init (const cGH* const cctkGH) +{ + DECLARE_CCTK_ARGUMENTS; + + *this_iteration = -1; + *next_output_iteration = 0; + *next_output_time = cctk_time; +} + + +// close the socket used for HDF5 streaming +void CarpetIOStreamedHDF5_Terminate (const cGH* const cctkGH) +{ + CarpetIOStreamedHDF5GH* myGH = + (CarpetIOStreamedHDF5GH*) CCTK_GHExtension (cctkGH, CCTK_THORNSTRING); + + Socket_CloseSocket (myGH->socket); +} + + +////////////////////////////////////////////////////////////////////////////// +// private routines +////////////////////////////////////////////////////////////////////////////// +static void* SetupGH (tFleshConfig* const fleshconfig, + const int convLevel, cGH* const cctkGH) +{ + DECLARE_CCTK_PARAMETERS; + + // processor 0 opens a socket on the given port to listen for clients + unsigned int real_port; + SOCKET real_socket = INVALID_SOCKET; + if (dist::rank() == 0) { + + real_socket = Socket_TCPOpenServerSocket (port, &real_port, 1); + + if (real_socket == INVALID_SOCKET) { + + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Couldn't open TCP server socket on output port %d. " + "No HDF5 streaming output will be available !", port); + + } else if (Socket_SetNonBlocking (real_socket) < 0) { + + CCTK_WARN (1, "Couldn't set output socket into non-blocking mode. " + "No HDF5 streaming output will be available !"); + Socket_CloseSocket (real_socket); + real_socket = INVALID_SOCKET; + + } + } + + // broadcast success of socket operation to all processors + int have_socket = real_socket != INVALID_SOCKET; + MPI_Bcast (&have_socket, 1, MPI_INT, 0, dist::comm); + + // do not continue here if no socket is available + if (not have_socket) { + return (NULL); + } + + if (dist::rank() == 0) { + char hostname[256]; + Util_GetHostName (hostname, sizeof (hostname)); + CCTK_VInfo (CCTK_THORNSTRING, + "data streaming service started on '%s:%u'", hostname, port); + } + + // register CarpetIOStreamedHDF5's routines as a new I/O method + const int IOMethod = CCTK_RegisterIOMethod ("IOStreamedHDF5"); + CCTK_RegisterIOMethodOutputGH (IOMethod, OutputGH); +#if 0 + // for now only register the OutputGH() callback + CCTK_RegisterIOMethodOutputVarAs (IOMethod, OutputVarAs); + CCTK_RegisterIOMethodTimeToOutput (IOMethod, TimeToOutput); + CCTK_RegisterIOMethodTriggerOutput (IOMethod, TriggerOutput); +#endif + + if (not CCTK_Equals (verbose, "none")) { + CCTK_INFO ("I/O Method 'IOStreamedHDF5' registered: AMR streamed HDF5 " + "output of grid variables"); + } + + const int numvars = CCTK_NumVars (); + + // allocate a new GH extension structure + CarpetIOStreamedHDF5GH* myGH = new CarpetIOStreamedHDF5GH; + + myGH->out_last.resize(numvars); + for (int i = 0; i < numvars; i++) { + myGH->out_last[i] = -1; + } + myGH->requests.resize(numvars); + myGH->out_vars = strdup (""); + myGH->out_every_default = out_every - 1; + + // initial I/O parameter check + myGH->stop_on_parse_errors = strict_io_parameter_check; + CheckSteerableParameters (cctkGH, myGH); + myGH->stop_on_parse_errors = 0; + + // no iterations have yet been output + last_output.resize(mglevels); + for (int i = 0; i < mglevels; i++) { + last_output[i].resize (maxreflevels); + for (int j = 0; j < maxreflevels; j++) { + last_output[i][j].resize (numvars, INT_MIN); + } + } + + myGH->socket = real_socket; + myGH->port = real_port; + + return (myGH); +} + + +static void CheckSteerableParameters (const cGH *const cctkGH, + CarpetIOStreamedHDF5GH *myGH) +{ + DECLARE_CCTK_PARAMETERS; + + // re-parse the 'IOHDF5::out_vars' parameter if it has changed + if (strcmp (out_vars, myGH->out_vars)) { + IOUtil_ParseVarsForOutput (cctkGH, CCTK_THORNSTRING, + "IOStreamedHDF5::out_vars", + myGH->stop_on_parse_errors, out_vars, + -1, &myGH->requests[0]); + + // notify the user about the new setting + if (not CCTK_Equals (verbose, "none")) { + char *msg = NULL; + for (int i = CCTK_NumVars () - 1; i >= 0; i--) { + if (myGH->requests[i]) { + char *fullname = CCTK_FullName (i); + if (not msg) { + Util_asprintf (&msg, "Periodic streamed HDF5 output requested " + "for '%s'", fullname); + } else { + Util_asprintf (&msg, "%s, '%s'", msg, fullname); + } + free (fullname); + } + } + if (msg) { + CCTK_INFO (msg); + free (msg); + } + } + + // save the last setting of 'IOHDF5::out_vars' parameter + free (myGH->out_vars); + myGH->out_vars = strdup (out_vars); + } +} + + +static int OutputGH (const cGH* const cctkGH) +{ + DECLARE_CCTK_PARAMETERS; + + CarpetIOStreamedHDF5GH *myGH = + (CarpetIOStreamedHDF5GH *) CCTK_GHExtension (cctkGH, CCTK_THORNSTRING); + + static hid_t file = -1; + static int client_is_ready = 0; + + // loop over all variables + for (int vindex = CCTK_NumVars () - 1; vindex >= 0; vindex--) { + if (TimeToOutput (cctkGH, vindex)) { + + // check if any client is ready to receive streamed data + // + // Even though this requires a broadcast operation, it should be + // by far less expensive than gathering all the data on processor 0 + // and find out afterwards that no client wants to have it. + if (not client_is_ready) { + if (file < 0 and dist::rank() == 0) { + fd_set read_set; + FD_ZERO (&read_set); + FD_SET (myGH->socket, &read_set); + + struct timeval timeout = {0, 0}; + client_is_ready = + select (FD_SETSIZE, &read_set, NULL, NULL, &timeout) == 1; + } + MPI_Bcast (&client_is_ready, 1, MPI_INT, 0, dist::comm); + } + + // short cut if there is nothing to do + if (not client_is_ready) { + continue; + } + + // when called the first time during the current iteration, + // open the file on processor 0 + if (file < 0 and dist::rank() == 0) { + + if (CCTK_Equals (verbose, "full")) { + CCTK_VInfo (CCTK_THORNSTRING, "Opening HDF5 output file on output " + "port %u", myGH->port); + } + + // set file access property list to use the Stream VFD and open the file + H5FD_stream_fapl_t fapl; + fapl.increment = 0; + fapl.socket = myGH->socket; + fapl.do_socket_io = 1; + fapl.backlog = 5; + fapl.broadcast_fn = NULL; + fapl.broadcast_arg = NULL; + + hid_t plist; + HDF5_ERROR (plist = H5Pcreate (H5P_FILE_ACCESS)); + HDF5_ERROR (H5Pset_fapl_stream (plist, &fapl)); + + // a filename is not used by Stream VFD + assert (file < 0); + HDF5_ERROR (file = H5Fcreate ("unused", H5F_ACC_TRUNC, H5P_DEFAULT, + plist)); + HDF5_ERROR (H5Pclose (plist)); + } + assert (dist::rank() or file >= 0); + + OutputVar (cctkGH, myGH->requests[vindex], file); + } + } + + // close an open file if the finest level has been output + if (reflevel == reflevels - 1) { + if (dist::rank() == 0 and file >= 0) { + if (CCTK_Equals (verbose, "full")) { + CCTK_VInfo (CCTK_THORNSTRING, "Closing HDF5 output file on port %u", + myGH->port); + } + HDF5_ERROR (H5Fclose (file)); + file = -1; + } + client_is_ready = 0; + } + + return (0); +} + + +static int TimeToOutput (const cGH* const cctkGH, const int vindex) +{ + DECLARE_CCTK_ARGUMENTS; + DECLARE_CCTK_PARAMETERS; + + const int numvars = CCTK_NumVars(); + assert (vindex>=0 and vindexrequests[vindex]) { + return (0); + } + + // check whether this refinement level should be output + if (not (myGH->requests[vindex]->refinement_levels & (1 << reflevel))) { + return (0); + } + + // check if output for this variable was requested individually + // by a "{ out_every = }" option string + // this will overwrite the output criterion setting + const char *myoutcriterion = CCTK_EQUALS (out_criterion, "default") ? + io_out_criterion : out_criterion; + if (myGH->requests[vindex]->out_every >= 0) { + myoutcriterion = "divisor"; + } + + if (CCTK_EQUALS (myoutcriterion, "never")) { + return (0); + } + + // check whether to output at this iteration + bool output_this_iteration = false; + + if (CCTK_EQUALS (myoutcriterion, "iteration")) { + int myoutevery = out_every == -2 ? io_out_every : out_every; + if (myoutevery > 0) { + if (*this_iteration == cctk_iteration) { + // we already decided to output this iteration + output_this_iteration = true; + } else if (cctk_iteration >= *next_output_iteration) { + // it is time for the next output + output_this_iteration = true; + *this_iteration = cctk_iteration; + *next_output_iteration = cctk_iteration + myoutevery; + } + } + } else if (CCTK_EQUALS (myoutcriterion, "divisor")) { + int myoutevery = out_every == -2 ? io_out_every : out_every; + if (myGH->requests[vindex]->out_every >= 0) { + myoutevery = myGH->requests[vindex]->out_every; + } + if (myoutevery > 0 and (cctk_iteration % myoutevery) == 0) { + // we already decided to output this iteration + output_this_iteration = true; + } + } else if (CCTK_EQUALS (myoutcriterion, "time")) { + CCTK_REAL myoutdt = out_dt == -2 ? io_out_dt : out_dt; + if (myoutdt == 0 or *this_iteration == cctk_iteration) { + output_this_iteration = true; + } else if (myoutdt > 0 and (cctk_time / cctk_delta_time + >= *next_output_time / cctk_delta_time - 1.0e-12)) { + // it is time for the next output + output_this_iteration = true; + *this_iteration = cctk_iteration; + *next_output_time = cctk_time + myoutdt; + } + } + + if (not output_this_iteration) { + return 0; + } + + if (last_output.at(mglevel).at(reflevel).at(vindex) == cctk_iteration) { + // Has already been output during this iteration + char* varname = CCTK_FullName(vindex); + CCTK_VWarn (5, __LINE__, __FILE__, CCTK_THORNSTRING, + "Skipping output for variable \"%s\", because this variable " + "has already been output during the current iteration -- " + "probably via a trigger during the analysis stage", + varname); + free (varname); + return 0; + } + + assert (last_output.at(mglevel).at(reflevel).at(vindex) < cctk_iteration); + + // Should be output during this iteration + return 1; +} + + +static int OutputVar (const cGH* const cctkGH, const ioRequest* const request, + hid_t file) +{ + DECLARE_CCTK_PARAMETERS; + + const int group = CCTK_GroupIndexFromVarI (request->vindex); + assert (group >= 0); + cGroup groupdata; + CCTK_GroupData (group, &groupdata); + if (groupdata.grouptype == CCTK_SCALAR or groupdata.grouptype == CCTK_ARRAY) { + assert (do_global_mode); + } + + // check for storage + if (not CCTK_QueryGroupStorageI (cctkGH, group)) { + char* fullname = CCTK_FullName (request->vindex); + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot output variable '%s' because it has no storage", + fullname); + free (fullname); + return (0); + } + + if (out_unchunked) { + WriteVarUnchunked (cctkGH, file, request, false); + } else { + WriteVarChunkedSequential (cctkGH, file, request, false); + } + + last_output.at(mglevel).at(reflevel).at(request->vindex) = + cctkGH->cctk_iteration; + + return (0); +} + + +} // namespace CarpetIOStreamedHDF5 diff --git a/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.hh b/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.hh new file mode 100644 index 000000000..9f8038e2b --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/src/CarpetIOStreamedHDF5.hh @@ -0,0 +1,48 @@ +#ifndef CARPETIOSTREAMEDHDF5_HH +#define CARPETIOSTREAMEDHDF5_HH + +#include "CarpetIOHDF5.hh" +#include "SocketUtils.h" + + +// CarpetIOStreamed GH extension structure +typedef struct +{ + // port for clients to connect to + unsigned int port; + + // socket to stream data over a TCP connection + SOCKET socket; + + // default number of times to output + int out_every_default; + + // the last iteration output for each variable + vector out_last; + + // list of variables to output + char *out_vars; + + // stop on I/O parameter parsing errors ? + int stop_on_parse_errors; + + // I/O request description list (for all variables) + vector requests; + +} CarpetIOStreamedHDF5GH; + + +namespace CarpetIOStreamedHDF5 +{ + // scheduled routines (must be declared as C according to schedule.ccl) + extern "C" { + + void CarpetIOStreamedHDF5_Startup (void); + void CarpetIOStreamedHDF5_Init (const cGH* const); + void CarpetIOStreamedHDF5_Terminate (const cGH* const); + + } // extern "C" + +} // namespace CarpetIOStreamedHDF5 + +#endif // !defined(CARPETIOSTREAMEDHDF5_HH) diff --git a/Carpet/CarpetIOStreamedHDF5/src/make.code.defn b/Carpet/CarpetIOStreamedHDF5/src/make.code.defn new file mode 100644 index 000000000..11bf54377 --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/src/make.code.defn @@ -0,0 +1,4 @@ +# Main make.code.defn file for thorn CarpetIOStreamedHDF5 + +# Source files in this directory +SRCS = CarpetIOStreamedHDF5.cc diff --git a/Carpet/CarpetIOStreamedHDF5/src/make.configuration.defn b/Carpet/CarpetIOStreamedHDF5/src/make.configuration.defn new file mode 100644 index 000000000..68f89cb2c --- /dev/null +++ b/Carpet/CarpetIOStreamedHDF5/src/make.configuration.defn @@ -0,0 +1,6 @@ +# make.configuration.defn for IOStreamedHDF5 + +# make sure that the HDF5 Stream Virtual File Driver is available +ifeq ($(strip $(HAVE_HDF5_STREAM_VFD)),) + $(error 'CarpetIOStreamedHDF5 requires an HDF5 installation with built-in Stream Virtual File Driver. Please reconfigure with an appropriate HDF5 installation or remove CarpetIOStreamedHDF5 from ThornList !') +endif -- cgit v1.2.3