/*@@ @file Output.c @date 01 Oct 1999 @author Jonghyun Lee @desc Functions to deal with IOPanda output of grid variables @enddesc @version $Id$ @@*/ #include #include #include #include "cctk.h" #include "cctk_Parameters.h" #include "util_String.h" #include "util_Table.h" #include "CactusBase/IOUtil/src/ioGH.h" #include "CactusPUGH/PUGH/src/include/pugh.h" #include "ioPandaGH.h" #include "Panda/c_interface.h" /* the rcs ID and its dummy funtion to use it */ static const char *rcsid = "$Header$"; CCTK_FILEVERSION(CactusPUGHIO_IOPanda_Output_c) /******************************************************************** ******************** Internal Routines ************************ ********************************************************************/ static int DumpVar (const cGH *GH, const ioRequest *request, const char *alias); static void IEEEIOStructDump (const cGH *GH, const char *fname); static void AddCommonAttributes (const cGH *GH, const ioRequest *request, const char *fname); static void AddChunkAttributes (const cGH *GH, const ioRequest *request, const char *fname); /*@@ @routine IOPanda_OutputGH @date Sat March 6 1999 @author Gabrielle Allen @desc Loops over all variables and outputs them if necessary @enddesc @calls IOPanda_TimeFor DumpVar @var GH @vdesc pointer to CCTK GH @vtype const cGH * @vio in @endvar @returntype int @returndesc the number of variables which were output at this iteration (or 0 if it wasn't time to output yet) @endreturndesc @@*/ int IOPanda_OutputGH (const cGH *GH) { int vindex, retval; const pandaGH *myGH; retval = 0; myGH = (const pandaGH *) CCTK_GHExtension (GH, "IOPanda"); /* loop over all variables */ for (vindex = CCTK_NumVars () - 1; vindex >= 0; vindex--) { if (IOPanda_TimeFor (GH, vindex) && DumpVar (GH, myGH->requests[vindex], CCTK_VarName (vindex)) == 0) { /* register variable as having output this iteration */ myGH->out_last[vindex] = GH->cctk_iteration; retval++; } } return (retval); } /*@@ @routine IOPanda_OutputVarAs @date Sat March 6 1999 @author Gabrielle Allen @desc Unconditional output of a variable using the IOPanda I/O method @enddesc @calls IOUtil_ParseVarsForOutput DumpVar IOUtil_FreeIORequest @var GH @vdesc pointer to CCTK GH @vtype const cGH * @vio in @endvar @var fullname @vdesc complete name of variable to output @vtype const char * @vio in @endvar @var alias @vdesc alias name of variable to output (used to generate output filename) @vtype const char * @vio in @endvar @returntype int @returndesc return code of @seeroutine DumpVar @endreturndesc @@*/ int IOPanda_OutputVarAs (const cGH *GH, const char *fullname, const char *alias) { int vindex, oneshot, retval; const pandaGH *myGH; DECLARE_CCTK_PARAMETERS vindex = CCTK_VarIndex (fullname); if (CCTK_Equals (verbose, "full")) { CCTK_VInfo (CCTK_THORNSTRING, "IOPanda_OutputVarAs: (fullname, alias, " "index) = (%s, %s, %d)", fullname, alias, vindex); } /* check whether the variable already has an I/O request entry */ myGH = (const pandaGH *) CCTK_GHExtension (GH, "IOPanda"); oneshot = myGH->requests[vindex] == NULL; if (oneshot) { IOUtil_ParseVarsForOutput (GH, CCTK_THORNSTRING, "IOPanda::out_vars", 0, fullname, 1, -1.0, myGH->requests); } /* do the output */ retval = DumpVar (GH, myGH->requests[vindex], alias); if (oneshot && myGH->requests[vindex]) { IOUtil_FreeIORequest (&myGH->requests[vindex]); } return (retval); } /*@@ @routine IOPanda_TimeFor @date Sat March 6 1999 @author Gabrielle Allen @desc Decides if it is time to output a variable using the IOPanda output method @enddesc @calls IOPanda_CheckSteerableParameters @var GH @vdesc pointer to CCTK GH @vtype const cGH * @vio in @endvar @var vindex @vdesc index of variable @vtype int @vio in @endvar @returntype int @returndesc 1 if output should take place at this iteration, or
0 if not @endreturndesc @@*/ int IOPanda_TimeFor (const cGH *GH, int vindex) { int retval; char *fullname; const pandaGH *myGH; IOPanda_CheckSteerableParameters (GH); /* check if this variable should be output */ myGH = (const pandaGH *) CCTK_GHExtension (GH, "IOPanda"); retval = myGH->requests[vindex] && myGH->requests[vindex]->out_every > 0 && GH->cctk_iteration % myGH->requests[vindex]->out_every == 0; if (retval) { /* check if variable was not already output this iteration */ if (myGH->out_last[vindex] == GH->cctk_iteration) { fullname = CCTK_FullName (vindex); CCTK_VWarn (6, __LINE__, __FILE__, CCTK_THORNSTRING, "Already done IOPanda output for variable '%s' in current " "iteration (probably via triggers)", fullname); free (fullname); retval = 0; } } return (retval); } /*@@ @routine IOPanda_TriggerOutput @date Sat March 6 1999 @author Gabrielle Allen @desc Triggers the output a variable using the IOPanda output method @enddesc @calls DumpVar @var GH @vdesc pointer to CCTK GH @vtype const cGH * @vio in @endvar @var vindex @vdesc index of variable to output @vtype int @vio in @endvar @returntype int @returndesc return code of @seeroutine DumpVar @endreturndesc @@*/ int IOPanda_TriggerOutput (const cGH *GH, int vindex) { int retval; const pandaGH *myGH; const char *varname; DECLARE_CCTK_PARAMETERS varname = CCTK_VarName (vindex); if (CCTK_Equals (verbose, "full")) { CCTK_VInfo (CCTK_THORNSTRING, "TriggerOutput: " "name, index = %s, %d", varname, vindex); } /* do the output */ myGH = (const pandaGH *) CCTK_GHExtension (GH, "IOPanda"); retval = DumpVar (GH, myGH->requests[vindex], varname); if (retval == 0) { /* register variable as having output this iteration */ myGH->out_last[vindex] = GH->cctk_iteration; } return (retval); } /*@@ @routine IOPanda_CheckSteerableParameters @date Mon Oct 10 2000 @author Thomas Radke @desc Checks if IOPanda steerable parameters were changed and does appropriate re-evaluation. @enddesc @calls IOUtil_ParseVarsForOutput @var GH @vdesc pointer to CCTK GH @vtype const cGH * @vio in @endvar @@*/ void IOPanda_CheckSteerableParameters (const cGH *GH) { int i, vindex, num_vars; pandaGH *myGH; char *msg, *fullname; DECLARE_CCTK_PARAMETERS /* how often to output */ myGH = (pandaGH *) CCTK_GHExtension (GH, "IOPanda"); i = myGH->out_every_default; myGH->out_every_default = out_every >= 0 ? out_every : io_out_every; /* report if frequency changed */ if (myGH->out_every_default != i && ! CCTK_Equals (verbose, "none")) { if (myGH->out_every_default > 0) { CCTK_VInfo (CCTK_THORNSTRING, "Periodic IOPanda output every %d " "iterations", myGH->out_every_default); } else { CCTK_INFO ("Periodic IOPanda output turned off"); } } /* re-parse the 'IOPanda::out_vars' parameter if it was changed */ if (strcmp (out_vars, myGH->out_vars) || myGH->out_every_default != i) { IOUtil_ParseVarsForOutput (GH, CCTK_THORNSTRING, "IOPanda::out_vars", myGH->stop_on_parse_errors, out_vars, myGH->out_every_default, -1.0, myGH->requests); /*** FIXME: IEEEIO doesn't provide a COMPLEX datatype so we should map CCTK_COMPLEX to two REALs here. We have to check for this already here because if an IEEEIO file is created and nothing is written to it, it will crash at re-opening. ***/ num_vars = CCTK_NumVars (); for (vindex = 0; vindex < num_vars; vindex++) { if (! myGH->requests[vindex]) { continue; } if (strncmp (CCTK_VarTypeName (CCTK_VarTypeI (vindex)), "CCTK_VARIABLE_COMPLEX", 21) == 0) { fullname = CCTK_FullName (vindex); CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "IOPanda output for complex variable '%s' not yet " "supported", fullname); free (fullname); IOUtil_FreeIORequest (&myGH->requests[vindex]); } else if (CCTK_GroupTypeFromVarI (vindex) == CCTK_SCALAR || CCTK_GroupDimFromVarI (vindex) != 3) { fullname = CCTK_FullName (vindex); CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "IOPanda output for variable '%s' not supported (can only " "do 3D grid arrays)", fullname); free (fullname); IOUtil_FreeIORequest (&myGH->requests[vindex]); } } if (myGH->out_every_default == i || ! CCTK_Equals (verbose, "none")) { msg = NULL; for (vindex = 0; vindex < num_vars; vindex++) { if (myGH->requests[vindex]) { fullname = CCTK_FullName (vindex); if (! msg) { Util_asprintf (&msg, "Periodic IOPanda output requested for '%s'", fullname); } else { char *tmp = msg; Util_asprintf (&msg, "%s, '%s'", msg, fullname); free (tmp); } free (fullname); } } if (msg) { CCTK_INFO (msg); free (msg); } } /* save the last setting of 'IOPanda::out_vars' parameter */ free (myGH->out_vars); myGH->out_vars = strdup (out_vars); } } /******************************************************************** ******************** Internal Routines ************************ ********************************************************************/ static int DumpVar (const cGH *GH, const ioRequest *request, const char *alias) { void *hdata; int tmp[1], tmp2[3]; Distribution dist1[3], dist2[3]; int i, hdatasize, mapping, retval; ArrayInfo ainfo; const ioGH *ioUtilGH; const pGH *pughGH; char *fullname; union { char *non_const_char; const char *const_char; } cast_to_const; DECLARE_CCTK_PARAMETERS /* check if variable has storage assigned */ if (! CCTK_QueryGroupStorageI (GH, CCTK_GroupIndexFromVarI (request->vindex))) { fullname = CCTK_FullName (request->vindex); CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, "No IOPanda output for '%s' (no storage)", fullname); free (fullname); return (-1); } /* define the hyperslab mapping */ mapping = Hyperslab_LocalMappingByIndex (GH, request->vindex, request->hdim, request->direction, request->origin, request->extent, request->downsample, -1, NULL, request->hsize_chunk, request->hsize, request->hoffset); if (mapping < 0) { fullname = CCTK_FullName (request->vindex); CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "Failed to define hyperslab mapping for variable '%s'", fullname); free (fullname); return (-1); } /* calculate the size of the hyperslab */ request->hsize_chunk[request->hdim] = 1; for (i = 0; i < request->hdim; i++) { request->hsize_chunk[request->hdim] *= request->hsize_chunk[i]; } /* get the hyperslab */ hdatasize = CCTK_VarTypeSize (request->hdatatype); hdata = request->hsize_chunk[request->hdim] > 0 ? malloc (request->hsize_chunk[request->hdim] * hdatasize) : NULL; retval = Hyperslab_Get (GH, mapping, -1, request->vindex, request->timelevel, request->hdatatype, hdata); /* release the mapping structure */ Hyperslab_FreeMapping (mapping); if (retval) { fullname = CCTK_FullName (request->vindex); CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "Failed to extract hyperslab for variable '%s'", fullname); free (fullname); if (hdata) { free (hdata); } return (-2); } /* prevent compiler warning "cast discards `const' from pointer target type"*/ cast_to_const.const_char = alias; ainfo.name_ = cast_to_const.non_const_char; /* copy from CCTK_INT[] to int[] */ ainfo.size_ = malloc (request->hdim * sizeof (int)); for (ainfo.rank_ = 0; ainfo.rank_ < request->hdim; ainfo.rank_++) { ainfo.size_[ainfo.rank_] = request->hsize[i]; } switch (request->hdatatype) { case CCTK_VARIABLE_BYTE: ainfo.esize_ = UCHAR; break; case CCTK_VARIABLE_INT: #ifdef CCTK_INTEGER_PRECISION_8 ainfo.esize_ = INT64; break; #elif CCTK_INTEGER_PRECISION_4 ainfo.esize_ = INT32; break; #elif CCTK_INTEGER_PRECISION_2 ainfo.esize_ = INT16; break; #elif CCTK_INTEGER_PRECISION_1 ainfo.esize_ = INT8; break; #endif case CCTK_VARIABLE_REAL: #ifdef CCTK_REAL_PRECISION_4 ainfo.esize_ = FLOAT32; break; #elif CCTK_REAL_PRECISION_8 ainfo.esize_ = FLOAT64; break; #endif #ifdef CCTK_INT1 case CCTK_VARIABLE_INT1: ainfo.esize_ = INT8; break; #endif #ifdef CCTK_INT2 case CCTK_VARIABLE_INT2: ainfo.esize_ = INT16; break; #endif #ifdef CCTK_INT4 case CCTK_VARIABLE_INT4: ainfo.esize_ = INT32; break; #endif #ifdef CCTK_INT8 case CCTK_VARIABLE_INT8: ainfo.esize_ = INT64; break; #endif #ifdef CCTK_REAL4 case CCTK_VARIABLE_REAL4: ainfo.esize_ = FLOAT32; break; #endif #ifdef CCTK_REAL8 case CCTK_VARIABLE_REAL8: ainfo.esize_ = FLOAT64; break; #endif default: CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, "Unsupported " "CCTK variable datatype %d", request->hdatatype); break; } /* FIXME: rank */ ainfo.mem_rank_ = 3; pughGH = PUGH_pGH (GH); tmp2[0] = pughGH->Connectivity[2]->nprocs[2]; tmp2[1] = pughGH->Connectivity[2]->nprocs[1]; tmp2[2] = pughGH->Connectivity[2]->nprocs[0]; ainfo.mem_layout_ = tmp2; dist1[0] = dist1[1] = dist1[2] = BLOCK; ainfo.mem_dist_ = dist1; ainfo.disk_rank_ = 1; dist2[0] = BLOCK; dist2[1] = dist2[2] = NONE; ioUtilGH = (const ioGH *) CCTK_GHExtension (GH, "IO"); tmp[0]= ((CCTK_nProcs(GH) - 1) / ioUtilGH->ioproc_every + 1); ainfo.disk_layout_ = tmp; ainfo.disk_dist_ = dist2; /*** FIXME: asymmetric ghostzones */ ainfo.stencil_width_ = GH->cctk_nghostzones[0]; ainfo.data_ = (char *) hdata; PandaTimestep(&ainfo); if (hdata) { free (hdata); } AddCommonAttributes(GH, request, ainfo.name_); if (! ioUtilGH->unchunked) { /* write chunk attributes */ AddChunkAttributes (GH, request, ainfo.name_); } if (PandaIsNewFile(ainfo.name_)) { IEEEIOStructDump(GH, ainfo.name_); } return (retval); } static void AddCommonAttributes (const cGH *GH, const ioRequest *request, const char *fname) { int hdim, vdim, coord_system_handle; char *name; CCTK_INT4 *itmp; CCTK_REAL *dtmp; CCTK_INT *coord_handles; const ioGH *ioUtilGH; ioUtilGH = CCTK_GHExtension (GH, "IO"); itmp = malloc (request->hdim * sizeof (CCTK_INT4)); name = CCTK_FullName (request->vindex); Panda_WriteAttribute (fname, "name", BYTE, strlen (name) + 1, name); free (name); name = CCTK_GroupNameFromVarI (request->vindex); Panda_WriteAttribute (fname, "groupname", BYTE, strlen (name) + 1, name); coord_system_handle = -1; if (CCTK_IsFunctionAliased ("Coord_GroupSystem")) { coord_system_handle = Coord_GroupSystem (GH, name); } free (name); itmp[0] = CCTK_GroupTypeFromVarI (request->vindex); Panda_WriteAttribute (fname, "grouptype", INT32, 1, itmp); itmp[0] = CCTK_MaxTimeLevelsVI (request->vindex); Panda_WriteAttribute (fname, "ntimelevels", INT32, 1, itmp); itmp[0] = request->timelevel; Panda_WriteAttribute (fname, "timelevel", INT32, 1, itmp); Panda_WriteAttribute (fname, "time", FLOAT64, 1, &GH->cctk_time); /* write bbox attributes if we have coordinate system info */ coord_handles = malloc (request->vdim * sizeof (CCTK_INT)); if (coord_system_handle >= 0 && Util_TableGetIntArray (coord_system_handle, request->vdim, coord_handles, "COORDINATES") >= 0) { dtmp = calloc (3 * request->vdim, sizeof (CCTK_REAL)); for (vdim = 0; vdim < request->vdim; vdim++) { for (hdim = 0; hdim < request->hdim; hdim++) { if (request->direction[hdim*request->hdim + vdim]) { Util_TableGetReal (coord_handles[vdim], &dtmp[hdim + 0*request->vdim], "COMPMIN"); if (Util_TableGetReal (coord_handles[vdim], &dtmp[hdim+2*request->vdim], "DELTA") >= 0) { dtmp[hdim+2*request->vdim] *= request->downsample[hdim]; dtmp[hdim+1*request->vdim] = dtmp[hdim+0*request->vdim]; dtmp[hdim+1*request->vdim] += ((request->extent[hdim] + request->downsample[hdim]-1) / request->downsample[hdim] - 1) * dtmp[hdim+2*request->vdim]; } } } } Panda_WriteAttribute (fname, "origin", FLOAT64, request->hdim, dtmp + 0*request->vdim); Panda_WriteAttribute (fname, "min_ext", FLOAT64, request->hdim, dtmp + 0*request->vdim); Panda_WriteAttribute (fname, "max_ext", FLOAT64, request->hdim, dtmp + 1*request->vdim); Panda_WriteAttribute (fname, "delta", FLOAT64, request->hdim, dtmp + 2*request->vdim); #if 0 if (ioUtilGH->downsample[0] > 1 || ioUtilGH->downsample[1] > 1 || ioUtilGH->downsample[2] > 1) { Panda_WriteAttribute (fname, "evolution_delta", FLOAT64, request->hdim, GH->cctk_delta_space); } #endif free (dtmp); } free (coord_handles); for (hdim = 0; hdim < request->hdim; hdim++) { itmp[hdim] = request->hsize[hdim]; } Panda_WriteAttribute (fname, "global_size", INT32, 3, itmp); itmp[0] = CCTK_nProcs (GH); Panda_WriteAttribute (fname, "nprocs", INT32, 1, itmp); itmp[0] = ioUtilGH->ioproc_every; Panda_WriteAttribute (fname, "ioproc_every", INT32, 1, itmp); itmp[0] = ioUtilGH->unchunked; Panda_WriteAttribute (fname, "unchunked", INT32, 1, itmp); itmp[0] = GH->cctk_iteration; Panda_WriteAttribute (fname, "iteration", INT32, 1, itmp); free (itmp); } static void AddChunkAttributes (const cGH *GH, const ioRequest *request, const char *fname) { int i; char *fullname; CCTK_INT4 *itmp; /* there is nothing to do for a serial run */ if (CCTK_nProcs (GH) == 1) { return; } itmp = malloc (2 * request->hdim * sizeof (CCTK_INT4)); /* copy from CCTK_INT[] to CCTK_INT4[] */ for (i = 0; i < request->hdim; i++) { itmp[0*request->hdim + i] = request->hoffset[i]; itmp[1*request->hdim + i] = request->hsize[i]; } Panda_WriteAttribute (fname, "chunk_origin", INT32, request->hdim, &itmp[0]); Panda_WriteAttribute (fname, "subchunk_lb", INT32, request->hdim, &itmp[0]); Panda_WriteAttribute (fname, "global_size", INT32, request->hdim, &itmp[request->hdim]); itmp[0] = GH->cctk_iteration; Panda_WriteAttribute (fname, "chunk_dataset", INT32, 1, itmp); fullname = CCTK_FullName (request->vindex); Panda_WriteAttribute (fname, "name", CHAR, strlen (fullname)+1, fullname); free (fullname); free (itmp); } static void IEEEIOStructDump (const cGH *GH, const char *fname) { int len; CCTK_INT4 itmp; CCTK_REAL dtmp; const ioGH *ioUtilGH; char buffer[128]; DECLARE_CCTK_PARAMETERS itmp = GH->cctk_iteration; Panda_WriteAttribute (fname, "GH$iteration", INT32, 1, &itmp); ioUtilGH = (const ioGH *) CCTK_GHExtension (GH, "IO"); itmp = ioUtilGH->ioproc_every; Panda_WriteAttribute (fname, "GH$ioproc_every", INT32, 1, &itmp); itmp = CCTK_nProcs (GH); Panda_WriteAttribute (fname, "GH$nprocs", INT32, 1, &itmp); dtmp = GH->cctk_time; Panda_WriteAttribute (fname, "GH$time", FLOAT64, 1, &dtmp); /* add the parameter filename and the creation date as file identification attributes */ if (CCTK_Equals (out_fileinfo, "parameter filename") || CCTK_Equals (out_fileinfo, "all")) { buffer[0] = 0; CCTK_ParameterFilename (sizeof (buffer), buffer); Panda_WriteAttribute (fname, "parameter file", CHAR, strlen (buffer) + 1, buffer); } if (CCTK_Equals (out_fileinfo, "creation date") || CCTK_Equals (out_fileinfo, "all")) { buffer[0] = 0; Util_CurrentDate (sizeof (buffer), buffer); len = strlen (buffer) + 1; buffer[len-1] = ' '; Util_CurrentTime (sizeof (buffer) - len, buffer + len); Panda_WriteAttribute (fname, "creation date", CHAR, strlen (buffer) + 1, buffer); } }