diff options
author | hawke <> | 2003-05-16 12:02:00 +0000 |
---|---|---|
committer | hawke <> | 2003-05-16 12:02:00 +0000 |
commit | 2e93ec24569402ca647b154f23c5e7e2dc91c857 (patch) | |
tree | 55d22ce13cb60b4cd034b3bf9696794c76ee1d55 /CarpetAttic | |
parent | 735acd6be1c9e457d0882b9bb70901214a4fa6e4 (diff) |
Initial revision
darcs-hash:20030516120218-58737-a67388e6af93425eb1ad092777a1eb64dab69a52.gz
Diffstat (limited to 'CarpetAttic')
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/README | 4 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl | 20 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl | 24 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl | 16 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc | 797 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc | 1018 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h | 15 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh | 98 | ||||
-rw-r--r-- | CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn | 4 |
9 files changed, 460 insertions, 1536 deletions
diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/README b/CarpetAttic/CarpetIOFlexIOCheckpoint/README index d9385ccff..f45645401 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/README +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/README @@ -1,6 +1,6 @@ Cactus Code Thorn CarpetIOFlexIO -Authors : Erik Schnetter <schnetter@uni-tuebingen.de>, Christian Ott <cott@aei.mpg.de> -CVS info : $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/README,v 1.2 2003/07/14 15:41:08 schnetter Exp $ +Authors : Erik Schnetter <schnetter@uni-tuebingen.de> +CVS info : $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/README,v 1.1 2003/05/16 14:02:18 hawke Exp $ -------------------------------------------------------------------------- Purpose of the thorn: diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl b/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl index 94d56dceb..6944ca62a 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl @@ -1,22 +1,8 @@ # Interface definition for thorn CarpetIOFlexIO -# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl,v 1.6 2003/09/17 13:47:00 cvs_anon Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/interface.ccl,v 1.1 2003/05/16 14:02:18 hawke Exp $ implements: IOFlexIO -uses include header: carpet.hh - -uses include header: bbox.hh -uses include header: vect.hh - -uses include header: data.hh -uses include header: gdata.hh +inherits: CarpetLib driver Hyperslab -uses include header: ggf.hh - -# FlexIO header files -uses include header: IO.hh -uses include header: IEEEIO.hh -uses include header: HDFIO.hh -uses include header: H5IO.hh -uses include header: AMRwriter.hh -uses include header: AmrGridReader.hh +uses include header: carpet.hh diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl b/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl index 079ebb72c..eba189d5c 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl @@ -1,19 +1,24 @@ # Parameter definitions for thorn CarpetIOFlexIO -# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl,v 1.3 2003/10/02 11:34:03 cvs_anon Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/param.ccl,v 1.1 2003/05/16 14:02:18 hawke Exp $ -#shares: IO -#shares: Carpet +shares: IO +USES STRING out_dir +USES CCTK_INT out_every -private: -BOOLEAN verbose "verbose info output" -{ -} "no" +restricted: + + + +#BOOLEAN verbose "Produce log output" +#{ +#} "no" + CCTK_STRING out3D_dir "Name of 3D FlexIO output directory, overrides out_dir" STEERABLE = ALWAYS @@ -85,6 +90,7 @@ CCTK_STRING in3D_extension "File extension to use for 3D FlexIO input" STEERABLE .* :: "A regex which matches everything" } "" + BOOLEAN checkpoint "Do checkpointing with IOFlexIO" STEERABLE = ALWAYS { } "no" @@ -102,7 +108,7 @@ USES BOOLEAN recover_and_remove USES BOOLEAN checkpoint_on_terminate USES KEYWORD recover USES STRING recover_file -USES STRING out_dir -USES CCTK_INT out_every +USES KEYWORD verbose + diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl b/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl index bfc9fd9b0..6ff43773e 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl @@ -1,27 +1,19 @@ # Schedule definitions for thorn CarpetIOFlexIO -# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl,v 1.3 2003/12/01 13:15:52 cott Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/schedule.ccl,v 1.1 2003/05/16 14:02:18 hawke Exp $ -schedule CarpetIOFlexIO_Startup at STARTUP after IOUtil_Startup +schedule CarpetIOFlexIOStartup at STARTUP after IOUtil_Startup { LANG: C } "Startup routine" -schedule CarpetIOFlexIO_ReadData at INITIAL +schedule CarpetIOFlexIOReadData at INITIAL { LANG: C OPTIONS: global } "Read initial data from file" -schedule CarpetIOFlexIO_EvolutionCheckpoint at CHECKPOINT +schedule CarpetChReEvolutionCheckpoint at CHECKPOINT { LANG: C OPTIONS: global } "Do checkpointing" - -if (! CCTK_Equals (recover, "no") && *recover_file) -{ - schedule CarpetIOFlexIO_RecoverParameters at RECOVER_PARAMETERS - { - LANG:C - } "Parameter recovery routine" -} diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc index 392c8619d..97552a6e6 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc @@ -13,56 +13,54 @@ -#include "cctk.h" -#include "cctk_Parameters.h" -#include "cctk_Version.h" - -#include "AMRwriter.hh" -#include "AmrGridReader.hh" -#ifdef HDF4 -# include "HDFIO.hh" -#endif -#ifdef HDF5 -# include "H5IO.hh" -#endif -#include "IEEEIO.hh" -#include "IO.hh" +#include <AMRwriter.hh> +#include <AmrGridReader.hh> +#include <H5IO.hh> +#include <HDFIO.hh> +#include <IEEEIO.hh> +#include <IO.hh> // Hack to stop FlexIO type clash + #undef BYTE #undef CHAR -#include "bbox.hh" -#include "data.hh" -#include "gdata.hh" -#include "ggf.hh" -#include "vect.hh" +#include "cctk.h" +#include "cctk_Parameters.h" + +#include "CactusBase/IOUtil/src/ioGH.h" + +#include "Carpet/CarpetLib/src/bbox.hh" +#include "Carpet/CarpetLib/src/data.hh" +#include "Carpet/CarpetLib/src/gdata.hh" +#include "Carpet/CarpetLib/src/ggf.hh" +#include "Carpet/CarpetLib/src/vect.hh" +#include "CactusBase/IOUtil/src/ioGH.h" +#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h" +#include "CactusBase/IOUtil/src/ioutil_Utils.h" +//#include "StoreNamedData.h" #include "carpet.hh" #include "ioflexio.hh" extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc,v 1.22 2004/01/12 10:50:31 cott Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc,v 1.1 2003/05/16 14:02:18 hawke Exp $"; CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_checkpointrestart_cc); } - namespace CarpetCheckpointRestart { using namespace std; using namespace Carpet; using namespace CarpetIOFlexIO; - using namespace CarpetIOFlexIOUtil; - int Checkpoint (const cGH* const cgh, int called_from); + static int Checkpoint (const cGH* const cgh, int called_from); + - int RecoverParameters (IObase* reader); - static int RecoverGHextensions (cGH* cgh, IObase* reader); - int RecoverVariables (cGH* cgh, IObase* reader, AmrGridReader* amrreader); - void CarpetIOFlexIO_EvolutionCheckpoint( const cGH* const cgh){ + void CarpetChReEvolutionCheckpoint( const cGH* const cgh){ DECLARE_CCTK_PARAMETERS @@ -70,14 +68,13 @@ namespace CarpetCheckpointRestart { ((checkpoint_every > 0 && cgh->cctk_iteration % checkpoint_every == 0) || checkpoint_next)) { - if (verbose) + if (! CCTK_Equals ("verbose", "none")) { CCTK_INFO ("---------------------------------------------------------"); CCTK_VInfo (CCTK_THORNSTRING, "Dumping periodic checkpoint at " "iteration %d", cgh->cctk_iteration); CCTK_INFO ("---------------------------------------------------------"); } - Checkpoint (cgh, CP_EVOLUTION_DATA); if (checkpoint_next) @@ -89,461 +86,41 @@ namespace CarpetCheckpointRestart { } -/*@@ - @routine CarpetIOFlexIO_RecoverParameters - @date Fri Oct 10 2003 - @author Christian Ott, Thomas Radke - @desc - @desc - Recovers the parameters from an HDF5 checkpoint file. - This routine is scheduled at CCTK_RECOVER_PARAMETERS. - - Note that it cannot be registered with IOUtil to be scheduled - from there (as done with the CarpetIOFlexIO_Recover routine) because - the registration mechanism isn't activated yet - at CCTK_RECOVER_PARAMETERS. - Instead we call the generic parameter recovery routine - from IOUtil here, and just pass the necessary callback function - and its arguments. - - Note also that this routine doesn't get passed any parameters, - not even a GH, because this doesn't exist yet at the time it is - being called. - @enddesc - - @calls IOUtil_RecoverParameters - - @returntype int - @returndesc - return code of @seeroutine IOUtil_RecoverParameters, ie. - positive for successful parameter recovery, or<BR> - 0 if recovery wasn't requested, or<BR> - negative if parameter recovery failed - @endreturndesc -@@*/ - - - int CarpetIOFlexIO_RecoverParameters(void){ - return (IOUtil_RecoverParameters (CarpetIOFlexIO_Recover, ".hdf5", "HDF5")); - } - -/*@@ - @routine CarpetIOFlexIO_Recover - @date Fri Oct 10 2003 - @author Christian Ott (Tom Goodale IOFlexIO version) - @desc - Recovers a Carpet GH from an HDF5 file. - This routine is registered with IOUtil as CarpetIOFlexIO's recovery - routine. - @enddesc - - @calls OpenFile - RecoverParameters - RecoverGHextensions - CarpetIOFlexIOi_RecoverVariables - IOUtil_PrintTimings - - @var GH - @vdesc Pointer to CCTK grid hierarchy - @vtype cGH * - @vio in - @endvar - @var basefilename - @vdesc the basefilename of the file to recover from - The file suffix is appended by the routine. - @vtype const char * - @vio in - @endvar - @var called_from - @vdesc flag indicating where this routine was called from - (either CP_RECOVER_DATA or FILEREADER_DATA) - @vtype int - @vio in - @endvar - - @returntype int - @returndesc - >0 = success - -1 = recovery failed - @endreturndesc -@@*/ - - -int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from) -{ - int result,myproc; - CarpetIOFlexIOGH *myGH; - char filename[1024]; - - static IObase* reader = NULL; - - DECLARE_CCTK_PARAMETERS - - /* to make the compiler happy */ - myGH = NULL; - result = 0; - - myproc = CCTK_MyProc (cgh); - - fprintf(stderr,"\n reflevel: %d\n",reflevel); - - - if (called_from == CP_RECOVER_PARAMETERS) - { - CCTK_VInfo (CCTK_THORNSTRING, "got this far... '%s'", basefilename); - if (myproc == 0){ - reader = new H5IO(basefilename,IObase::Read); - CCTK_VInfo (CCTK_THORNSTRING, "blah '%s'", basefilename); - if ( ! reader->isValid() ) - { - CCTK_VInfo(CCTK_THORNSTRING,"file is not open"); - return (-1); - } - CCTK_VInfo(CCTK_THORNSTRING,"file is open"); - } - } - else - { - /* This is the case for CP_RECOVER_DATA. - CCTK_RECOVER_PARAMETERS must have been called before - and set up the file info structure. */ - if (myproc == 0){ - if (! reader->isValid() ) - { - CCTK_VInfo(CCTK_THORNSTRING,"file is not open2"); - return (-1); - } - } - } - - /* Recover parameters */ - - if (called_from == CP_RECOVER_PARAMETERS) - { - return (RecoverParameters (reader)); - } - - - if (called_from == CP_RECOVER_DATA) { - - AmrGridReader* amrreader = 0; - - if(myproc==0) - amrreader = new AmrGridReader(*reader); - - - CCTK_INT4 numberoftimes; - IObase::DataType datatype; - int i,dim; - if (myproc == 0) { - - /* we need all the times on the individual levels */ - i = reader->readAttributeInfo ("numberoftimes", datatype, dim); - if(i >=0 && datatype == FLEXIO_INT && dim == 1) { - char buffer[100]; - reader->readAttribute (i, &numberoftimes); - assert(numberoftimes==refleveltimes.size()); - for(int lcv=0;lcv<numberoftimes;lcv++) { - sprintf(buffer,"refleveltime%d",lcv); - i = reader->readAttributeInfo (buffer, datatype, dim); - if(i >=0 && datatype == FLEXIO_REAL && dim == 1) { - reader->readAttribute (i, &refleveltimes[lcv]); - } - else { - CCTK_WARN(0,"BAD BAD BAD! Can't read refleveltime!!"); - } - } - } - else - { - CCTK_WARN (0, "Unable to restore reflevel times!"); - } - } - - CACTUS_MPI_ERROR (MPI_Bcast (&numberoftimes, 1, CARPET_MPI_INT4, 0,MPI_COMM_WORLD)); - CACTUS_MPI_ERROR (MPI_Bcast (&refleveltimes[0], numberoftimes, CARPET_MPI_REAL, 0, MPI_COMM_WORLD)); - - - - - BEGIN_REFLEVEL_LOOP(cgh) { - BEGIN_MGLEVEL_LOOP(cgh) { - - - /* make sure we are looking at the first dataset where - all the good stuff ist stored! */ - if(myproc==0) - reader->seek(0); - - /* Recover GH extentions */ - CCTK_INFO ("Recovering GH extensions"); - result += RecoverGHextensions (cgh, reader); - - // cout << refleveltimes[reflevel]<<endl; - tt->set_time(reflevel,mglevel,(CCTK_REAL) cgh->cctk_iteration/maxreflevelfact); - cout << "tt " << tt->time(0,reflevel,mglevel) << endl; - - } END_MGLEVEL_LOOP; - } END_REFLEVEL_LOOP; - - if (! result) - { - /* Recover variables */ - CCTK_VInfo (CCTK_THORNSTRING, "Recovering data! "); - result = RecoverVariables (cgh, reader,amrreader); - } - if(myproc==0) { - delete reader; - delete amrreader; - } - - - } - - - - if (called_from == CP_RECOVER_DATA) - { - CCTK_VInfo (CCTK_THORNSTRING, - "Restarting simulation at iteration %d (physical time %g)", - cgh->cctk_iteration, (double) cgh->cctk_time); - } - - // CCTK_WARN (-1,"STOPSTOPSTOP2"); - - return (result); -} - - /******************************************************************** ******************** Internal Routines ************************ ********************************************************************/ - int RecoverParameters(IObase* reader){ - - int myproc, retval; - int i, asize; - char *parameters; - CCTK_INT4 parameterSize; - - IObase::DataType datatype; - CCTK_REAL bogusdata; - - int dims[3]; - int rank=0; - int maxdims=3; - - DECLARE_CCTK_PARAMETERS - - myproc = CCTK_MyProc (NULL); - - if (myproc == 0){ - CCTK_VInfo (CCTK_THORNSTRING, "Recovering parameters from checkpoint "); - - /* read the first (bogus) dataset to which the parameters an GHExtensions - are attached */ - - reader->readInfo(datatype,rank,dims,maxdims); - CCTK_VInfo (CCTK_THORNSTRING, "blahbalh: datatype %d FLEXIO_REAL: %d",datatype,FLEXIO_REAL); - - if(datatype != FLEXIO_REAL || rank !=1 ) - CCTK_WARN (-1,"Wrong recover file format! First dataset type mismatch!"); - - reader->read(&bogusdata); - - /* get the parameters attribute. */ - i = reader->readAttributeInfo ("global_parameters", datatype, asize); - if (i >= 0 && datatype == FLEXIO_CHAR && asize > 0) - { - parameterSize = (CCTK_INT4) asize; - parameters = (char*) malloc (parameterSize + 1); - reader->readAttribute (i, parameters); - } - else - { - CCTK_WARN (1, "Can't read global parameters. " - "Is this really a Cactus IEEEIO checkpoint file ?"); - } - CCTK_VInfo (CCTK_THORNSTRING, "\n%s\n",parameters); - } - -#ifdef CCTK_MPI - /* Broadcast the parameter buffer size to all processors */ - /* NOTE: We have to use MPI_COMM_WORLD here - because CARPET_COMM_WORLD is not yet set up at parameter recovery time. - We also assume that CARPET_MPI_INT4 is a compile-time defined datatype. */ - CACTUS_MPI_ERROR (MPI_Bcast (¶meterSize, 1, CARPET_MPI_INT4, 0, - MPI_COMM_WORLD)); -#endif - - if (parameterSize > 0) - { -#ifdef CCTK_MPI - if (myproc) - { - parameters = (char*) malloc (parameterSize + 1); - } - - CACTUS_MPI_ERROR (MPI_Bcast (parameters, parameterSize + 1, CARPET_MPI_CHAR, - 0, MPI_COMM_WORLD)); -#endif - - IOUtil_SetAllParameters (parameters); - - free (parameters); - } - - /* return positive value for success otherwise negative */ - retval = (parameterSize > 0 ? 1 : -1); - return (retval); - - - // CCTK_WARN (-1,"STOPSTOPSTOP"); - - - } - - - static int RecoverGHextensions (cGH *GH, IObase* reader) - { - int i, type,dim; - CCTK_REAL realBuffer; - CCTK_INT4 int4Buffer[2]; - CCTK_INT4 intbuffer,numberoftimes; - - IObase::DataType datatype; - - if (CCTK_MyProc (GH) == 0) - { - - /* get the iteration number */ - i = reader->readAttributeInfo ("GH$iteration", datatype, dim); - - - if (i >= 0 && datatype == FLEXIO_INT4 && dim == 1) - { - reader->readAttribute (i, &int4Buffer[0]); - } - else - { - CCTK_WARN (1, "Unable to restore GH->cctk_iteration, defaulting to 0"); - int4Buffer[0] = 0; - } - - /* get the main loop index */ - i = reader->readAttributeInfo ( "main loop index", datatype, dim); - if (i >= 0 && datatype == FLEXIO_INT4 && dim == 1) - { - reader->readAttribute (i, &int4Buffer[1]); - } - else - { - CCTK_WARN (1, "Unable to restore main loop index, defaulting to 0"); - int4Buffer[1] = 0; - } - - /* get cctk_time */ - i = reader->readAttributeInfo ("GH$time", datatype, dim); - if (i >= 0 && datatype == FLEXIO_REAL && dim == 1) - { - reader->readAttribute (i, &realBuffer); - } - else - { - CCTK_WARN (1, "Unable to restore GH->cctk_time, defaulting to 0.0"); - realBuffer = 0.0; - } - } - -#ifdef CCTK_MPI - /* Broadcast the GH extensions to all processors */ - /* NOTE: We have to use MPI_COMM_WORLD here - because PUGH_COMM_WORLD is not yet set up at parameter recovery time. - We also assume that PUGH_MPI_INT4 is a compile-time defined datatype. */ - - CACTUS_MPI_ERROR (MPI_Bcast (int4Buffer, 2, CARPET_MPI_INT4, 0,MPI_COMM_WORLD)); - CACTUS_MPI_ERROR (MPI_Bcast (int4Buffer, 2, CARPET_MPI_INT4, 0,MPI_COMM_WORLD)); - CACTUS_MPI_ERROR (MPI_Bcast (&realBuffer, 1, CARPET_MPI_REAL,0,MPI_COMM_WORLD)); -#endif - - GH->cctk_time = refleveltimes[reflevel]; - - - GH->cctk_iteration = (int) int4Buffer[0]; - CCTK_SetMainLoopIndex ((int) int4Buffer[1]); - - - return (0); - } - - - - - int DumpParams (const cGH* const cgh, int all, IObase* writer){ + static int DumpParams (const cGH* const cgh, int all, IObase* writer){ char *parameters; parameters = IOUtil_GetAllParameters(cgh,all); - + if(parameters) { - WriteAttribute(writer,"global_parameters",parameters); + writer->writeAttribute("global_parameters",IObase::Char, + strlen(parameters)+1,parameters); free(parameters); } - return 0; } - int DumpGHExtensions (const cGH* const cgh, IObase* writer){ - - CCTK_INT4 itmp; - CCTK_REAL dtmp; - const char *version; - ioGH *ioUtilGH; - - - /* get the handle for IOUtil extensions */ - ioUtilGH = (ioGH *) CCTK_GHExtension (cgh, "IO"); - - itmp = CCTK_MainLoopIndex (); - WriteAttribute(writer,"main loop index",itmp); - itmp = cgh->cctk_iteration; - WriteAttribute(writer,"GH$iteration",itmp); - - itmp = ioUtilGH->ioproc_every; - WriteAttribute(writer,"GH$ioproc_every",itmp); - - itmp = CCTK_nProcs (cgh); - WriteAttribute(writer,"GH$nprocs",itmp); - - dtmp = cgh->cctk_time; - WriteAttribute(writer,"GH$time", dtmp); - - version = CCTK_FullVersion (); - WriteAttribute(writer,"Cactus version", version); - - return 0; - } - - - int Checkpoint (const cGH* const cgh, int called_from) + static int Checkpoint (const cGH* const cgh, int called_from) { - char cp_filename[1024], cp_tempname[1024]; - int myproc, first_vindex, gindex; + char filename[1024]; + int myproc, first_vindex, gindex, retval; char *fullname; const char *timer_descriptions[3] = {"Time to dump parameters: ", "Time to dump datasets: ", "Total time to checkpoint:"}; const ioGH *ioUtilGH; - // const int varindex = CCTK_VarIndex("ADMBASE:gxx"); int varindex = 0; int group = 0; - int retval = 0; cGroup gdata; IObase* writer = 0; @@ -551,241 +128,103 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from) ioRequest *request; DECLARE_CCTK_PARAMETERS - - CarpetIOFlexIOGH *myGH; - myGH = (CarpetIOFlexIOGH *) CCTK_GHExtension (cgh, "CarpetIOFlexIO"); - - /* check if CarpetIOFlexIO was registered as I/O method */ - if (myGH == NULL) - { - CCTK_WARN (-1, "No CarpetIOFlexIO I/O methods registered"); - return (-1); - } - - + myproc = CCTK_MyProc (cgh); ioUtilGH = (const ioGH *) CCTK_GHExtension (cgh, "IO"); - IOUtil_PrepareFilename (cgh, NULL, cp_filename, called_from, + IOUtil_PrepareFilename (cgh, NULL, filename, called_from, myproc / ioUtilGH->ioproc_every, ioUtilGH->unchunked); + + sprintf(filename, "%s.ieee",filename); - // Invent a file name - const char* extension = 0; - if (CCTK_Equals(out3D_format, "IEEE")) { - extension = ".raw"; -#ifdef HDF4 - } else if (CCTK_Equals(out3D_format, "HDF4")) { - extension = ".hdf"; -#endif -#ifdef HDF5 - } else if (CCTK_Equals(out3D_format, "HDF5")) { - extension = ".h5"; -#endif - } else { - assert (0); - } - - sprintf(cp_tempname,"%s.tmp%s",cp_filename,extension); - - - sprintf(cp_filename,"%s%s",cp_filename,extension); - + fprintf(stderr,"%s\n",filename); if (CCTK_MyProc(cgh)==0) { - fprintf(stderr,"\n verbose: %d\n",verbose); - if (verbose) + if (CCTK_Equals ("verbose", "full")) { - CCTK_VInfo (CCTK_THORNSTRING, "Creating temporary checkpoint file '%s'", cp_tempname); + CCTK_VInfo (CCTK_THORNSTRING, "Creating file '%s'", filename); } - // writer = new IEEEIO(cp_tempname, IObase::Create); - - if (CCTK_Equals(out3D_format, "IEEE")) { - writer = new IEEEIO(cp_tempname, IObase::Create); -#ifdef HDF4 - } else if (CCTK_Equals(out3D_format, "HDF4")) { - writer = new HDFIO(cp_tempname, IObase::Create); -#endif -#ifdef HDF5 - } else if (CCTK_Equals(out3D_format, "HDF5")) { - writer = new H5IO(cp_tempname, IObase::Create); -#endif - } else { - assert (0); - } + writer = new IEEEIO(filename, IObase::Create); if (! (writer->isValid()) ) { CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, "Can't open checkpoint file '%s'. Checkpointing is skipped", - cp_tempname); + filename); return (-1); } - amrwriter = new AMRwriter(*writer); - /* now we are writing a first (bogus) dataset to which - we will attach all parameters and GHextensions as Attributes - */ - - CCTK_REAL startdata = 666.66; - int rank=1; - int dim[1]={1}; - writer->write(FLEXIO_REAL,rank,dim,&startdata); - - /* now dump parameters */ - if (verbose) - { - CCTK_VInfo (CCTK_THORNSTRING, "Dumping Parameters'"); - } + // dump parameters DumpParams (cgh, 1, writer); + } - /* and now dump GH extentions */ - if (verbose) + + // now dump the grid varibles, sorted by groups + + if (CCTK_Equals ("verbose", "full")) + { + CCTK_INFO ("Dumping Grid Variables ..."); + } + for (group = CCTK_NumGroups () - 1; group >= 0; group--) + { + /* only dump groups which have storage assigned */ + if (CCTK_QueryGroupStorageI (cgh, group) <= 0) { - CCTK_VInfo (CCTK_THORNSTRING, "Dumping GHExtensions"); + continue; } - DumpGHExtensions(cgh,writer); - - /* finally, we need all the times on the individual levels */ - const int numberoftimes=refleveltimes.size(); - writer->writeAttribute("numberoftimes",FlexIODataType(CCTK_VARIABLE_INT),1,&numberoftimes); - for(int i=0;i < numberoftimes;i++) { - char buffer[100]; - sprintf(buffer,"refleveltime%d",i); - writer->writeAttribute(buffer,FlexIODataType(CCTK_VARIABLE_REAL),1,&refleveltimes[i]); + /* dump all timelevels except the oldest (for multi-level groups) */ + CCTK_GroupData (group, &gdata); + if (gdata.numtimelevels > 1) + { + gdata.numtimelevels--; } - } - - // now dump the grid varibles for all reflevels and components, sorted by groups - BEGIN_REFLEVEL_LOOP(cgh) { - - BEGIN_MGLEVEL_LOOP(cgh) { - - if (verbose) - { - CCTK_INFO ("Dumping Grid Variables ..."); - } - for (group = CCTK_NumGroups () - 1; group >= 0; group--) - { - /* only dump groups which have storage assigned */ - - if (CCTK_QueryGroupStorageI (cgh, group) <= 0) - { - continue; - } - const int grouptype = CCTK_GroupTypeI(group); - - /* scalars and grid arrays only have 1 reflevel: */ - if ( (grouptype != CCTK_GF) && (reflevel != 0) ) - continue; - - /* now check if there is any memory allocated - for GFs and GAs. GSs should always have - memory allocated and there is at this point - no CCTK function to check this :/ - */ - - if ( (grouptype == CCTK_GF) || (grouptype == CCTK_ARRAY)){ - const int gpdim = CCTK_GroupDimI(group); - int gtotalsize=1; - for(int d=0;d<gpdim;d++){ - const int* gpsize= CCTK_ArrayGroupSizeI(cgh,d,group); - assert(gpsize != NULL); - gtotalsize*=gpsize[d]; - } - if(gtotalsize == 0){ - if (verbose) CCTK_VInfo(CCTK_THORNSTRING, "Group %s is zero-sized. No checkpoint info written",CCTK_GroupName(group)); - continue; - } - } - - - /* get the number of allocated timelevels */ - CCTK_GroupData (gindex, &gdata); - gdata.numtimelevels = 0; - gdata.numtimelevels = CCTK_GroupStorageIncrease (cgh, 1, &group, - &gdata.numtimelevels,NULL); - - - CCTK_GroupData (group, &gdata); - - /* dump all timelevels except the oldest (for multi-level groups) */ - /* COMMENTED OUT!! We _need_ all timelevels!!! - if (gdata.numtimelevels > 1) - { - gdata.numtimelevels--; - } - - */ + int first_vindex = CCTK_FirstVarIndexI (group); - int first_vindex = CCTK_FirstVarIndexI (group); + /* get the default I/O request for this group */ + request = IOUtil_DefaultIORequest (cgh, first_vindex, 1); + /* disable checking for old data objects, disable datatype conversion + and downsampling */ + request->check_exist = 0; + request->hdatatype = gdata.vartype; + for (request->hdim = 0; request->hdim < request->vdim; request->hdim++) + { + request->downsample[request->hdim] = 1; + } - - /* get the default I/O request for this group */ - request = IOUtil_DefaultIORequest (cgh, first_vindex, 1); - - /* disable checking for old data objects, disable datatype conversion - and downsampling */ - request->check_exist = 0; - request->hdatatype = gdata.vartype; - for (request->hdim = 0; request->hdim < request->vdim; request->hdim++) - { - request->downsample[request->hdim] = 1; - } - - /* loop over all variables in this group */ - for (request->vindex = first_vindex; - request->vindex < first_vindex + gdata.numvars; - request->vindex++) + /* loop over all variables in this group */ + for (request->vindex = first_vindex; + request->vindex < first_vindex + gdata.numvars; + request->vindex++) + { + /* loop over all timelevels of this variable */ + for (request->timelevel = 0; + request->timelevel < gdata.numtimelevels; + request->timelevel++) { - /* loop over all timelevels of this variable */ - for (request->timelevel = 0; - request->timelevel < gdata.numtimelevels; - request->timelevel++) - { - - if (verbose) + + if (CCTK_Equals (verbose, "full")) { fullname = CCTK_FullName (request->vindex); CCTK_VInfo (CCTK_THORNSTRING, " %s (timelevel %d)", fullname, request->timelevel); free (fullname); } - // write the var - - - if (grouptype == CCTK_ARRAY || grouptype == CCTK_GF || grouptype == CCTK_SCALAR) - { - char* fullname = CCTK_FullName (request->vindex); - if (verbose) - CCTK_VInfo (CCTK_THORNSTRING,"%s:: reflevel: %d component: %d grouptype: %d ",fullname,reflevel,component,grouptype); - free(fullname); - retval += WriteGF(cgh,writer,amrwriter,request,1); - } - else - { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Invalid group type %d for variable '%s'", grouptype, fullname); - retval = -1; - } + // write the var + retval += WriteVarAs(cgh,writer,amrwriter,request->vindex,group); + } + } /* end of loop over all variables */ - } - } /* end of loop over all variables */ - - } /* end of loop over all groups */ - - } END_MGLEVEL_LOOP; - - } END_REFLEVEL_LOOP; + } /* end of loop over all groups */ - // Close the temporary file + // Close the file if (CCTK_MyProc(cgh)==0) { delete amrwriter; amrwriter = 0; @@ -793,69 +232,15 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from) writer = 0; } - CCTK_VInfo(CCTK_THORNSTRING,"retval: %d",retval); - - if (retval == 0) - { - if (CCTK_MyProc(cgh)==0) { - if (rename (cp_tempname, cp_filename)) - { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Could not rename temporary checkpoint file '%s' to '%s'", - cp_tempname, cp_filename); - retval = -1; - } - else { - if (myGH->cp_filename_list[myGH->cp_filename_index]) - { - if (checkpoint_keep > 0) - { - remove (myGH->cp_filename_list[myGH->cp_filename_index]); - } - free (myGH->cp_filename_list[myGH->cp_filename_index]); - } - myGH->cp_filename_list[myGH->cp_filename_index] = strdup (cp_filename); - myGH->cp_filename_index = (myGH->cp_filename_index+1) % abs (checkpoint_keep); - } - } - } return 0; - } - - int RecoverVariables (cGH* cgh, IObase* reader, AmrGridReader* amrreader){ - - int retval = 0; - int myproc = CCTK_MyProc (cgh); - int currdataset,ndatasets; - - CCTK_VInfo(CCTK_THORNSTRING,"Starting to recover data!!!"); + } - if(myproc==0) { - ndatasets = reader->nDatasets(); - //CCTK_VInfo (CCTK_THORNSTRING, "ndatasets=%d", ndatasets); - assert (ndatasets>=0); - } - - // Broadcast number of datasets - MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm); - assert (ndatasets>=0); +} // namespace CarpetCheckpointRestart - for (currdataset=1;currdataset < ndatasets;currdataset++) { - if(myproc==0) { - reader->seek(currdataset); - } - - ReadGF(cgh,reader,amrreader,currdataset); - } - - return retval; - } - -} // namespace CarpetCheckpointRestart diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc index 1fba5c365..c7bd5fe72 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc @@ -13,39 +13,37 @@ -#include "cctk.h" -#include "cctk_Parameters.h" +#include <AMRwriter.hh> +#include <AmrGridReader.hh> +#include <H5IO.hh> +#include <HDFIO.hh> +#include <IEEEIO.hh> +#include <IO.hh> -#include "AMRwriter.hh" -#include "AmrGridReader.hh" -#ifdef HDF4 -# include "HDFIO.hh" -#endif -#ifdef HDF5 -# include "H5IO.hh" -#endif -#include "IEEEIO.hh" -#include "IO.hh" -// Hack to stop FlexIO data type clash with LAM MPI +// Hack to stop FlexIO type clash + #undef BYTE #undef CHAR + +#include "cctk.h" +#include "cctk_Parameters.h" + #include "CactusBase/IOUtil/src/ioGH.h" -#include "bbox.hh" -#include "data.hh" -#include "gdata.hh" -#include "ggf.hh" -#include "vect.hh" +#include "Carpet/CarpetLib/src/bbox.hh" +#include "Carpet/CarpetLib/src/data.hh" +#include "Carpet/CarpetLib/src/gdata.hh" +#include "Carpet/CarpetLib/src/ggf.hh" +#include "Carpet/CarpetLib/src/vect.hh" #include "carpet.hh" #include "ioflexio.hh" - extern "C" { - static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc,v 1.20 2004/01/13 15:46:52 cott Exp $"; + static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc,v 1.1 2003/05/16 14:02:18 hawke Exp $"; CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_ioflexio_cc); } @@ -55,18 +53,16 @@ namespace CarpetIOFlexIO { using namespace std; using namespace Carpet; - using namespace CarpetIOFlexIOUtil; - using namespace CarpetCheckpointRestart; + + // Variable definitions - // int GHExtension; + int GHExtension; int IOMethod; vector<bool> do_truncate; vector<vector<int> > last_output; - - - static const char* GetStringParameter (const char* const parametername, + const char* GetStringParameter (const char* const parametername, const char* const fallback); static int GetIntParameter (const char* const parametername, int fallback); static bool CheckForVariable (const cGH* const cgh, @@ -75,37 +71,29 @@ namespace CarpetIOFlexIO { - int CarpetIOFlexIO_Startup () + int CarpetIOFlexIOStartup () { CCTK_RegisterBanner ("AMR 3D FlexIO I/O provided by CarpetIOFlexIO"); - // GHExtension = CCTK_RegisterGHExtension("CarpetIOFlexIO"); - CCTK_RegisterGHExtensionSetupGH (CCTK_RegisterGHExtension("CarpetIOFlexIO"),SetupGH); + GHExtension = CCTK_RegisterGHExtension("CarpetIOFlexIO"); + CCTK_RegisterGHExtensionSetupGH (GHExtension, SetupGH); - IOMethod = CCTK_RegisterIOMethod ("CarpetIOFlexIO"); + IOMethod = CCTK_RegisterIOMethod ("IOFlexIO"); CCTK_RegisterIOMethodOutputGH (IOMethod, OutputGH); CCTK_RegisterIOMethodOutputVarAs (IOMethod, OutputVarAs); CCTK_RegisterIOMethodTimeToOutput (IOMethod, TimeToOutput); CCTK_RegisterIOMethodTriggerOutput (IOMethod, TriggerOutput); - - /* register the CarpetIOFlexIO recovery routine to thorn IOUtil */ - if (IOUtil_RegisterRecover ("CarpetIOFlexIO recovery", CarpetIOFlexIO_Recover) < 0) - { - CCTK_WARN (1, "Failed to register IOFlexIO recovery routine"); - } - return 0; } - void* SetupGH (tFleshConfig* const fc, const int convLevel, cGH* const cgh) + void* SetupGH (tFleshConfig* const fc, + const int convLevel, cGH* const cgh) { DECLARE_CCTK_PARAMETERS; - CarpetIOFlexIOGH* myGH; - CCTK_INT i; - + // Truncate all files if this is not a restart do_truncate.resize(CCTK_NumVars(), true); @@ -118,33 +106,11 @@ namespace CarpetIOFlexIO { // We register only once, ergo we get only one handle. We store // that statically, so there is no need to pass anything to // Cactus. - - /* allocate a new GH extension structure */ - - - CCTK_INT numvars = CCTK_NumVars (); - myGH = (CarpetIOFlexIOGH*) malloc (sizeof (CarpetIOFlexIOGH)); - myGH->out_last = (int *) malloc (numvars * sizeof (int)); - myGH->requests = (ioRequest **) calloc (numvars, sizeof (ioRequest *)); - myGH->cp_filename_list = (char **) calloc (abs (checkpoint_keep), sizeof (char *)); - myGH->cp_filename_index = 0; - myGH->out_vars = strdup (""); - myGH->out_every_default = out_every - 1; - - for (i = 0; i < numvars; i++) - { - myGH->out_last [i] = -1; - } - - myGH->open_output_files = NULL; - - - return (myGH); - return 0; } + int OutputGH (const cGH* const cgh) { for (int vindex=0; vindex<CCTK_NumVars(); ++vindex) { if (TimeToOutput(cgh, vindex)) { @@ -153,46 +119,91 @@ namespace CarpetIOFlexIO { } return 0; } - + + + + static IObase::DataType FlexIODataType (int cctk_type){ + + + int retval; + + switch (cctk_type) + { + case CCTK_VARIABLE_CHAR: retval = FLEXIO_CHAR; break; + case CCTK_VARIABLE_INT: retval = FLEXIO_INT; break; + case CCTK_VARIABLE_REAL: retval = FLEXIO_REAL; break; +#ifdef CCTK_INT2 + case CCTK_VARIABLE_INT2: retval = FLEXIO_INT2; break; +#endif +#ifdef CCTK_INT4 + case CCTK_VARIABLE_INT4: retval = FLEXIO_INT4; break; +#endif +#ifdef CCTK_INT8 + case CCTK_VARIABLE_INT8: retval = FLEXIO_INT8; break; +#endif +#ifdef CCTK_REAL4 + case CCTK_VARIABLE_REAL4: retval = FLEXIO_REAL4; break; +#endif +#ifdef CCTK_REAL8 + case CCTK_VARIABLE_REAL8: retval = FLEXIO_REAL8; break; +#endif +#ifdef CCTK_REAL16 + case CCTK_VARIABLE_REAL16: retval = FLEXIO_REAL16; break; +#endif + + default: CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "Unsupported CCTK variable datatype %d", cctk_type); + retval = -1; + break; + } + + + + return (IObase::DataType)retval; + } + + + + + - int WriteGF (const cGH* const cgh, IObase* writer, AMRwriter* amrwriter, ioRequest* request, const int called_from_checkpoint) + int WriteVarAs (const cGH* const cgh, IObase* writer, AMRwriter* amrwriter, int varindex, int group) { DECLARE_CCTK_PARAMETERS; - const int varindex = request->vindex; + /* I have got no idea why this stuff below is needed... ask Eric Schnetter */ - const int group = CCTK_GroupIndexFromVarI (varindex); const int n0 = CCTK_FirstVarIndexI(group); assert (n0>=0 && n0<CCTK_NumVars()); const int var = varindex - n0; assert (var>=0 && var<CCTK_NumVars()); - int tl = 0; - const int grouptype = CCTK_GroupTypeI(group); - - // let's get the correct Carpet time level (which is the (-1) * Cactus timelevel): - if (request->timelevel==0) - tl = 0; - else - tl = - request->timelevel; - - assert (! ( (grouptype != CCTK_GF) && reflevel>0)); + const int tl = 0; if (CCTK_MyProc(cgh)==0) { + // Set datatype + //fprintf(stderr,"%d\n",CCTK_VarTypeI(varindex)); + /* should be obsolete now + assert (CCTK_VarTypeI(varindex) == CCTK_VARIABLE_REAL8 + || (sizeof(CCTK_REAL) == sizeof(CCTK_REAL8) + && CCTK_VarTypeI(varindex) == CCTK_VARIABLE_REAL)); + */ amrwriter->setType (FlexIODataType(CCTK_VarTypeI(varindex))); + // Set name information + - int gpdim = CCTK_GroupDimI(group); - // need gpdim=1 if scalar (flexio wants this) - if(gpdim == 0) - if(grouptype == CCTK_SCALAR) - gpdim = 1; - else - CCTK_WARN(0,"Non-scalar variable with dimension 0!!!"); - + // Set name information + + // char *name = CCTK_FullName (varindex); + //IOwriteAttribute(amrwriter->file,"name",IObase::Char,strlen(name)+1,name); + //free(name); + + const int gpdim = CCTK_GroupDimI(group); + // Set coordinate information CCTK_REAL lower[dim], upper[dim]; double origin[dim], delta[dim], timestep; @@ -203,7 +214,7 @@ namespace CarpetIOFlexIO { origin[d] = lower[d]; delta[d] = cgh->cctk_delta_space[d]; } - timestep = cgh->cctk_delta_time; + timestep = base_delta_time; amrwriter->setTopLevelParameters (gpdim, origin, delta, timestep, maxreflevels); @@ -226,137 +237,69 @@ namespace CarpetIOFlexIO { // Set current time amrwriter->setTime (cgh->cctk_iteration); } - - // Traverse all components on this refinement and multigrid level - BEGIN_COMPONENT_LOOP(cgh, grouptype) { - + BEGIN_COMPONENT_LOOP(cgh) { + const ggf<dim>* ff = 0; - + assert (var < (int)arrdata[group].data.size()); ff = (ggf<dim>*)arrdata[group].data[var]; - const gdata<dim>* const data = (*ff) (tl, reflevel, component, mglevel); - - // get some more group information - cGroupDynamicData gdyndata; - - int ierr = CCTK_GroupDynamicData(cgh,group,&gdyndata); - assert(ierr==0); - - cGroup cgdata; - ierr = CCTK_GroupData(group,&cgdata); - assert(ierr==0); - - /* handle CCTK_DISTRIB_CONSTANT scalar and arrays */ -#if 0 - if (cgdata.disttype == CCTK_DISTRIB_CONSTANT) { - assert(grouptype == CCTK_ARRAY || grouptype == CCTK_SCALAR); - if (hh->processors[reflevel][component] == 0) { - if (grouptype == CCTK_SCALAR) { - CCTK_VInfo (CCTK_THORNSTRING, "dumping SCALAR distrib const"); - int rank=1; - int dim[1]={1}; - writer -> write(FlexIODataType(CCTK_VarTypeI(varindex)),rank,dim,CCTK_VarDataPtrI(cgh,tl,varindex)); - DumpCommonAttributes(cgh,writer,request); - continue; - } - else { - writer -> write(FlexIODataType(CCTK_VarTypeI(varindex)),cgdata.dim,gdyndata.lsh,CCTK_VarDataPtrI(cgh,tl,varindex)); - DumpCommonAttributes(cgh,writer,request); - continue; - } - } - else { - continue; - } - - } -#endif - + const gdata<dim>* const data + = (*ff) (tl, reflevel, component, mglevel); + // Make temporary copy on processor 0 bbox<int,dim> ext = data->extent(); vect<int,dim> lo = ext.lower(); vect<int,dim> hi = ext.upper(); vect<int,dim> str = ext.stride(); - - // Ignore ghost zones if desired - - const int out3D_output_outer_boundary_var = (called_from_checkpoint) ? -1 : out3D_output_outer_boundary; - const int out3D_max_num_lower_ghosts_var = (called_from_checkpoint) ? -1 : out3D_max_num_lower_ghosts; - const int out3D_max_num_upper_ghosts_var = (called_from_checkpoint) ? -1 : out3D_max_num_upper_ghosts; - + // Ignore ghost zones if desired for (int d=0; d<dim; ++d) { - const int max_lower_ghosts = (gdyndata.bbox[2*d ] && out3D_output_outer_boundary_var) ? -1 : out3D_max_num_lower_ghosts_var; - const int max_upper_ghosts = (gdyndata.bbox[2*d+1] && out3D_output_outer_boundary_var) ? -1 : out3D_max_num_upper_ghosts_var; + const int max_lower_ghosts = (cgh->cctk_bbox[2*d ] && !out3D_output_outer_boundary) ? -1 : out3D_max_num_lower_ghosts; + const int max_upper_ghosts = (cgh->cctk_bbox[2*d+1] && !out3D_output_outer_boundary) ? -1 : out3D_max_num_upper_ghosts; - const int num_lower_ghosts = max_lower_ghosts == -1 ? gdyndata.nghostzones[d] : min(out3D_max_num_lower_ghosts_var, gdyndata.nghostzones[d]); - const int num_upper_ghosts = max_upper_ghosts == -1 ? gdyndata.nghostzones[d] : min(out3D_max_num_upper_ghosts_var, gdyndata.nghostzones[d]); + const int num_lower_ghosts = max_lower_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_lower_ghosts, cgh->cctk_nghostzones[d]); + const int num_upper_ghosts = max_upper_ghosts == -1 ? cgh->cctk_nghostzones[d] : min(out3D_max_num_upper_ghosts, cgh->cctk_nghostzones[d]); - lo[d] += (gdyndata.nghostzones[d] - num_lower_ghosts) * str[d]; - hi[d] -= (gdyndata.nghostzones[d] - num_upper_ghosts) * str[d]; + lo[d] += (cgh->cctk_nghostzones[d] - num_lower_ghosts) * str[d]; + hi[d] -= (cgh->cctk_nghostzones[d] - num_upper_ghosts) * str[d]; } ext = bbox<int,dim>(lo,hi,str); - gdata<dim>* const tmp = data->make_typed (varindex); + gdata<dim>* const tmp = data->make_typed (); tmp->allocate (ext, 0); - //fprintf(stderr,"\n writing1 %d\n",CCTK_MyProc(cgh)); - if ( !((cgdata.disttype == CCTK_DISTRIB_CONSTANT) && (hh->processors[reflevel][component]!=0))) { - - if (cgdata.disttype == CCTK_DISTRIB_CONSTANT) { - assert(grouptype == CCTK_ARRAY || grouptype == CCTK_SCALAR); - //fprintf(stderr,"\n scalar %d %d comp: %d\n",CCTK_MyProc(cgh),varindex,component); - int origin[dim], dims[dim]; - for (int d=0; d<dim; ++d) { - origin[d] = (ext.lower() / ext.stride())[d]; - dims[d] = (ext.shape() / ext.stride())[d]; - } - if (CCTK_MyProc(cgh)==0) { - amrwriter->write (origin, dims, (void*)data->storage()); - DumpCommonAttributes(cgh,writer,request); - } - delete tmp; - continue; - } else { - - for (comm_state<dim> state; !state.done(); state.step()) { - tmp->copy_from (state, data, ext); - } - - //fprintf(stderr,"\n writing2 %d component: %d varindex: %d distrib_const: %d\n",CCTK_MyProc(cgh),component,varindex,(cgdata.disttype == CCTK_DISTRIB_CONSTANT)); - // Write data - if (CCTK_MyProc(cgh)==0) { - int origin[dim], dims[dim]; - for (int d=0; d<dim; ++d) { - origin[d] = (ext.lower() / ext.stride())[d]; - dims[d] = (ext.shape() / ext.stride())[d]; - } - - amrwriter->write (origin, dims, (void*)tmp->storage()); - - // dump attributes - DumpCommonAttributes(cgh,writer,request); - - } - // Delete temporary copy - - delete tmp; - + tmp->copy_from (data, ext); + + // Write data + if (CCTK_MyProc(cgh)==0) { + int origin[dim], dims[dim]; + for (int d=0; d<dim; ++d) { + origin[d] = (ext.lower() / ext.stride())[d]; + dims[d] = (ext.shape() / ext.stride())[d]; } + amrwriter->write (origin, dims, (void*)tmp->storage()); + char *name = CCTK_FullName (varindex); + writer->writeAttribute("name",IObase::Char,strlen(name)+1,name); + free(name); } - } END_COMPONENT_LOOP; + + // Delete temporary copy + delete tmp; + + } END_COMPONENT_LOOP(cgh); - return 0; + return 0; } - - + + int OutputVarAs (const cGH* const cgh, const char* const varname, const char* const alias) { DECLARE_CCTK_PARAMETERS; + const int n = CCTK_VarIndex(varname); assert (n>=0 && n<CCTK_NumVars()); const int group = CCTK_GroupIndexFromVarI (n); @@ -375,13 +318,6 @@ namespace CarpetIOFlexIO { return 0; } - const int grouptype = CCTK_GroupTypeI(group); - if (grouptype != CCTK_GF && reflevel>0) return 0; - - int first_vindex = CCTK_FirstVarIndexI (group); - /* get the default I/O request for this group */ - ioRequest* request = IOUtil_DefaultIORequest (cgh, first_vindex, 1); - // Get grid hierarchy extentsion from IOUtil const ioGH * const iogh = (const ioGH *)CCTK_GHExtension (cgh, "IO"); assert (iogh); @@ -396,11 +332,9 @@ namespace CarpetIOFlexIO { const char* extension = 0; if (CCTK_Equals(out3D_format, "IEEE")) { extension = ".raw"; -#ifdef HDF4 +#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF4")) { extension = ".hdf"; -#endif -#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF5")) { extension = ".h5"; #endif @@ -428,11 +362,9 @@ namespace CarpetIOFlexIO { writer = 0; if (CCTK_Equals(out3D_format, "IEEE")) { writer = new IEEEIO(filename, IObase::Create); -#ifdef HDF4 +#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF4")) { writer = new HDFIO(filename, IObase::Create); -#endif -#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF5")) { writer = new H5IO(filename, IObase::Create); #endif @@ -447,11 +379,9 @@ namespace CarpetIOFlexIO { // Open the file if (CCTK_Equals(out3D_format, "IEEE")) { writer = new IEEEIO(filename, IObase::Append); -#ifdef HDF4 +#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF4")) { writer = new HDFIO(filename, IObase::Append); -#endif -#ifdef HDF5 } else if (CCTK_Equals(out3D_format, "HDF5")) { writer = new H5IO(filename, IObase::Append); #endif @@ -464,8 +394,7 @@ namespace CarpetIOFlexIO { amrwriter = new AMRwriter(*writer); } - - WriteGF(cgh,writer,amrwriter,request,0); + WriteVarAs(cgh,writer,amrwriter,n,group); // Close the file if (CCTK_MyProc(cgh)==0) { @@ -481,7 +410,7 @@ namespace CarpetIOFlexIO { return 0; } - + int TimeToOutput (const cGH* const cgh, const int vindex) { DECLARE_CCTK_PARAMETERS; @@ -537,295 +466,6 @@ namespace CarpetIOFlexIO { return retval; } - - - int ReadGF (const cGH* const cgh, IObase* reader, AmrGridReader* amrreader,int currdataset) { - - /* this functions reads in a variable on the current reflevel from an already open file. At - some point it should be called from InputVarAs */ - - - DECLARE_CCTK_PARAMETERS; - - int tl = -1; - int mglevel = -1; - int rl = -1; - int comp = -1; - int myproc = CCTK_MyProc (cgh); - int rank; - int dims[dim]; - int nbytes; - - char* varname; - char warnstring[256]; - int asize,i; - IObase::DataType datatype; - int group,varindex; - CCTK_REAL cctk_time; - - if(myproc==0) { - // read the name of the variable - i = reader->readAttributeInfo ("name", datatype, asize); - if (i >= 0 && datatype == FLEXIO_CHAR && asize > 0) - { - varname = (char*) malloc(sizeof(char)*asize+1); - reader->readAttribute (i, varname); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read dataset names!!!"); - } - - varindex = CCTK_VarIndex(varname); - - assert(varindex > -1); - - group = CCTK_GroupIndexFromVarI(varindex); - assert(group > -1); - - // Check for storage - if (! CCTK_QueryGroupStorageI(cgh, group)) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot recover variable \"%s\" because it has no storage", - varname); - return 0; - } - - - // read reflevel - i = reader->readAttributeInfo ("reflevel", datatype, asize); - if (i >= 0 && datatype == FLEXIO_INT && asize > 0) - { - reader->readAttribute (i, &rl); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read refinement level!!!"); - } - - i = reader->readAttributeInfo ("component", datatype, asize); - if (i >= 0 && datatype == FLEXIO_INT && asize > 0) - { - reader->readAttribute (i, &comp); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read component!!!"); - } - - i = reader->readAttributeInfo ("timelevel", datatype, asize); - if (i >= 0 && datatype == FLEXIO_INT && asize > 0) - { - reader->readAttribute (i, &tl); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read timelevel!!!"); - } - - i = reader->readAttributeInfo ("mglevel", datatype, asize); - if (i >= 0 && datatype == FLEXIO_INT && asize > 0) - { - reader->readAttribute (i, &mglevel); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read multi group level!!!"); - } - - i = reader->readAttributeInfo ("cctk_time", datatype, asize); - if (i >= 0 && datatype == FLEXIO_REAL && asize > 0) - { - reader->readAttribute (i, &cctk_time); - } - else - { - CCTK_WARN (0, "Something is wrong! Can't read coordinate time!!!"); - } - - - - // Read information about dataset - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading dataset info"); - reader->readInfo (datatype, rank, dims); - nbytes = IObase::nBytes(datatype,rank,dims); - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "type=%d rank=%d dims=[%d,%d,%d] nbytes=%d", (int)datatype, rank, dims[0], dims[1], dims[2], nbytes); - - int gpdim = CCTK_GroupDimI(group); - const int grouptype = CCTK_GroupTypeI(group); - - // need gpdim=1 if scalar (flexio wants this) - if(gpdim == 0) - if(grouptype == CCTK_SCALAR) - gpdim = 1; - else - CCTK_WARN(0,"Non-scalar variable with dimension 0!!!"); - - - CCTK_VInfo(CCTK_THORNSTRING,"Recovering varindex: %d grouptype: %d varname: %s tl: %d, rl: %d, c: %d",varindex,grouptype,varname,tl,rl,comp); - - free(varname); - - // Check rank - assert (rank==gpdim); - } - - - // Broadcast varindex,group,rank, dimensions, and nbytes,rl,tl,mglevel - MPI_Bcast (&varindex, 1, MPI_INT, 0, dist::comm); - assert (varindex>=0); - MPI_Bcast (&group, 1, MPI_INT, 0, dist::comm); - assert (group>=0); - MPI_Bcast (&rank, 1, MPI_INT, 0, dist::comm); - assert (rank>=1); - MPI_Bcast (&dims, rank, MPI_INT, 0, dist::comm); - for (int d=0; d<rank; ++d) assert (dims[d]>=0); - MPI_Bcast (&nbytes, 1, MPI_INT, 0, dist::comm); - assert (nbytes>=0); - - MPI_Bcast (&rl, 1, MPI_INT, 0, dist::comm); - MPI_Bcast (&tl, 1, MPI_INT, 0, dist::comm); - MPI_Bcast (&mglevel, 1, MPI_INT, 0, dist::comm); - MPI_Bcast (&comp, 1, MPI_INT, 0, dist::comm); - - - int gpdim = CCTK_GroupDimI(group); - const int grouptype = CCTK_GroupTypeI(group); - - cGroup cgdata; - int ierr = CCTK_GroupData(group,&cgdata); - assert(ierr==0); - - // Read grid - AmrGrid* amrgrid = 0; - int amr_origin[dim]; - int amr_dims[dim]; - if (myproc==0) { - - // Read data - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading AMR data"); - - amrgrid = amrreader->getGrid(currdataset); - assert (amrgrid!=0); - assert (amrgrid->data!=0); - - IObase::DataType atype; - int alength; - // If iorigin attribute is absent, assume file has unigrid - // data. Initialize iorigin to 0. - if (reader->readAttributeInfo("iorigin", atype, alength) < 0) { - for (int d=0; d<gpdim; ++d) { - amrgrid->iorigin[d] = 0; - } - } - for (int d=0; d<gpdim; ++d) { - amr_origin[d] = amrgrid->iorigin[d]; - // fprintf(stderr,"\n amr_origin[%d]=%d",d,amr_origin[d]); - amr_dims[d] = amrgrid->dims[d]; - //fprintf(stderr,"\n amr_dims[%d]=%d",d,amr_dims[d]); - } - for (int d=gpdim; d<dim; ++d) { - amr_origin[d] = 0; - amr_dims[d] = 1; - } - - } // MyProc == 0 - - - MPI_Bcast (amr_origin, dim, MPI_INT, 0, dist::comm); - MPI_Bcast (amr_dims, dim, MPI_INT, 0, dist::comm); - - const int n0 = CCTK_FirstVarIndexI(group); - assert (n0>=0 && n0<CCTK_NumVars()); - const int var = varindex - n0; - assert (var>=0 && var<CCTK_NumVars()); - - - // Traverse all components on this refinement and multigrid - // level - - - // fprintf(stderr,"\n bogus! reflevel:%d mglevel:%d\n",reflevel,mglevel); - //fprintf(stderr,"blahblah: rank: %d dims[0,1,2]: %d,%d,%d\n",rank,dims[0],dims[1],dims[2]); - - // cout << "var " << varindex << " has " << CCTK_NumTimeLevelsFromVarI(varindex) << " timelevels" << endl; - - // BEGIN_COMPONENT_LOOP(cgh, grouptype) { - - //cout << "compontents " << hh->components(rl) << endl; - - //cout << "myproc: " << CCTK_MyProc(cgh) << endl; - // fprintf(stderr,"%d amr_dims: %d,%d,%d\n",CCTK_MyProc(cgh),amr_dims[0],amr_dims[1],amr_dims[2]); - //fprintf(stderr,"%d amr_origin: %d,%d,%d\n",CCTK_MyProc(cgh),amr_origin[0],amr_origin[1],amr_origin[2]); - - for(int c=0;c<hh->components(rl);c++) { - - ggf<dim>* ff = 0; - - assert (var < (int)arrdata[group].data.size()); - ff = (ggf<dim>*)arrdata[group].data[var]; - - gdata<dim>* const data = (*ff) (tl, rl, c, mglevel); - - - // Create temporary data storage on processor 0 - const int reflevelfact_local=ipow(reffact,rl); - vect<int,dim> str = vect<int,dim>(maxreflevelfact/reflevelfact_local); - - if(grouptype == CCTK_SCALAR || grouptype == CCTK_ARRAY) - str = vect<int,dim> (1); - - - vect<int,dim> lb = vect<int,dim>(amr_origin) * str; - vect<int,dim> ub = lb + (vect<int,dim>(amr_dims) - 1) * str; - - - - gdata<dim>* const tmp = data->make_typed (varindex); - - - // Copy into grid function - - if (cgdata.disttype == CCTK_DISTRIB_CONSTANT) { - assert(grouptype == CCTK_ARRAY || grouptype == CCTK_SCALAR); - if (grouptype == CCTK_SCALAR) { - lb[0] = hh->processors.at(rl).at(c); - ub[0] = hh->processors.at(rl).at(c); - } else { - lb[dim-1] = lb[dim-1] + (ub[dim-1]-lb[dim-1]+1)*hh->processors.at(rl).at(c); - ub[dim-1] = ub[dim-1] + (ub[dim-1]-lb[dim-1]+1)*hh->processors.at(rl).at(c); - } - } - - const bbox<int,dim> ext(lb,ub,str); - - - if (myproc==0) { - tmp->allocate (ext, 0, amrgrid->data); - } else { - tmp->allocate (ext, 0, 0); - } - - for (comm_state<dim> state; !state.done(); state.step()) { - data->copy_from (state, tmp, ext & data->extent() ); - } - - - // Delete temporary copy - delete tmp; - - } // manual component loop - - if (myproc==0) { - free (amrgrid->data); - free (amrgrid); - amrgrid = 0; - } - - - return 0; - } - int InputGH (const cGH* const cgh) { @@ -855,211 +495,222 @@ namespace CarpetIOFlexIO { assert (n0>=0 && n0<CCTK_NumVars()); const int var = n - n0; assert (var>=0 && var<CCTK_NumVars()); - const int tl = 0; // CCTK_VInfo (CCTK_THORNSTRING, "boguscheck reflevel,component,mglevel %d,%d,%d",reflevel,component,mglevel); - + const int tl = 0; - // Check for storage - if (! CCTK_QueryGroupStorageI(cgh, group)) { - CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, - "Cannot input variable \"%s\" because it has no storage", - varname); + switch (CCTK_GroupTypeI(group)) { + + case CCTK_SCALAR: { + // Don't input scalars + CCTK_VWarn (2, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannout input variable \"%s\" because it is a scalar", + varname); return 0; } - const int grouptype = CCTK_GroupTypeI(group); - const int rl = grouptype==CCTK_GF ? reflevel : 0; - - // Find the input directory - const char* myindir = GetStringParameter("indir3D", ""); - - // Invent a file name - const char* extension = 0; - if (CCTK_Equals(in3D_format, "IEEE")) { - extension = ".raw"; -#ifdef HDF4 - } else if (CCTK_Equals(in3D_format, "HDF4")) { - extension = ".hdf"; -#endif -#ifdef HDF5 - } else if (CCTK_Equals(in3D_format, "HDF5")) { - extension = ".h5"; -#endif - } else { - assert (0); - } - extension = GetStringParameter ("in3D_extension", extension); - - ostringstream filenamebuf; - filenamebuf << myindir << "/" << alias << extension; - string filenamestr = filenamebuf.str(); - const char * const filename = filenamestr.c_str(); - - IObase* reader = 0; - AmrGridReader* amrreader = 0; - int ndatasets = -1; - - const int gpdim = CCTK_GroupDimI(group); - - int rank; - int dims[dim]; - int nbytes; - - // Read the file only on the root processor - if (CCTK_MyProc(cgh)==0) { + case CCTK_ARRAY: + case CCTK_GF: { - // Open the file - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Opening file \"%s\"", filename); + // Check for storage + if (! CCTK_QueryGroupStorageI(cgh, group)) { + CCTK_VWarn (1, __LINE__, __FILE__, CCTK_THORNSTRING, + "Cannot input variable \"%s\" because it has no storage", + varname); + return 0; + } + + // Find the input directory + const char* myindir = GetStringParameter("indir3D", ""); + + // Invent a file name + const char* extension = 0; if (CCTK_Equals(in3D_format, "IEEE")) { - reader = new IEEEIO(filename, IObase::Read); -#ifdef HDF4 - } else if (CCTK_Equals(in3D_format, "HDF4")) { - reader = new HDFIO(filename, IObase::Read); -#endif + extension = ".raw"; #ifdef HDF5 + } else if (CCTK_Equals(in3D_format, "HDF4")) { + extension = ".hdf"; } else if (CCTK_Equals(in3D_format, "HDF5")) { - reader = new H5IO(filename, IObase::Read); + extension = ".h5"; #endif } else { - assert (0); - } - if (!reader->isValid()) { - CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, - "Could not open file \"%s\" for reading", filename); + assert (0); } - assert (reader->isValid()); + extension = GetStringParameter ("in3D_extension", extension); - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading AMR info"); - amrreader = new AmrGridReader(*reader); + ostringstream filenamebuf; + filenamebuf << myindir << "/" << alias << extension; + string filenamestr = filenamebuf.str(); + const char * const filename = filenamestr.c_str(); - // Read information about dataset - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading dataset info"); - IObase::DataType numbertype; - reader->readInfo (numbertype, rank, dims); - nbytes = IObase::nBytes(numbertype,rank,dims); - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "type=%d rank=%d dims=[%d,%d,%d] nbytes=%d", (int)numbertype, rank, dims[0], dims[1], dims[2], nbytes); + IObase* reader = 0; + AmrGridReader* amrreader = 0; + int ndatasets = -1; - // Check rank - assert (rank==gpdim); + const int gpdim = CCTK_GroupDimI(group); - // Check datatype - // TODO: Check datatype correctly - assert (CCTK_VarTypeI(n) == CCTK_VARIABLE_REAL8 - || (sizeof(CCTK_REAL) == sizeof(CCTK_REAL8) - && CCTK_VarTypeI(n) == CCTK_VARIABLE_REAL)); + int rank; + int dims[dim]; + int nbytes; - // TODO: check grid spacing - - // Number of datasets - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading number of datasets"); - ndatasets = reader->nDatasets(); - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "ndatasets=%d", ndatasets); - assert (ndatasets>=0); - } - - // Broadcast rank, dimensions, and nbytes - MPI_Bcast (&rank, 1, MPI_INT, 0, dist::comm); - assert (rank>=1); - MPI_Bcast (&dims, rank, MPI_INT, 0, dist::comm); - for (int d=0; d<rank; ++d) assert (dims[d]>=0); - MPI_Bcast (&nbytes, 1, MPI_INT, 0, dist::comm); - assert (nbytes>=0); - - // Broadcast number of datasets - MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm); - assert (ndatasets>=0); - - // Read all datasets - // TODO: read only some datasets - for (int dataset=0; dataset<ndatasets; ++dataset) { - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Handling dataset #%d", dataset); + // Read the file only on the root processor + if (CCTK_MyProc(cgh)==0) { + + // Open the file + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Opening file \"%s\"", filename); + if (CCTK_Equals(in3D_format, "IEEE")) { + reader = new IEEEIO(filename, IObase::Read); +#ifdef HDF5 + } else if (CCTK_Equals(in3D_format, "HDF4")) { + reader = new HDFIO(filename, IObase::Read); + } else if (CCTK_Equals(in3D_format, "HDF5")) { + reader = new H5IO(filename, IObase::Read); +#endif + } else { + assert (0); + } + if (!reader->isValid()) { + CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING, + "Could not open file \"%s\" for reading", filename); + } + assert (reader->isValid()); + + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading AMR info"); + amrreader = new AmrGridReader(*reader); + + // Read information about dataset + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading dataset info"); + IObase::DataType numbertype; + reader->readInfo (numbertype, rank, dims); + nbytes = IObase::nBytes(numbertype,rank,dims); + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "type=%d rank=%d dims=[%d,%d,%d] nbytes=%d", (int)numbertype, rank, dims[0], dims[1], dims[2], nbytes); + + // Check rank + assert (rank==gpdim); + + // Check datatype + // TODO: Check datatype correctly + assert (CCTK_VarTypeI(n) == CCTK_VARIABLE_REAL8 + || (sizeof(CCTK_REAL) == sizeof(CCTK_REAL8) + && CCTK_VarTypeI(n) == CCTK_VARIABLE_REAL)); + + // TODO: check grid spacing + + // Number of datasets + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading number of datasets"); + ndatasets = reader->nDatasets(); + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "ndatasets=%d", ndatasets); + assert (ndatasets>=0); + } - // Read grid - AmrGrid* amrgrid = 0; - int amr_origin[dim]; - int amr_dims[dim]; + // Broadcast rank, dimensions, and nbytes + MPI_Bcast (&rank, 1, MPI_INT, 0, dist::comm); + assert (rank>=1); + MPI_Bcast (&dims, rank, MPI_INT, 0, dist::comm); + for (int d=0; d<rank; ++d) assert (dims[d]>=0); + MPI_Bcast (&nbytes, 1, MPI_INT, 0, dist::comm); + assert (nbytes>=0); - if (CCTK_MyProc(cgh)==0) { - - // Read data - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading AMR data"); - amrgrid = amrreader->getGrid(dataset); - assert (amrgrid!=0); - assert (amrgrid->data!=0); - - // If iorigin attribute is absent, assume file has unigrid - // data. Initialize iorigin to 0. - IObase::DataType atype; - int alength; - if (reader->readAttributeInfo("iorigin", atype, alength) < 0) { - for (int d=0; d<gpdim; ++d) { - amrgrid->iorigin[d] = 0; - } - } - - for (int d=0; d<gpdim; ++d) { - amr_origin[d] = amrgrid->iorigin[d]; - amr_dims[d] = amrgrid->dims[d]; - } - for (int d=gpdim; d<dim; ++d) { - amr_origin[d] = 0; - amr_dims[d] = 1; - } - - } // MyProc == 0 - MPI_Bcast (amr_origin, dim, MPI_INT, 0, dist::comm); - MPI_Bcast (amr_dims, dim, MPI_INT, 0, dist::comm); + // Broadcast number of datasets + MPI_Bcast (&ndatasets, 1, MPI_INT, 0, dist::comm); + assert (ndatasets>=0); - // Traverse all components on this refinement and multigrid - // level - BEGIN_COMPONENT_LOOP(cgh, grouptype) { - - ggf<dim>* ff = 0; - - assert (var < (int)arrdata[group].data.size()); - ff = (ggf<dim>*)arrdata[group].data[var]; - - gdata<dim>* const data = (*ff) (tl, rl, component, mglevel); - - // Create temporary data storage on processor 0 - const vect<int,dim> str = vect<int,dim>(reflevelfact); - const vect<int,dim> lb = vect<int,dim>(amr_origin) * str; - const vect<int,dim> ub - = lb + (vect<int,dim>(amr_dims) - 1) * str; - const bbox<int,dim> ext(lb,ub,str); - gdata<dim>* const tmp = data->make_typed (n); - - if (CCTK_MyProc(cgh)==0) { - tmp->allocate (ext, 0, amrgrid->data); - } else { - tmp->allocate (ext, 0, 0); - } - - // Copy into grid function - for (comm_state<dim> state; !state.done(); state.step()) { - data->copy_from (state, tmp, ext); - } - - // Delete temporary copy - delete tmp; - - } END_COMPONENT_LOOP; + // Read all datasets + // TODO: read only some datasets + for (int dataset=0; dataset<ndatasets; ++dataset) { + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Handling dataset #%d", dataset); + + // Read grid + AmrGrid* amrgrid = 0; + int amr_origin[dim]; + int amr_dims[dim]; + + if (CCTK_MyProc(cgh)==0) { + + // Read data + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Reading AMR data"); + amrgrid = amrreader->getGrid(dataset); + assert (amrgrid!=0); + assert (amrgrid->data!=0); + + // If iorigin attribute is absent, assume file has unigrid + // data. Initialize iorigin to 0. + IObase::DataType atype; + int alength; + if (reader->readAttributeInfo("iorigin", atype, alength) < 0) { + for (int d=0; d<gpdim; ++d) { + amrgrid->iorigin[d] = 0; + } + } + + for (int d=0; d<gpdim; ++d) { + amr_origin[d] = amrgrid->iorigin[d]; + amr_dims[d] = amrgrid->dims[d]; + } + for (int d=gpdim; d<dim; ++d) { + amr_origin[d] = 0; + amr_dims[d] = 1; + } + + } // MyProc == 0 + MPI_Bcast (amr_origin, dim, MPI_INT, 0, dist::comm); + MPI_Bcast (amr_dims, dim, MPI_INT, 0, dist::comm); + + // Traverse all components on this refinement and multigrid + // level + BEGIN_COMPONENT_LOOP(cgh) { + + ggf<dim>* ff = 0; + + assert (var < (int)arrdata[group].data.size()); + ff = (ggf<dim>*)arrdata[group].data[var]; + + gdata<dim>* const data + = (*ff) (tl, reflevel, component, mglevel); + + // Create temporary data storage on processor 0 + const vect<int,dim> str = vect<int,dim>(reflevelfact); + const vect<int,dim> lb = vect<int,dim>(amr_origin) * str; + const vect<int,dim> ub + = lb + (vect<int,dim>(amr_dims) - 1) * str; + const bbox<int,dim> ext(lb,ub,str); + gdata<dim>* const tmp = data->make_typed (); + + if (CCTK_MyProc(cgh)==0) { + tmp->allocate (ext, 0, amrgrid->data); + } else { + tmp->allocate (ext, 0, 0); + } + + // Copy into grid function + data->copy_from (tmp, ext); + + // Delete temporary copy + delete tmp; + + } END_COMPONENT_LOOP(cgh); + + if (CCTK_MyProc(cgh)==0) { + free (amrgrid->data); + free (amrgrid); + amrgrid = 0; + } + + } // loop over datasets + // Close the file if (CCTK_MyProc(cgh)==0) { - free (amrgrid->data); - free (amrgrid); - amrgrid = 0; + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Deleting AMR info"); + delete amrreader; + amrreader = 0; + if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Closing file"); + delete reader; + reader = 0; } - } // loop over datasets - - // Close the file - if (CCTK_MyProc(cgh)==0) { - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Deleting AMR info"); - delete amrreader; - amrreader = 0; - if (verbose) CCTK_VInfo (CCTK_THORNSTRING, "Closing file"); - delete reader; - reader = 0; + break; + } // ARRAY or GROUP + + default: + assert (0); } return 0; @@ -1067,7 +718,7 @@ namespace CarpetIOFlexIO { - int CarpetIOFlexIO_ReadData (CCTK_ARGUMENTS) + int CarpetIOFlexIOReadData (CCTK_ARGUMENTS) { DECLARE_CCTK_ARGUMENTS; return InputGH(cctkGH); @@ -1129,6 +780,9 @@ namespace CarpetIOFlexIO { flags[index] = true; } - + } // namespace CarpetIOFlexIO + + + diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h index d57890a74..4c9941e4e 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h @@ -1,4 +1,4 @@ -/* $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h,v 1.3 2003/12/01 13:15:21 cott Exp $ */ +/* $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.h,v 1.1 2003/05/16 14:02:18 hawke Exp $ */ #ifndef CARPETIOFLEXIO_H #define CARPETIOFLEXIO_H @@ -13,8 +13,8 @@ namespace CarpetIOFlexIO { #endif /* Scheduled functions */ - int CarpetIOFlexIO_Startup (void); - int CarpetIOFlexIO_ReadData (CCTK_ARGUMENTS); + int CarpetIOFlexIOStartup (void); + int CarpetIOFlexIOReadData (CCTK_ARGUMENTS); #ifdef __cplusplus } /* extern "C" */ } /* namespace CarpetIOFlexIO */ @@ -26,9 +26,7 @@ namespace CarpetCheckpointRestart { #endif /* Scheduled functions */ - void CarpetIOFlexIO_EvolutionCheckpoint (const cGH*); - int CarpetIOFlexIO_RecoverParameters (void); - int CarpetIOFlexIO_Recover (cGH *GH, const char *basefilename, int called_from); + void CarpetChReEvolutionCheckpoint (const cGH*); #ifdef __cplusplus } /* extern "C" */ } /* namespace CarpetIOFlexIO */ @@ -36,8 +34,3 @@ namespace CarpetCheckpointRestart { #endif /* !defined(CARPETIOFLEXIO_H) */ - - - - - diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh index 4f42bdf5b..bd235fd66 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh @@ -1,20 +1,14 @@ -// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh,v 1.13 2004/01/12 10:50:31 cott Exp $ +// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh,v 1.1 2003/05/16 14:02:18 hawke Exp $ #ifndef CARPETIOFLEXIO_HH #define CARPETIOFLEXIO_HH - #include <vector> #include "cctk.h" #include "cctk_Arguments.h" -#include "CactusBase/IOUtil/src/ioGH.h" -#include "CactusBase/IOUtil/src/ioutil_CheckpointRecovery.h" -#include "CactusBase/IOUtil/src/ioutil_Utils.h" - #include "ioflexio.h" -#include "ioflexioGH.h" /* define the IOFlexIO datatypes according to CCTK_??? datatypes */ #define FLEXIO_CHAR IObase::Char @@ -55,63 +49,6 @@ #elif CCTK_REAL_PRECISION_16 #define FLEXIO_REAL FLEXIO_REAL16 #endif - -/* some macros needed for recovery */ -#ifdef CCTK_MPI - -#define CACTUS_MPI_ERROR(fn_call) \ - do { \ - int errcode; \ - \ - if ((errcode = fn_call) != MPI_SUCCESS) \ - { \ - char mpi_error_string[MPI_MAX_ERROR_STRING+1]; \ - int resultlen; \ - \ - MPI_Error_string (errcode, mpi_error_string, &resultlen); \ - fprintf (stderr, "MPI call '%s' returned error code %d (%s)\n", \ - #fn_call, errcode, mpi_error_string); \ - fprintf(stderr, "At line %d of file %s\n", __LINE__, __FILE__); \ - } \ - } while (0) - - -#ifdef CCTK_INT4 -#define CARPET_MPI_INT4 (sizeof (CCTK_INT4) == sizeof (int) ? MPI_INT : \ - sizeof (CCTK_INT4) == sizeof (short) ? MPI_SHORT : \ - MPI_DATATYPE_NULL) -#endif - -#define CARPET_MPI_CHAR MPI_CHAR - -/* floating point types are architecture-independent, - ie. a float has always 4 bytes, and a double has 8 bytes - - PUGH_MPI_REAL is used for communicating reals of the generic CCTK_REAL type - PUGH_MPI_REALn is used to explicitely communicate n-byte reals */ -#ifdef CCTK_REAL4 -#define CARPET_MPI_REAL4 MPI_FLOAT -#endif -#ifdef CCTK_REAL8 -#define CARPET_MPI_REAL8 MPI_DOUBLE -#endif -#ifdef CCTK_REAL16 -#define CARPET_MPI_REAL16 (sizeof (CCTK_REAL16) == sizeof (long double) ? \ - MPI_LONG_DOUBLE : MPI_DATATYPE_NULL) -#endif - - -#ifdef CCTK_REAL_PRECISION_16 -#define CARPET_MPI_REAL CARPET_MPI_REAL16 -#elif CCTK_REAL_PRECISION_8 -#define CARPET_MPI_REAL CARPET_MPI_REAL8 -#elif CCTK_REAL_PRECISION_4 -#define CARPET_MPI_REAL CARPET_MPI_REAL4 -#endif - - -#endif - namespace CarpetIOFlexIO { // Variable definitions @@ -133,41 +70,12 @@ namespace CarpetIOFlexIO { int InputVarAs (const cGH* const cgh, const char* const varname, const char* const alias); - static const char* GetStringParameter (const char* const parametername, + const char* GetStringParameter (const char* const parametername, const char* const fallback); - int WriteGF (const cGH* const cgh, IObase* writer, AMRwriter* amrwriter, ioRequest* request, const int called_from_checkpoint); - int ReadGF (const cGH* const cgh, IObase* reader, AmrGridReader* amrreader, int currdataset); + int WriteVarAs (const cGH* const cgh, IObase* writer,AMRwriter* amrwriter, int varindex, int group); } // namespace CarpetIOFlexIO -namespace CarpetIOFlexIOUtil { - - IObase::DataType FlexIODataType (int cctk_type); - - void WriteAttribute (IObase* writer, const char* name, - int value); - void WriteAttribute (IObase* writer, const char* name, - const int* values, int nvalues); - void WriteAttribute (IObase* writer, const char* name, - CCTK_REAL value); - void WriteAttribute (IObase* writer, const char* name, - const CCTK_REAL* values, int nvalues); - void WriteAttribute (IObase* writer, const char* name, - const char* valuestring); - - void DumpCommonAttributes (const cGH *cgh, IObase* writer, ioRequest* request); -} - -namespace CarpetCheckpointRestart { - - int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from); - -} - - #endif // !defined(CARPETIOFLEXIO_HH) - - - diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn index 0e1021e99..b15bcae91 100644 --- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn +++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn @@ -1,8 +1,8 @@ # Main make.code.defn file for thorn CarpetIOFlexIO -# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn,v 1.2 2003/09/23 12:34:43 cvs_anon Exp $ +# $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/make.code.defn,v 1.1 2003/05/16 14:02:18 hawke Exp $ # Source files in this directory -SRCS = ioflexio.cc checkpointrestart.cc ioflexioUtil.cc +SRCS = ioflexio.cc checkpointrestart.cc # Subdirectories containing source files SUBDIRS = |