Getting there... Skeleton of reovery routine for data is in place.

Getting there... Skeleton of reovery routine for data is in place.
Need to modify scheduling for IOUtil_RecoverGH: OPTIONS: global

darcs-hash:20040107115756-19929-6e1501955bbe1a8811de898e2803d3e22875265d.gz

3 files changed, 169 insertions, 19 deletions

diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc
index 634d137ba..30a32641d 100644
--- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc
+++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc
@@ -48,7 +48,7 @@
 #include "ioflexio.hh"
 
 extern "C" {
-  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc,v 1.16 2004/01/06 08:56:39 cott Exp $";
+  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/checkpointrestart.cc,v 1.17 2004/01/07 12:57:56 cott Exp $";
   CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_checkpointrestart_cc);
 }
 
@@ -63,7 +63,8 @@ namespace CarpetCheckpointRestart {
   int Checkpoint (const cGH* const cgh, int called_from);
 
   int RecoverParameters (IObase* reader);
-
+  int RecoverGHextensions (cGH* cgh, IObase* reader);
+  int RecoverVariables (cGH* cgh, IObase* reader);
 
   void CarpetIOFlexIO_EvolutionCheckpoint( const cGH* const cgh){
     
@@ -177,8 +178,9 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
   int result,myproc;
   CarpetIOFlexIOGH *myGH;
   char filename[1024];
-  static IObase* reader;
-
+  
+  static IObase* reader = NULL;
+  
   DECLARE_CCTK_PARAMETERS
 
   /* to make the compiler happy */
@@ -187,15 +189,15 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
 
   myproc = CCTK_MyProc (cgh);
 
+  fprintf(stderr,"\n reflevel: %d\n",reflevel);
 
-  CCTK_VInfo (CCTK_THORNSTRING, "got this far... '%s'", basefilename);  
 
-  if (called_from == CP_RECOVER_PARAMETERS ||
-      called_from == FILEREADER_DATA ||
-      (cgh && (cgh->cctk_levfac[0] > 1 || cgh->cctk_convlevel > 0)))
+  if (called_from == CP_RECOVER_PARAMETERS)
   {
+    CCTK_VInfo (CCTK_THORNSTRING, "got this far... '%s'", basefilename);  
     if (myproc == 0){
       reader = new H5IO(basefilename,IObase::Read);
+      CCTK_VInfo (CCTK_THORNSTRING, "blah '%s'", basefilename);  
       if ( ! reader->isValid() )
 	{
 	  CCTK_VInfo(CCTK_THORNSTRING,"file is not open");
@@ -218,17 +220,49 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
     }
   }
 
+  /* Recover parameters */
+
   if (called_from == CP_RECOVER_PARAMETERS)
   {
-       CCTK_VInfo(CCTK_THORNSTRING,"called from recover parameters");
-   return (RecoverParameters (reader));
+    CCTK_VInfo(CCTK_THORNSTRING,"called from recover parameters");
+    return (RecoverParameters (reader));
   }
 
-  if (myproc == 0)
-    delete reader;
+  if (called_from == CP_RECOVER_DATA) {
+
+    BEGIN_REFLEVEL_LOOP(cgh) {
+      BEGIN_MGLEVEL_LOOP(cgh) {
+	
+	/* Recover GH extentions */
+	CCTK_INFO ("Recovering GH extensions");
+	result = RecoverGHextensions (cgh, reader);
+	
+	if (! result)
+	  {
+	    /* Recover variables */
+	    CCTK_VInfo (CCTK_THORNSTRING, "Recovering data! ");
+	    result = RecoverVariables (cgh, reader);
+	  }
+
+      } END_MGLEVEL_LOOP;
+    } END_REFLEVEL_LOOP;
+
+    if (myproc == 0)
+      delete reader;
+
+  }
 
-  CCTK_WARN (-1,"STOPSTOPSTOP2");
 
+  
+  if (called_from == CP_RECOVER_DATA)
+    {
+	  CCTK_VInfo (CCTK_THORNSTRING,
+		      "Restarting simulation at iteration %d (physical time %g)",
+		      cgh->cctk_iteration, (double) cgh->cctk_time);
+    }
+  
+  //  CCTK_WARN (-1,"STOPSTOPSTOP2");
+  
   return (result);
 }
 
@@ -283,7 +317,6 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
 		     "Is this really a Cactus IEEEIO checkpoint file ?");
 	}
       CCTK_VInfo (CCTK_THORNSTRING, "\n%s\n",parameters);
-      delete reader;
     }
 
 #ifdef CCTK_MPI
@@ -318,12 +351,80 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
     return (retval);
 
 
-    CCTK_WARN (-1,"STOPSTOPSTOP");
+    //    CCTK_WARN (-1,"STOPSTOPSTOP");
+
 
-    return 0;
   }
 
 
+  static int RecoverGHextensions (cGH *GH, IObase* reader)
+  {
+    int i, type,dim;
+    CCTK_REAL realBuffer;
+    CCTK_INT4 int4Buffer[2];
+
+    IObase::DataType datatype;
+    
+    if (CCTK_MyProc (GH) == 0)
+      {
+		
+	/* get the iteration number */
+	i = reader->readAttributeInfo ("GH$iteration", datatype, dim);
+
+
+	if (i >= 0 && datatype == FLEXIO_INT4 && dim == 1)
+	  {
+	    reader->readAttribute (i, &int4Buffer[0]);
+	  }
+	else
+	  {
+	    CCTK_WARN (1, "Unable to restore GH->cctk_iteration, defaulting to 0");
+	    int4Buffer[0] = 0;
+	  }
+
+	/* get the main loop index */
+	i = reader->readAttributeInfo ( "main loop index", datatype, dim);
+	if (i >= 0 && datatype == FLEXIO_INT4 && dim == 1)
+	  {
+	    reader->readAttribute (i, &int4Buffer[1]);
+	  }
+	else
+	  {
+	    CCTK_WARN (1, "Unable to restore main loop index, defaulting to 0");
+	    int4Buffer[1] = 0;
+	  }
+	
+	/* get cctk_time */
+	i = reader->readAttributeInfo ("GH$time", datatype, dim);
+	if (i >= 0 && datatype == FLEXIO_REAL && dim == 1)
+	  {
+	    reader->readAttribute (i, &realBuffer);
+	  }
+	else
+	  {
+	    CCTK_WARN (1, "Unable to restore GH->cctk_time, defaulting to 0.0");
+	    realBuffer = 0.0;
+	  }
+      }
+    
+#ifdef CCTK_MPI
+  /* Broadcast the GH extensions to all processors */
+  /* NOTE: We have to use MPI_COMM_WORLD here
+     because PUGH_COMM_WORLD is not yet set up at parameter recovery time.
+     We also assume that PUGH_MPI_INT4 is a compile-time defined datatype. */
+    CACTUS_MPI_ERROR (MPI_Bcast (int4Buffer, 2, CARPET_MPI_INT4, 0,MPI_COMM_WORLD));
+    CACTUS_MPI_ERROR (MPI_Bcast (&realBuffer, 1, CARPET_MPI_REAL,0,MPI_COMM_WORLD));
+#endif
+
+    GH->cctk_time = realBuffer;
+    GH->cctk_iteration = (int) int4Buffer[0];
+    CCTK_SetMainLoopIndex ((int) int4Buffer[1]);
+
+    return (0);
+}
+
+
+
 
   int DumpParams (const cGH* const cgh, int all, IObase* writer){
 
@@ -665,6 +766,14 @@ int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from)
     return 0;
   }
 
+  int RecoverVariables (cGH* cgh, IObase* reader){
+
+    CCTK_VInfo(CCTK_THORNSTRING,"Starting to recover data for refinement level %d",reflevel);
+
+
+    return 0;
+  }
+
 
 } // namespace CarpetCheckpointRestart
 
diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc
index ac9df7848..eb4558896 100644
--- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc
+++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc
@@ -45,7 +45,7 @@
 
 
 extern "C" {
-  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc,v 1.13 2003/12/10 14:49:10 cott Exp $";
+  static const char* rcsid = "$Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.cc,v 1.14 2004/01/07 12:57:56 cott Exp $";
   CCTK_FILEVERSION(Carpet_CarpetIOFlexIO_ioflexio_cc);
 }
 
@@ -56,7 +56,7 @@ namespace CarpetIOFlexIO {
   using namespace std;
   using namespace Carpet;
   using namespace CarpetIOFlexIOUtil;  
-  
+  using namespace CarpetCheckpointRestart;
   
   // Variable definitions
   //  int GHExtension;
@@ -87,7 +87,14 @@ namespace CarpetIOFlexIO {
     CCTK_RegisterIOMethodOutputVarAs (IOMethod, OutputVarAs);
     CCTK_RegisterIOMethodTimeToOutput (IOMethod, TimeToOutput);
     CCTK_RegisterIOMethodTriggerOutput (IOMethod, TriggerOutput);
+  
+    /* register the CarpetIOFlexIO recovery routine to thorn IOUtil */
+    if (IOUtil_RegisterRecover ("CarpetIOFlexIO recovery", CarpetIOFlexIO_Recover) < 0)
+      {
+	CCTK_WARN (1, "Failed to register IOFlexIO recovery routine");
+      }
     
+  
     return 0;
   }
   
diff --git a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh
index be467adfc..f8ca0b2f4 100644
--- a/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh
+++ b/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh
@@ -1,4 +1,4 @@
-// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh,v 1.8 2004/01/06 08:56:39 cott Exp $
+// $Header: /home/eschnett/C/carpet/Carpet/CarpetAttic/CarpetIOFlexIOCheckpoint/src/ioflexio.hh,v 1.9 2004/01/07 12:57:56 cott Exp $
 
 #ifndef CARPETIOFLEXIO_HH
 #define CARPETIOFLEXIO_HH
@@ -84,6 +84,32 @@
 
 #define CARPET_MPI_CHAR      MPI_CHAR
 
+/* floating point types are architecture-independent,
+   ie. a float has always 4 bytes, and a double has 8 bytes
+
+   PUGH_MPI_REAL  is used for communicating reals of the generic CCTK_REAL type
+   PUGH_MPI_REALn is used to explicitely communicate n-byte reals */
+#ifdef  CCTK_REAL4
+#define CARPET_MPI_REAL4  MPI_FLOAT
+#endif
+#ifdef  CCTK_REAL8
+#define CARPET_MPI_REAL8  MPI_DOUBLE
+#endif
+#ifdef  CCTK_REAL16
+#define CARPET_MPI_REAL16  (sizeof (CCTK_REAL16) == sizeof (long double) ?      \
+                          MPI_LONG_DOUBLE : MPI_DATATYPE_NULL)
+#endif
+
+
+#ifdef  CCTK_REAL_PRECISION_16
+#define CARPET_MPI_REAL   CARPET_MPI_REAL16
+#elif   CCTK_REAL_PRECISION_8
+#define CARPET_MPI_REAL   CARPET_MPI_REAL8
+#elif   CCTK_REAL_PRECISION_4
+#define CARPET_MPI_REAL   CARPET_MPI_REAL4
+#endif
+
+
 #endif
 
 namespace CarpetIOFlexIO {
@@ -121,6 +147,14 @@ namespace CarpetIOFlexIOUtil {
   void DumpCommonAttributes (const cGH *cgh, IObase* writer, ioRequest* request);
 
 }
+
+namespace CarpetCheckpointRestart {
+
+  int CarpetIOFlexIO_Recover (cGH* cgh, const char *basefilename, int called_from);
+
+}
+
+
 #endif // !defined(CARPETIOFLEXIO_HH)
 
 /* structure holding necessary information about a recovery file */