Timers: Move all timer-related code into a new thorn Timers

15 files changed, 1514 insertions, 0 deletions

diff --git a/Carpet/Timers/README b/Carpet/Timers/README
new file mode 100644
index 000000000..60888f344
--- /dev/null
+++ b/Carpet/Timers/README
@@ -0,0 +1,14 @@
+Cactus Code Thorn Timers
+Author(s)    : Erik Schnetter <schnetter@gmail.com>
+Maintainer(s): Erik Schnetter <schnetter@gmail.com>
+Licence      : n/a
+--------------------------------------------------------------------------
+
+1. Purpose
+
+Provide various kinds of timers:
+
+CactusTimer:    C++ wrapper around standard Cactus timers
+CactusTimerSet: knows about all CactusTimers that were ever created
+Timer:          hierarchical set of CactusTimers
+TimerTree:      helper class for Timer
diff --git a/Carpet/Timers/configuration.ccl b/Carpet/Timers/configuration.ccl
new file mode 100644
index 000000000..57a129f66
--- /dev/null
+++ b/Carpet/Timers/configuration.ccl
@@ -0,0 +1,8 @@
+# Configuration definitions for thorn Timers
+
+PROVIDES Timers
+{
+}
+
+# This is necessary for some convenient communication routines
+REQUIRES CarpetLib
diff --git a/Carpet/Timers/doc/documentation.tex b/Carpet/Timers/doc/documentation.tex
new file mode 100644
index 000000000..41d7149be
--- /dev/null
+++ b/Carpet/Timers/doc/documentation.tex
@@ -0,0 +1,144 @@
+% *======================================================================*
+%  Cactus Thorn template for ThornGuide documentation
+%  Author: Ian Kelley
+%  Date: Sun Jun 02, 2002
+%  $Header$
+%
+%  Thorn documentation in the latex file doc/documentation.tex
+%  will be included in ThornGuides built with the Cactus make system.
+%  The scripts employed by the make system automatically include
+%  pages about variables, parameters and scheduling parsed from the
+%  relevant thorn CCL files.
+%
+%  This template contains guidelines which help to assure that your
+%  documentation will be correctly added to ThornGuides. More
+%  information is available in the Cactus UsersGuide.
+%
+%  Guidelines:
+%   - Do not change anything before the line
+%       % START CACTUS THORNGUIDE",
+%     except for filling in the title, author, date, etc. fields.
+%        - Each of these fields should only be on ONE line.
+%        - Author names should be separated with a \\ or a comma.
+%   - You can define your own macros, but they must appear after
+%     the START CACTUS THORNGUIDE line, and must not redefine standard
+%     latex commands.
+%   - To avoid name clashes with other thorns, 'labels', 'citations',
+%     'references', and 'image' names should conform to the following
+%     convention:
+%       ARRANGEMENT_THORN_LABEL
+%     For example, an image wave.eps in the arrangement CactusWave and
+%     thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
+%   - Graphics should only be included using the graphicx package.
+%     More specifically, with the "\includegraphics" command.  Do
+%     not specify any graphic file extensions in your .tex file. This
+%     will allow us to create a PDF version of the ThornGuide
+%     via pdflatex.
+%   - References should be included with the latex "\bibitem" command.
+%   - Use \begin{abstract}...\end{abstract} instead of \abstract{...}
+%   - Do not use \appendix, instead include any appendices you need as
+%     standard sections.
+%   - For the benefit of our Perl scripts, and for future extensions,
+%     please use simple latex.
+%
+% *======================================================================*
+%
+% Example of including a graphic image:
+%    \begin{figure}[ht]
+% 	\begin{center}
+%    	   \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
+% 	\end{center}
+% 	\caption{Illustration of this and that}
+% 	\label{MyArrangement_MyThorn_MyLabel}
+%    \end{figure}
+%
+% Example of using a label:
+%   \label{MyArrangement_MyThorn_MyLabel}
+%
+% Example of a citation:
+%    \cite{MyArrangement_MyThorn_Author99}
+%
+% Example of including a reference
+%   \bibitem{MyArrangement_MyThorn_Author99}
+%   {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
+%   1--16. {\tt http://www.nowhere.com/}}
+%
+% *======================================================================*
+
+% If you are using CVS use this line to give version information
+% $Header$
+
+\documentclass{article}
+
+% Use the Cactus ThornGuide style file
+% (Automatically used from Cactus distribution, if you have a
+%  thorn without the Cactus Flesh download this from the Cactus
+%  homepage at www.cactuscode.org)
+\usepackage{../../../../doc/latex/cactus}
+
+\begin{document}
+
+% The author of the documentation
+\author{Erik Schnetter \textless schnetter@gmail.com\textgreater}
+
+% The title of the document (not necessarily the name of the Thorn)
+\title{Timers}
+
+% the date your document was last changed, if your document is in CVS,
+% please use:
+%    \date{$ $Date: 2004-01-07 15:12:39 -0500 (Wed, 07 Jan 2004) $ $}
+\date{August 07 2013}
+
+\maketitle
+
+% Do not delete next line
+% START CACTUS THORNGUIDE
+
+% Add all definitions used in this documentation here
+%   \def\mydef etc
+
+% Add an abstract for this thorn's documentation
+\begin{abstract}
+
+\end{abstract}
+
+% The following sections are suggestive only.
+% Remove them or add your own.
+
+\section{Introduction}
+
+\section{Physical System}
+
+\section{Numerical Implementation}
+
+\section{Using This Thorn}
+
+\subsection{Obtaining This Thorn}
+
+\subsection{Basic Usage}
+
+\subsection{Special Behaviour}
+
+\subsection{Interaction With Other Thorns}
+
+\subsection{Examples}
+
+\subsection{Support and Feedback}
+
+\section{History}
+
+\subsection{Thorn Source Code}
+
+\subsection{Thorn Documentation}
+
+\subsection{Acknowledgements}
+
+
+\begin{thebibliography}{9}
+
+\end{thebibliography}
+
+% Do not delete next line
+% END CACTUS THORNGUIDE
+
+\end{document}
diff --git a/Carpet/Timers/interface.ccl b/Carpet/Timers/interface.ccl
new file mode 100644
index 000000000..7d07d3a0c
--- /dev/null
+++ b/Carpet/Timers/interface.ccl
@@ -0,0 +1,14 @@
+# Interface definition for thorn Timers
+
+IMPLEMENTS: Timers
+
+INCLUDE HEADER: CactusTimer.hh IN CactusTimer.hh
+INCLUDE HEADER: CactusTimerSet.hh IN CactusTimerSet.hh
+INCLUDE HEADER: Timer.hh IN Timer.hh
+INCLUDE HEADER: TimerTree.hh IN TimerTree.hh
+
+
+
+# Check whether existing output files should be truncated or not
+CCTK_INT FUNCTION IO_TruncateOutputFiles(CCTK_POINTER_TO_CONST IN cctkGH)
+REQUIRES FUNCTION IO_TruncateOutputFiles
diff --git a/Carpet/Timers/param.ccl b/Carpet/Timers/param.ccl
new file mode 100644
index 000000000..2029006c9
--- /dev/null
+++ b/Carpet/Timers/param.ccl
@@ -0,0 +1,29 @@
+# Parameter definitions for thorn Timers
+
+SHARES: IO
+
+USES STRING out_dir
+
+
+
+PRIVATE:
+
+BOOLEAN verbose "Output (debug) messages when a timer is started or stopped" STEERABLE=always
+{
+} "no"
+
+STRING xml_clock "Which clock to use in the XML timer output file" STEERABLE=always
+{
+  ".*" :: "must be a legal clock name"
+} "gettimeofday"
+
+CCTK_REAL threshold_percentage "The percentage of the root timer below which timers are omitted" STEERABLE=always
+{
+  0:* :: ""
+} 1.0
+
+INT output_precision "Number of decimal places to use in standard output for timer tree" STEERABLE=always
+{
+  1:* :: "number of decimal places"
+} 1
+
diff --git a/Carpet/Timers/schedule.ccl b/Carpet/Timers/schedule.ccl
new file mode 100644
index 000000000..a7f2cbc03
--- /dev/null
+++ b/Carpet/Timers/schedule.ccl
@@ -0,0 +1,11 @@
+# Schedule definitions for thorn Timers
+
+SCHEDULE Timer_Startup AT startup BEFORE Driver_Startup
+{
+  LANG: C
+} "Prepare hierarchical timers"
+
+SCHEDULE Timer_Shutdown AT shutdown AFTER Driver_Shutdown
+{
+  LANG: C
+} "Prepare hierarchical timers"
diff --git a/Carpet/Timers/src/CactusTimer.cc b/Carpet/Timers/src/CactusTimer.cc
new file mode 100644
index 000000000..51803d435
--- /dev/null
+++ b/Carpet/Timers/src/CactusTimer.cc
@@ -0,0 +1,254 @@
+#include <cassert>
+#include <cstdio>
+#include <cstring>
+#include <iomanip>
+#include <list>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <cctk.h>
+#include <cctk_Parameters.h>
+#include <util_String.h>
+
+#if HAVE_UNISTD_H
+#  include <fcntl.h>
+#  include <unistd.h>
+#endif
+
+#include <defs.hh>
+
+#include "CactusTimer.hh"
+#include "CactusTimerSet.hh"
+
+
+
+namespace Timers {
+  
+  using namespace std;
+  
+  // Create a new Cactus timer with the given name
+  CactusTimer::CactusTimer(string timername)
+    : running(false)
+  {
+    handle = CCTK_TimerCreate(timername.c_str());
+    assert(handle >= 0);
+    
+    timerSet.add(this);
+  }
+
+  // Destroy a timer
+  CactusTimer::~CactusTimer ()
+  {
+    timerSet.remove (this);
+    check (not CCTK_TimerDestroyI (handle));
+  }
+
+  // Start the timer
+  void CactusTimer::start ()
+  {
+    msgStart ();
+//    cout << "CactusTimer::start this = " << this << endl;
+    running = true;
+    CCTK_TimerStartI (handle);
+  }
+
+  // Stop the timer
+  void CactusTimer::stop ()
+  {
+    CCTK_TimerStopI (handle);
+    running = false;
+    msgStop ();
+  }
+
+  // Reset the timer
+  void CactusTimer::reset ()
+  {
+    CCTK_TimerResetI (handle);
+  }
+
+  // Timer name
+  string CactusTimer::name () const
+  {
+    const char* const timername = CCTK_TimerName (handle);
+    assert (timername);
+    return string(timername);
+  }
+
+  double CactusTimer::getTime()
+  {
+    DECLARE_CCTK_PARAMETERS;
+
+    static cTimerData * timer = 0;
+    if (not timer) timer = CCTK_TimerCreateData ();
+    assert (timer);
+    CCTK_TimerI (handle, timer);
+
+    const bool was_running = running;
+    if (was_running) stop();
+    const cTimerVal* tv = CCTK_GetClockValue(xml_clock, timer);
+    if (was_running) start();
+    
+    if (not tv) {
+      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                 "Clock \"%s\" not found for timer #%d \"%s\"",
+                 xml_clock, handle, CCTK_TimerName(handle));
+      return -1.0;
+    }
+
+    return CCTK_TimerClockSeconds(tv);
+  }
+
+  void CactusTimer::getGlobalTime(double& avg, double& max)
+  {
+    const cGH *const cctkGH = 0;
+    
+    int ierr;
+    
+    static int op_sum = -1;
+    static int op_max = -1;
+    if (op_sum<0) op_sum = CCTK_ReductionArrayHandle("sum");
+    if (op_max<0) op_max = CCTK_ReductionArrayHandle("maximum");
+    
+    const double val = getTime();
+    const CCTK_REAL val1 = val;
+    
+    CCTK_REAL sum1;
+    ierr = CCTK_ReduceLocScalar(cctkGH, -1, op_sum,
+                                &val1, &sum1, CCTK_VARIABLE_REAL);
+    if (ierr) {
+      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                 "Error in sum reduction");
+    }
+    avg = sum1 / CCTK_nProcs(cctkGH);
+    
+    CCTK_REAL max1;
+    ierr = CCTK_ReduceLocScalar(cctkGH, -1, op_max,
+                                &val1, &max1, CCTK_VARIABLE_REAL);
+    if (ierr) {
+      CCTK_VWarn(CCTK_WARN_ALERT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                 "Error in maximum reduction");
+    }
+    max = max1;
+  }
+  
+  vector<pair<string,string> > CactusTimer::getAllTimerNames() const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    static cTimerData *timer = NULL;
+    if (not timer) timer = CCTK_TimerCreateData();
+    assert(timer);
+    
+    CCTK_TimerI(handle, timer);
+    
+    vector<pair<string,string> > names(timer->n_vals);
+    for (int i=0; i<timer->n_vals; ++i) {
+      names[i].first  = timer->vals[i].heading;
+      names[i].second = timer->vals[i].units;
+    }
+    
+    return names;
+  }
+  
+  vector<double> CactusTimer::getAllTimerValues()
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    const bool was_running = running;
+    if (was_running) stop();
+    
+    static cTimerData *timer = NULL;
+    if (not timer) timer = CCTK_TimerCreateData();
+    assert(timer);
+    
+    CCTK_TimerI(handle, timer);
+    
+    vector<double> vals(timer->n_vals);
+    for (int i=0; i<timer->n_vals; ++i) {
+      switch (timer->vals[i].type) {
+      case val_int:    vals[i] = timer->vals[i].val.i; break;
+      case val_long:   vals[i] = timer->vals[i].val.l; break;
+      case val_double: vals[i] = timer->vals[i].val.d; break;
+      default: CCTK_BUILTIN_UNREACHABLE();
+      }
+    }
+    
+    if (was_running) start();
+    
+    return vals;
+  }
+
+  // Print timer data
+  void CactusTimer::printData ()
+  {
+    const bool was_running = running;
+    if (was_running) stop();
+
+#if 0
+    check (not CCTK_TimerPrintDataI (handle, -1)); // -1 means: all clocks
+#endif
+
+    static cTimerData * timer = 0;
+    if (not timer) timer = CCTK_TimerCreateData ();
+    assert (timer);
+    CCTK_TimerI (handle, timer);
+
+    static bool firsttime = true;
+    if (firsttime) {
+      printf ("# 1: timer name");
+      for (int i=0; i<timer->n_vals; ++i) {
+        printf (" %d: %s [%s]",
+                i+2, timer->vals[i].heading, timer->vals[i].units);
+      }
+      printf ("\n");
+      firsttime = false;
+    }
+
+    printf ("%s:", name().c_str());
+    for (int i=0; i<timer->n_vals; ++i) {
+      switch (timer->vals[i].type) {
+      case val_int:
+        printf (" %d", timer->vals[i].val.i);
+        break;
+      case val_long:
+        printf (" %ld", timer->vals[i].val.l);
+        break;
+      case val_double:
+        printf (" %g", timer->vals[i].val.d);
+        break;
+      case val_none:
+        break;
+      default:
+        assert (0);
+      }
+    }
+    printf ("\n");
+
+    if (was_running) start();
+  }
+
+  // Output (debug) messages that a timer is starting or stopping
+  void CactusTimer::msgStart () const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    if (verbose) {
+      CCTK_VInfo (CCTK_THORNSTRING, "Timer \"%s\" starting", name().c_str());
+    }
+  }
+
+  void CactusTimer::msgStop () const
+  {
+    DECLARE_CCTK_PARAMETERS;
+    if (verbose) {
+      CCTK_VInfo (CCTK_THORNSTRING, "Timer \"%s\" stopping", name().c_str());
+    }
+  }
+
+  ostream& CactusTimer::serialise(ostream &os)
+  {
+    os << scientific << setprecision(19) << getTime() << " " << name();
+    return os;
+  }
+
+}  // namespace Carpet
diff --git a/Carpet/Timers/src/CactusTimer.hh b/Carpet/Timers/src/CactusTimer.hh
new file mode 100644
index 000000000..c198d9014
--- /dev/null
+++ b/Carpet/Timers/src/CactusTimer.hh
@@ -0,0 +1,69 @@
+#ifndef CACTUSTIMER_HH
+#define CACTUSTIMER_HH
+
+#include <cctk.h>
+
+#include <iostream>
+#include <list>
+#include <string>
+#include <utility>
+#include <vector>
+
+
+
+namespace Timers {
+
+  /** The CactusTimer class wraps the Cactus timer mechanism. All
+      times are returned as doubles for now. */
+  
+  class CactusTimer {
+    int handle;
+    bool running;
+    
+  public:
+    
+    /// Create a new Cactus timer with the given name
+    CactusTimer(std::string timername);
+    
+    /// Destruct the timer
+    ~CactusTimer();
+    
+    /// Start the timer
+    void start();
+    
+    /// Stop the timer
+    void stop();
+    
+    /// Reset the timer
+    void reset();
+
+    /// Timer name
+    std::string name() const;
+    
+    /// Return the current time of the timer in seconds as a double
+    double getTime();
+    
+    /// Return the average and maximum current time over all MPI processes
+    void getGlobalTime(double& avg, double& max);
+    
+    /// Return all clock names and their units
+    std::vector<std::pair<std::string,std::string> > getAllTimerNames() const;
+    
+    /// Return all clock values of the timer as double
+    std::vector<double> getAllTimerValues();
+    
+    /// Print timer data
+    void printData ();
+    
+    std::ostream& serialise(std::ostream &os);
+    
+  private:
+    
+    // Output (debug) messages that a timer is starting or stopping
+    void msgStart() const;
+    void msgStop() const;
+  };
+  
+} // namespace Timers
+
+#endif // CACTUSTIMER_HH
diff --git a/Carpet/Timers/src/CactusTimerSet.cc b/Carpet/Timers/src/CactusTimerSet.cc
new file mode 100644
index 000000000..54eed2f70
--- /dev/null
+++ b/Carpet/Timers/src/CactusTimerSet.cc
@@ -0,0 +1,179 @@
+#include <cassert>
+#include <cstdio>
+#include <cstring>
+#include <list>
+
+#include <cctk.h>
+#include <cctk_Parameters.h>
+#include <util_String.h>
+
+#if HAVE_UNISTD_H
+#  include <fcntl.h>
+#  include <unistd.h>
+#endif
+
+#include <defs.hh>
+
+#include <CactusTimer.hh>
+#include <CactusTimerSet.hh>
+#include <Timer.hh>
+#include <TimerTree.hh>
+
+
+
+namespace Timers {
+  
+  using namespace std;
+  
+  // A global timer set
+  CactusTimerSet timerSet;
+
+  // Add a timer
+  void CactusTimerSet::add(CactusTimer* const timer)
+  {
+    timers.insert(timer);
+  }
+  
+  // Remove a timer
+  void CactusTimerSet::remove(CactusTimer* const timer)
+  {
+    timers.erase(timer);
+  }
+  
+  // Print all timer names
+  void CactusTimerSet::printNames() const
+  {
+    printf("Timer names:\n");
+    int n = 0;
+    for (timers_t::const_iterator
+           itimer = timers.begin(); itimer != timers.end(); ++itimer)
+    {
+      printf("   [%4d] %s\n", n, (*itimer)->name().c_str());
+      ++n;
+    }
+  }
+  
+  // Print all timer data
+  void CactusTimerSet::printData()
+  {
+    for (timers_t::const_iterator
+           itimer = timers.begin(); itimer != timers.end(); ++itimer)
+    {
+      (*itimer)->printData();
+    }
+    printf("\n");
+  }
+
+  // Print all timer data
+  void CactusTimerSet::writeData(const cGH* const cctkGH,
+                                 const char* const filename)
+  {
+    const int oldfd = redirect(cctkGH, filename);
+#if 0
+    printf("********************************************************************************\n");
+#endif
+    printf("# Carpet timing information at iteration %d time %g:\n",
+           cctkGH->cctk_iteration, double(cctkGH->cctk_time));
+    timerSet.printData();
+    unredirect(oldfd);
+  }
+  
+  // If filename is not empty, then redirect stdout to a file
+  int CactusTimerSet::redirect(const cGH* const cctkGH,
+                               const char* const filename)
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (CCTK_EQUALS (filename, "")) {
+      return -1;
+    }
+    
+#ifndef HAVE_UNISTD_H
+    CCTK_WARN(CCTK_WARN_ALERT,
+              "Cannot redirect timer output to a file; the operating system does not support this");
+    return -1;
+#else
+    
+    const int myproc = CCTK_MyProc(cctkGH);
+    char fullname[10000];
+    Util_snprintf(fullname, sizeof fullname,
+                  "%s/%s.%04d.txt", out_dir, filename, myproc);
+    
+    int flags = O_WRONLY | O_CREAT | O_APPEND; // append
+    static bool first_time = true;
+    if (first_time) {
+      first_time = false;
+      if (IO_TruncateOutputFiles(cctkGH)) {
+        flags = O_WRONLY | O_CREAT | O_TRUNC; // truncate
+      }
+    }
+    
+    // Temporarily redirect stdout
+    fflush(stdout);
+    const int oldfd = dup(1);   // fd 1 is stdout
+    const int mode = 0644;      // rw-r--r--, or a+r u+w
+    const int fdfile = open(fullname, flags, mode);
+    if (fdfile < 0) {
+      CCTK_VWarn(CCTK_WARN_ABORT, __LINE__, __FILE__, CCTK_THORNSTRING,
+                 "Could not open timer output file \"%s\"", fullname);
+      close(oldfd);
+      return -1;
+    }
+    // close(1);
+    // const int fd = dup(fdfile); // dup to 1, i.e., stdout again
+    const int fd = dup2(fdfile, 1); // dup to 1, i.e., stdout again
+    assert(fd == 1);
+    close(fdfile);
+    return oldfd;
+#endif
+  }
+  
+  // Redirect stdout back
+  void CactusTimerSet::unredirect(const int oldfd)
+  {
+    if (oldfd < 0) return;
+    
+#ifdef HAVE_UNISTD_H
+    fflush (stdout);
+    // close(1);
+    // const int fd = dup(oldfd);
+    const int fd = dup2(oldfd, 1);
+    if (not (fd == 1)) {
+      fprintf(stderr, "oldfd=%d fd=%d\n", oldfd, fd);
+    }
+    assert(fd == 1);
+    close(oldfd);
+#endif
+  }
+  
+
+#if 0
+  /// Reduce each timer in the set across all processes and update
+  /// each timer with the reduction information.
+  void CactusTimerSet::reduce()
+  {
+    // Collect timer names that each process has
+    
+    // Construct union of all timer names, sort canonically and assign
+    // integer identifiers
+    
+    // For each timer, identify which processes have that timer
+    
+    // Reduce the timer across all those processes (return to root proc only)
+    
+    serialise(cout);
+  }
+  
+  ostream& CactusTimerSet::serialise(ostream &os)
+  {
+    for (timers_t::const_iterator
+           itimer = timers.begin(); itimer != timers.end(); ++itimer)
+    {
+      (*itimer)->serialise(os);
+      os << endl;
+    }
+    return os;
+  }
+#endif
+  
+} // namespace Carpet
diff --git a/Carpet/Timers/src/CactusTimerSet.hh b/Carpet/Timers/src/CactusTimerSet.hh
new file mode 100644
index 000000000..c53fa5bc3
--- /dev/null
+++ b/Carpet/Timers/src/CactusTimerSet.hh
@@ -0,0 +1,62 @@
+#ifndef CACTUSTIMERSET_HH
+#define CACTUSTIMERSET_HH
+
+#include <iostream>
+#include <set>
+
+#include <cctk.h>
+#include "CactusTimer.hh"
+
+
+
+namespace Timers {
+  
+  class CactusTimerSet;
+  extern CactusTimerSet timerSet;
+  
+  using namespace std;
+  
+  // A set of timers
+  class CactusTimerSet {
+    
+    typedef set<CactusTimer*> timers_t;
+    timers_t timers;
+    
+  public:
+    
+    // Add a timer
+    void add(CactusTimer* timer);
+    
+    // Remove a timer
+    void remove(CactusTimer* timer);
+
+    // Print all timer names
+    void printNames() const;
+    
+    // Print all timer data
+    void printData();
+    
+    // Write all timer data of the global timer set to a file
+    static void writeData(cGH const* cctkGH, char const* filename);
+    
+#if 0
+    // Reduce each timer in the set across all processes and update
+    // each timer with the reduction information.
+    void reduce();
+    
+    ostream& serialise(ostream &os);
+#endif
+    
+  private:
+    
+    // If filename is not empty, then redirect stdout to a file,
+    // returning the old stdout file descriptor
+    static int redirect(cGH const* cctkGH, char const* filename);
+    
+    // Redirect stdout back
+    static void unredirect(int oldfd);
+  }; // class CactusTimerSet
+  
+} // namespace Timers
+
+#endif // CACTUSTIMERSET_HH
diff --git a/Carpet/Timers/src/Timer.cc b/Carpet/Timers/src/Timer.cc
new file mode 100644
index 000000000..50631c374
--- /dev/null
+++ b/Carpet/Timers/src/Timer.cc
@@ -0,0 +1,135 @@
+#include <cassert>
+
+#include <cctk.h>
+#include <cctk_Parameters.h>
+
+#include <Timer.hh>
+#include <TimerTree.hh>
+
+
+
+namespace Timers {
+  
+  using namespace std;
+  
+  /*********************************************************************
+   Timer
+  *********************************************************************/
+  
+  
+  
+  TimerTree main_timer_tree;
+  TimerTree mode_timer_tree;
+  
+  
+  
+  /// Create a timer with a given name, but do not start it, and do
+  /// not associate it with a point in the timer hierarchy.
+  Timer::Timer(string name_p, int tree):
+    d_name(name_p), d_tree(tree==0 ? &main_timer_tree : &mode_timer_tree)
+  {
+  }
+  
+  /// Destroy the timer
+  Timer::~Timer()
+  {
+  }
+  
+  /// Insert the timer into the tree of timers as a child of the most
+  /// recently started timer that has not been stopped. Don't start
+  /// the timer. This routine ensures a timer is created even if it is
+  /// never started.
+  void Timer::instantiate()
+  {
+    TimerNode *current_timer = d_tree->current;
+    if (not d_tree->root) return; // do nothing if there is no root
+    assert(current_timer);
+    current_timer->getChildTimer(name())->instantiate();
+  }
+  
+  /// Start the timer and insert it into the tree of timers as a child
+  /// of the most recently started timer that has not been stopped.
+  void Timer::start()
+  {
+    TimerNode *current_timer = d_tree->current;
+    if (not d_tree->root) return; // do nothing if there is no root
+    assert(current_timer);
+    current_timer->getChildTimer(name())->start();
+  }
+  
+  /// Stop the timer - it must be the most recently started timer
+  void Timer::stop()
+  {
+    TimerNode *current = d_tree->current;
+    if (not d_tree->root) return; // do nothing if there is no root
+    if (current->getName() != name())
+      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Trying to stop enclosing timer '%s' before enclosed time '%s'",
+                  name().c_str(), current->getName().c_str());
+    current->stop();
+  }
+  
+  /// Return the name of the timer
+  string Timer::name() const
+  {
+    return d_name;
+  }
+  
+  /// Return the current time of the timer as a double
+  double Timer::getTime()
+  {
+    return d_tree->current->getTime();
+  }
+  
+  void Timer::outputTree(string name)
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    TimerNode *tt = main_timer_tree.root->getChildTimer(name.c_str());
+    double total_avg, total_max;
+    tt->getGlobalTime(total_avg, total_max);
+    tt->print
+      (cout, total_max, 0, threshold_percentage, output_precision);
+    mode_timer_tree.root->getGlobalTime(total_avg, total_max);
+    mode_timer_tree.root->print
+      (cout, total_max, 0, threshold_percentage, output_precision);
+  }
+  
+  void Timer::outputTreeXML()
+  {
+    DECLARE_CCTK_PARAMETERS;
+    
+    main_timer_tree.root->outputXML(out_dir, CCTK_MyProc(0));
+  }
+  
+  
+  
+  extern "C"
+  int Timer_Startup()
+  {
+    // This must happen before any Timer objects are created
+    main_timer_tree.root = new TimerNode(&main_timer_tree, "main");
+    main_timer_tree.current = 0; // No timer has been started yet
+    main_timer_tree.root->start();
+    
+    mode_timer_tree.root = new TimerNode(&mode_timer_tree, "meta mode");
+    mode_timer_tree.current = 0; // No timer has been started yet
+    mode_timer_tree.root->start();
+    
+    return 0;
+  }
+  
+  extern "C"
+  int Timer_Shutdown()
+  {
+    // main_timer_tree.root->stop();
+    // mode_timer_tree.root->stop();
+    
+    // Delete timer tree
+    delete main_timer_tree.root; main_timer_tree.root = 0;
+    delete mode_timer_tree.root; mode_timer_tree.root = 0;
+    
+    return 0;
+  }
+  
+} // namespace Timers
diff --git a/Carpet/Timers/src/Timer.hh b/Carpet/Timers/src/Timer.hh
new file mode 100644
index 000000000..651659fec
--- /dev/null
+++ b/Carpet/Timers/src/Timer.hh
@@ -0,0 +1,89 @@
+#ifndef TIMER_HH
+#define TIMER_HH
+
+#include <iostream>
+#include <list>
+
+#include <cctk.h>
+
+#include <TimerTree.hh>
+
+
+
+namespace Timers {
+  
+  /**
+     This class allows the user to instrument their code with named
+     timers which can then be later queried to determine the amount of
+     time spent in the code between "start" and "end" calls. The
+     sequence of start and end calls of different timers determines a
+     dynamical hierarchical tree structure implemented by the
+     TimerNode class.
+     
+     To use this class, create a timer object with a particular name:
+     
+        Timer timer("MyTimer")
+     
+     Now wrap the code to be timed with start() and stop() calls:
+     
+        timer.start()
+        
+     some code
+        
+        timer.stop()
+
+     You can start and stop a timer multiple times. The timer will be
+     created as a child of whatever timer is current (i.e. has been
+     started and not stopped) at the time of the first start() call.
+     Any timers which are started between the start() and stop(),
+     whether or not they are in the same file, will be stored as
+     children of this timer.
+     
+     Timer objects must be started and stopped in a non-overlapping
+     manner. Specifically, a timer cannot be stopped if it is not the
+     most recently started timer. Doing so will generate an error.
+     
+     Timer objects can be allocated as "static" or not - it does not
+     matter.
+  */
+  
+  class Timer {
+    
+  public:
+    
+    Timer(std::string name, int tree=0);
+    ~Timer();
+
+    void instantiate();
+    void start();
+    void stop();
+    std::string name() const;
+    double getTime();
+    
+    static void outputTree(std::string name);
+    static void outputTreeXML();
+    
+  private:
+    
+    std::string d_name;
+    TimerTree *d_tree;
+  };
+  
+} // namespace Timers
+
+
+
+// Macros for using timers in a convenient manner
+  
+#define TIMING_BEGIN(name)                      \
+  do {                                          \
+    static Timers::timer(name);                 \
+    timer.start();                              \
+    {
+
+#define TIMING_END                              \
+    }                                           \
+    timer.stop();                               \
+  } while (0)
+
+#endif // TIMER_HH
diff --git a/Carpet/Timers/src/TimerTree.cc b/Carpet/Timers/src/TimerTree.cc
new file mode 100644
index 000000000..e75281bf7
--- /dev/null
+++ b/Carpet/Timers/src/TimerTree.cc
@@ -0,0 +1,378 @@
+/*
+ *
+ * The MIT License
+ *
+ * Copyright (c) 1997-2010 Center for the Simulation of Accidental Fires and
+ * Explosions (CSAFE), and  Scientific Computing and Imaging Institute (SCI),
+ * University of Utah.
+ *
+ * License for the specific language governing rights and limitations under
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * */
+
+#include <algorithm>
+#include <cassert>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <ostream>
+#include <string>
+#include <utility>
+
+#include <dist.hh>
+#include <mpi_string.hh>
+
+#include "TimerTree.hh"
+
+
+
+namespace Timers {
+
+  using namespace std;
+
+  TimerNode::TimerNode(TimerTree *tree, const string& name):
+    d_name(name), d_parent(0), d_tree(tree), d_running(0), d_timer(0)
+  {
+  }
+
+  TimerNode::~TimerNode()
+  {
+    for (map<string,TimerNode*>::iterator
+           iter=d_children.begin(); iter!=d_children.end(); ++iter)
+    {
+      delete iter->second;
+    }
+    d_children.clear();
+    delete d_timer;
+  }
+
+  string TimerNode::pathName() const
+  {
+    assert(d_parent != this);
+    if (d_parent)
+      return d_parent->pathName() + string("/") + getName();
+    else
+      return getName();
+  }
+
+  void TimerNode::instantiate()
+  {
+    assert(!d_running);
+    d_parent = d_tree->current;
+    d_tree->current = this;
+    if (!d_timer)
+      d_timer = new CactusTimer(pathName());
+    d_tree->current = d_parent;
+  }
+
+  void TimerNode::start()
+  {
+    assert(!d_running);
+    
+    d_running = true;
+    d_parent = d_tree->current;
+    d_tree->current = this;
+    if (!d_timer)
+      d_timer = new CactusTimer(pathName());
+    assert(d_timer);
+    d_timer->start();
+  }
+
+  void TimerNode::stop()
+  {
+    assert(d_running);
+    
+    // A timer can only be stopped if it is the current timer
+    if (this != d_tree->current)
+      CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
+                  "Tried to stop non-current timer '%s'", getName().c_str());
+    
+    d_timer->stop();
+    
+    d_running = false;
+    d_tree->current = d_parent;
+  }
+  
+  /// Get the name of the timer
+  string TimerNode::getName() const
+  {
+    assert(not d_name.empty());
+    return d_name;
+  }
+  
+  /// Determine if the timer is running
+  bool TimerNode::isRunning() const
+  {
+    return d_running;
+  }
+  
+  /// Find the child timer that matches the name provided.  If it is
+  /// not found then a new timer with that name is allocated.
+  TimerNode* TimerNode::getChildTimer(const string& name)
+  {
+    // Find child
+    TimerNode *child = d_children[name];
+    
+    // If the pointer is null then allocate it
+    if (not child)
+      d_children[name] = child = new TimerNode(d_tree, name);
+    
+    return child;
+  }
+  
+  /// Get the time measured by this timer
+  double TimerNode::getTime()
+  {
+    return d_timer->getTime();
+  }
+  
+  /// Get the global time measured by this timer
+  void TimerNode::getGlobalTime(double& avg, double& max)
+  {
+    return d_timer->getGlobalTime(avg, max);
+  }
+  
+  /// Get the names of all clocks of this timer
+  vector<pair<string,string> > TimerNode::getAllTimerNames() const
+  {
+    return d_timer->getAllTimerNames();
+  }
+  
+  /// Get the values of all clocks of this timer
+  vector<double> TimerNode::getAllTimerValues()
+  {
+    return d_timer->getAllTimerValues();
+  }
+  
+  /// Print this node and its children as an ASCII tree
+  void TimerNode::print(ostream& out,
+                        double total, int level, double threshold,
+                        int precision)
+  {
+    string space;
+    
+    // Compute the level of indentation for this depth
+    for (int i=0;i<level-1;i++)
+      space += "| ";
+    
+    if (level != 0)
+      space += "|_";
+    
+    const int pcw = 6;
+    const int tw = 8;
+    const int tnw = 40;         // timer name
+    const int vw = 9;           // clock values
+    const streamsize oldprecision = out.precision();
+    const ios_base::fmtflags oldflags = out.flags();
+    
+    // const double t = getTime();
+    double tavg, tmax;
+    getGlobalTime(tavg, tmax);
+    const vector<double> values = getAllTimerValues();
+    const string hyphens = string(precision-1, '-');
+    const string spaces  = string(precision-1, ' ');
+    
+    if (level == 0) {
+      const vector<pair<string,string> > names = getAllTimerNames();
+      
+      out << "--------" << hyphens
+          << "--------" << hyphens
+          << "--------" << hyphens
+          << "--" << string(tnw, '-');
+      for (size_t i=0; i<values.size(); ++i) {
+        out << "--" << string(vw, '-');
+      }
+      out << "\n";
+      
+      // timer names
+      out << "Time    " << spaces
+          << "  Time  " << spaces
+          << " Imblnc " << spaces
+          << "  " << setw(tnw) << left << "Timer" << right;
+      for (size_t i=0; i<names.size(); ++i) {
+        out << "  " << setw(vw) << names[i].first.substr(0,vw);
+      }
+      out << "\n";
+      
+      // timer units
+      out << "percent " << spaces
+          << "  secs  " << spaces
+          << " percent" << spaces
+          << "  " << setw(tnw) << "     ";
+      for (size_t i=0; i<names.size(); ++i) {
+        out << "  " << setw(vw) << names[i].second.substr(0,vw);
+      }
+      out << "\n";
+      
+      out << "--------" << hyphens
+          << "--------" << hyphens
+          << "--------" << hyphens
+          << "--" << string(tnw, '-');
+      for (size_t i=0; i<values.size(); ++i) {
+        out << "--" << string(vw, '-');
+      }
+      out << "\n";
+    }
+    
+    // Print this timer value
+    out << fixed << setw(pcw) << setprecision(precision)
+        << 100.0 * tavg / total << "%"
+        << " " << fixed << setw(tw) << setprecision(precision) << tavg
+        << " " << fixed << setw(pcw) << setprecision(precision)
+        << 100.0 * (1.0 - tavg / tmax) << "%"
+        << "  " << space << setw(max(size_t(0), tnw - space.length())) << left
+        << d_name.substr(0, max(size_t(10), tnw - space.length())) << right;
+    for (size_t i=0; i<values.size(); ++i) {
+      out.unsetf(ios_base::floatfield);
+      out << "  " << setw(vw) << setprecision(vw-5) << values[i];
+    }
+    out << "\n";
+
+    // TODO: Don't call getGlobalTime for all timers separately.
+    // Instead, call a single function that takes a snapshot of all
+    // timers, and reduces these snapshots to average and maximum.
+    // While printing, access these snapshots.
+    
+    //double children_time = 0;
+    double children_tavg = 0.0;
+    bool printed_children = false;
+    
+    // Recursively print the children
+    for (map<string,TimerNode*>::iterator iter = d_children.begin();
+         iter != d_children.end(); iter++)
+    {
+      const string timername = iter->first;
+      const string root_timername =
+        CarpetLib::broadcast_string(dist::comm(), 0, timername);
+      if (timername != root_timername) {
+        CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
+                   "Timers are inconsistent across processes: root process expects timer %s, this process has timer %s instead",
+                    root_timername.c_str(), timername.c_str());
+      }
+      double child_avg, child_max;
+      iter->second->getGlobalTime(child_avg, child_max);
+      if (child_max * 100.0 / total > threshold) {
+        iter->second->print(out,total,level+1,threshold,precision);
+        printed_children = true;
+      }
+      //children_time += iter->second->getTime();
+      children_tavg += child_avg;
+    }
+    {
+      const string timername = "[done]";
+      const string root_timername =
+        CarpetLib::broadcast_string(dist::comm(), 0, timername);
+      if (timername != root_timername) {
+        CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
+                    "Timers are inconsistent across processes: root process expects timer %s, this process has timer %s instead",
+                    root_timername.c_str(), timername.c_str());
+      }
+    }
+    
+    if (d_children.size() > 0 && printed_children) {
+      //const double untimed = t - children_time;
+      const double untimed = tavg - children_tavg;
+      
+      if (100.0 * untimed / total > threshold) {
+        // Print the untimed portion
+        out << fixed << setw(pcw) << setprecision(1)
+            << 100.0 * untimed / total << "%"
+            << " " << fixed << setw(tw) << setprecision(1) << untimed
+            << "        "
+            << "  | " << space << "untimed" << "\n";
+      }
+    }
+    out.precision (oldprecision);
+    out.setf (oldflags);
+    
+    if (level == 0) {
+      out << "--------" << hyphens
+          << "--------" << hyphens
+          << "--------" << hyphens
+          << "--" << string(tnw, '-');
+      for (size_t i=0; i<values.size(); ++i) {
+        out << "--" << string(vw, '-');
+      }
+      out << "\n";
+    }
+  }
+  
+  void TimerNode::outputXML(const string& out_dir, int proc)
+  {
+    ostringstream filenamebuf;
+    filenamebuf << out_dir << "/timertree." << proc << ".xml";
+    string filenamestr = filenamebuf.str();
+    const char* filename = filenamestr.c_str();
+    ofstream file;
+    file.open (filename, ios::out | ios::trunc);
+    
+    printXML(file, 0);
+    
+    file.close();
+    assert (file.good());
+  }
+  
+  /// Print this node and its children as an XML file
+  void TimerNode::printXML(ostream& out, int level)
+  {
+    string space;
+    
+    // Compute the level of indentation for this node
+    for (int i=0;i<level;i++)
+      space += "  ";
+    
+    out << space << "<timer name = " << "\"" << escapeForXML(d_name) << "\"> ";
+    out << getTime() << " ";
+    
+    // For compactness, only use multiple lines if there are children
+    if (d_children.size() != 0)
+    {
+      out << "\n";
+      
+      // Recursively print the children
+      for (map<string,TimerNode*>::iterator
+             iter=d_children.begin(); iter!=d_children.end(); ++iter)
+        iter->second->printXML(out,level+1);
+      out << space;
+    }
+    
+    out << "</timer>" << "\n";
+  }
+  
+  /// Make a string suitable for inclusion in an XML file
+  string TimerNode::escapeForXML(const string& s) const
+  {
+    ostringstream res;
+    for (string::const_iterator si=s.begin(); si!=s.end(); ++si) {
+      switch (*si) {
+      case '\'': res << "&apos;"; break;
+      case '"': res << "&quot;"; break;
+      case '&': res << "&amp;"; break;
+      case '<': res << "&lt;"; break;
+      case '>': res << "&gt;"; break;
+      default: res << *si; break;
+      }
+    }    
+    
+    return res.str();
+  }
+  
+} // namespace Timers
diff --git a/Carpet/Timers/src/TimerTree.hh b/Carpet/Timers/src/TimerTree.hh
new file mode 100644
index 000000000..55d61a0f8
--- /dev/null
+++ b/Carpet/Timers/src/TimerTree.hh
@@ -0,0 +1,121 @@
+/*
+ *
+ * The MIT License
+ *
+ * Copyright (c) 1997-2010 Center for the Simulation of Accidental Fires and
+ * Explosions (CSAFE), and  Scientific Computing and Imaging Institute (SCI),
+ * University of Utah.
+ *
+ * License for the specific language governing rights and limitations under
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * */
+
+/* This class was originally written by Justin Luitjens and
+   subsequently integrated with Cactus/Carpet by Ian Hinder and
+   heavily modified. */
+
+
+
+#ifndef TIMERTREE_HH
+#define TIMERTREE_HH
+
+#include <cassert>
+#include <iostream>
+#include <list>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "CactusTimer.hh"
+
+
+
+namespace Timers {
+  
+  class TimerNode;
+  
+  class TimerTree {
+  public:
+    TimerTree(): root(0), current(0)
+    {
+    }
+    TimerNode *root;
+    TimerNode *current;
+  };
+  
+  
+  
+  /**
+     The TimerNode class implements a tree structure where each node
+     represents a timer, implemented as a CactusTimer. Each node of
+     the tree can have zero of more children, where the names of the
+     child nodes are unique within a single parent, but not
+     necessarily unique within the entire tree. A tree formed of
+     TimerNode objects represents the execution profile of the
+     program, inasmuch as it is instrumented by timers.
+     
+     Child nodes of a given name are accessed using the getChildTimer
+     method, where a node is created with the given name if none
+     exists already. This ensures that the names of the child timers
+     are unique.
+  */
+  
+  class TimerNode {
+  public:
+    TimerNode(TimerTree *root, const std::string& name);
+    ~TimerNode();
+    
+    void instantiate();
+    void start();
+    void stop();
+    
+    std::string getName() const;
+    std::string pathName() const;
+    
+    // Find the child timer that matches the name provided. If it is
+    // not found then that timer is allocated.
+    TimerNode* getChildTimer(const std::string& name);
+    
+    double getTime();
+    void getGlobalTime(double& avg, double &max);
+    std::vector<std::pair<std::string, std::string> > getAllTimerNames() const;
+    std::vector<double> getAllTimerValues();
+    bool isRunning() const;
+    
+    void print(std::ostream& out,
+               double total, int level=0, double threshold=0.0,
+               int precision=1);
+    void printXML(std::ostream& out, int level=0);
+    void outputXML(const std::string& out_dir, int proc);
+    
+  private:
+    std::string escapeForXML(const std::string& s) const;
+    
+    std::string d_name;
+    std::map<std::string,TimerNode*> d_children;
+    TimerNode *d_parent;
+    TimerTree *d_tree;
+    bool d_running;
+    CactusTimer *d_timer;
+  };
+  
+} // namespace Timers
+
+#endif // TIMERTREE_HH
diff --git a/Carpet/Timers/src/make.code.defn b/Carpet/Timers/src/make.code.defn
new file mode 100644
index 000000000..4779d34ae
--- /dev/null
+++ b/Carpet/Timers/src/make.code.defn
@@ -0,0 +1,7 @@
+# Main make.code.defn file for thorn Timers
+
+# Source files in this directory
+SRCS = CactusTimer.cc CactusTimerSet.cc Timer.cc TimerTree.cc
+
+# Subdirectories containing source files
+SUBDIRS =