CarpetLib: Improve timers

Implement a second timer based on Intel's rdtsc instruction, which is
much faster and much more accurate than MPI_Wtime.

Place the timer classes into the CarpetLib namespace.

Create a TimerSet class.  Make the Timer class automatically register
all timers with a singleton object, removing all global variables.

darcs-hash:20070203211128-dae7b-42765e79446eda6a2337ba22cd390869055c555a.gz

7 files changed, 629 insertions, 388 deletions

diff --git a/Carpet/CarpetLib/param.ccl b/Carpet/CarpetLib/param.ccl
index a3000fe74..a81275309 100644
--- a/Carpet/CarpetLib/param.ccl
+++ b/Carpet/CarpetLib/param.ccl
@@ -54,7 +54,18 @@ STRING timestat_file "File name in which timestat output is collected (because s
 {
   "^$"   :: "empty filename: no file output"
   "^.+$" :: "file name"
-} "carpetlib-time-statistics"
+} "carpetlib-timing-statistics"
+
+KEYWORD timestat_timer "Timer for timing statistics" STEERABLE=recover
+{
+  "MPI_Wtime" :: "MPI_Wtime (expensive; uses system call)"
+  "rdtsc"     :: "Intel rdtsc instruction (exists only on Intel CPUs)"
+  "none"      :: "don't measure time"
+} "MPI_Wtime"
+
+BOOLEAN timestat_disable "Disable timing statistics" STEERABLE=always
+{
+} "no"
 
 
 
diff --git a/Carpet/CarpetLib/src/commstate.cc b/Carpet/CarpetLib/src/commstate.cc
index 9eecec8f2..b770a706a 100644
--- a/Carpet/CarpetLib/src/commstate.cc
+++ b/Carpet/CarpetLib/src/commstate.cc
@@ -15,6 +15,7 @@
 
 
 using namespace std;
+using namespace CarpetLib;
 
 
 
@@ -36,6 +37,8 @@ comm_state::comm_state ()
 
   DECLARE_CCTK_PARAMETERS;
 
+  static Timer timer ("commstate::create");
+  timer.start ();
   thestate = use_collective_communication_buffers ?
              state_get_buffer_sizes : state_post;
 
@@ -57,12 +60,16 @@ comm_state::comm_state ()
   recvbuffers_ready.resize (dist::c_ndatatypes() * dist::size());
   srequests.resize (dist::c_ndatatypes() * dist::size(), MPI_REQUEST_NULL);
   rrequests.resize (dist::c_ndatatypes() * dist::size(), MPI_REQUEST_NULL);
+  
+  timer.stop (0);
 }
 
 
 void comm_state::step ()
 {
   DECLARE_CCTK_PARAMETERS;
+  static Timer total ("commstate::step");
+  total.start ();
   assert (thestate != state_done);
   switch (thestate) {
     case state_post:
@@ -108,7 +115,8 @@ void comm_state::step ()
           procbuf.recvbuf = procbuf.recvbufbase;
 
           if (procbuf.recvbufsize > 0) {
-            wtime_commstate_sizes_irecv.start();
+            static Timer timer ("commstate_sizes_irecv");
+            timer.start ();
             int const tag =
               vary_tags
               ? (dist::rank() + dist::size() * (proc + dist::size() * type)) % 32768
@@ -116,7 +124,7 @@ void comm_state::step ()
             MPI_Irecv (procbuf.recvbufbase, procbuf.recvbufsize,
                        typebufs[type].mpi_datatype, proc, tag,
                        dist::comm(), &rrequests[dist::size()*type + proc]);
-            wtime_commstate_sizes_irecv.stop();
+            timer.stop (0);
             num_posted_recvs++;
           }
         }
@@ -163,27 +171,30 @@ void comm_state::step ()
                 : type;
               if (use_mpi_send) {
                 // use MPI_Send
-                wtime_commstate_send.start();
+                static Timer timer ("commstate_send");
+                timer.start ();
                 MPI_Send (procbuf.sendbufbase, procbuf.sendbufsize,
                           typebufs[type].mpi_datatype, proc, tag,
                           dist::comm());
                 srequests[dist::size()*type + proc] = MPI_REQUEST_NULL;
-                wtime_commstate_send.stop(procbuf.sendbufsize * datatypesize);
+                timer.stop (procbuf.sendbufsize * datatypesize);
               } else if (use_mpi_ssend) {
                 // use MPI_Ssend
-                wtime_commstate_ssend.start();
+                static Timer timer ("commstate_ssend");
+                timer.start ();
                 MPI_Ssend (procbuf.sendbufbase, procbuf.sendbufsize,
                            typebufs[type].mpi_datatype, proc, tag,
                            dist::comm());
                 srequests[dist::size()*type + proc] = MPI_REQUEST_NULL;
-                wtime_commstate_ssend.stop(procbuf.sendbufsize * datatypesize);
+                timer.stop (procbuf.sendbufsize * datatypesize);
               } else {
                 // use MPI_Isend
-                wtime_commstate_isend.start();
+                static Timer timer ("commstate_isend");
+                timer.start ();
                 MPI_Isend (procbuf.sendbufbase, procbuf.sendbufsize,
                            typebufs[type].mpi_datatype, proc, tag,
                            dist::comm(), &srequests[dist::size()*type + proc]);
-                wtime_commstate_isend.stop(procbuf.sendbufsize * datatypesize);
+                timer.stop (procbuf.sendbufsize * datatypesize);
               }
             }
           
@@ -216,6 +227,7 @@ void comm_state::step ()
     default:
       assert (0 && "invalid state");
   }
+  total.stop (0);
 }
 
 
@@ -266,13 +278,15 @@ bool comm_state::AllPostedCommunicationsFinished ()
         }
       }
       assert (nreqs == reqs.size());
-      wtime_commstate_waitall_final.start();
+      static Timer timer ("commstate_waitall_final");
+      timer.start ();
       MPI_Waitall (reqs.size(), &reqs.front(), MPI_STATUSES_IGNORE);
-      wtime_commstate_waitall_final.stop();
+      timer.stop (0);
     } else {
-      wtime_commstate_waitall_final.start();
+      static Timer timer ("commstate_waitall_final");
+      timer.start ();
       MPI_Waitall (srequests.size(), &srequests.front(), MPI_STATUSES_IGNORE);
-      wtime_commstate_waitall_final.stop();
+      timer.stop (0);
     }
 
     return true;
@@ -304,13 +318,15 @@ bool comm_state::AllPostedCommunicationsFinished ()
         }
       }
       assert (nreqs == reqs.size());
-      wtime_commstate_waitall.start();
+      static Timer timer ("commstate_waitall");
+      timer.start ();
       MPI_Waitall (reqs.size(), &reqs.front(), MPI_STATUSES_IGNORE);
-      wtime_commstate_waitall.stop();
+      timer.stop (0);
     } else {
-      wtime_commstate_waitall.start();
+      static Timer timer ("commstate_waitall");
+      timer.start ();
       MPI_Waitall (rrequests.size(), &rrequests.front(), MPI_STATUSES_IGNORE);
-      wtime_commstate_waitall.stop();
+      timer.stop (0);
     }
     num_completed_recvs = num_posted_recvs;
   } else {
@@ -318,10 +334,11 @@ bool comm_state::AllPostedCommunicationsFinished ()
     vector<int> completed_recvs(rrequests.size(), -1);
 
     // wait for completion of at least one posted receive operation
-    wtime_commstate_waitsome.start();
+    static Timer timer ("commstate_waitsome");
+    timer.start ();
     MPI_Waitsome (rrequests.size(), &rrequests.front(), &num_completed_recvs_,
                   &completed_recvs.front(), MPI_STATUSES_IGNORE);
-    wtime_commstate_waitsome.stop();
+    timer.stop (0);
     assert (0 < num_completed_recvs_);
     num_completed_recvs += num_completed_recvs_;
 
diff --git a/Carpet/CarpetLib/src/data.cc b/Carpet/CarpetLib/src/data.cc
index e3feae50f..bdcfcd43e 100644
--- a/Carpet/CarpetLib/src/data.cc
+++ b/Carpet/CarpetLib/src/data.cc
@@ -252,7 +252,8 @@ void data<T>::change_processor_recv (comm_state& state,
     return;
   }
   
-  wtime_changeproc_recv.start();
+  static Timer total ("change_processor_recv");
+  total.start ();
   
   assert (vectorlength == 1);
   
@@ -264,12 +265,13 @@ void data<T>::change_processor_recv (comm_state& state,
       _memory = new mem<T> (1, _size, (T*)memptr);
       _memory->register_client (0);
       
-      wtime_irecv.start();
+      static Timer timer ("irecv");
+      timer.start ();
       T dummy;
       MPI_Irecv (_memory->storage(0),
                  _size, dist::datatype(dummy), proc(),
                  tag, dist::comm(), &request);
-      wtime_irecv.stop(_size * sizeof(T));
+      timer.stop (_size * sizeof(T));
       if (use_waitall) {
         state.requests.push_back (request);
       }
@@ -283,7 +285,7 @@ void data<T>::change_processor_recv (comm_state& state,
     }
   }
   
-  wtime_changeproc_recv.stop();
+  total.stop (0);
 }
 
 
@@ -302,7 +304,8 @@ void data<T>::change_processor_send (comm_state& state,
     return;
   }
   
-  wtime_changeproc_send.start();
+  static Timer total ("change_processor_send");
+  total.start();
   
   assert (vectorlength == 1);
   
@@ -316,12 +319,13 @@ void data<T>::change_processor_send (comm_state& state,
       assert (not memptr);
       assert (_memory);
       
-      wtime_isend.start();
+      static Timer timer ("isend");
+      timer.start ();
       T dummy;
       MPI_Isend (_memory->storage(0),
                  _size, dist::datatype(dummy), newproc,
                  tag, dist::comm(), &request);
-      wtime_isend.stop(_size * sizeof(T));
+      timer.stop (_size * sizeof(T));
       if (use_waitall) {
         state.requests.push_back (request);
       }
@@ -332,7 +336,7 @@ void data<T>::change_processor_send (comm_state& state,
     }
   }
   
-  wtime_changeproc_send.stop();
+  total.stop (0);
 }
 
 
@@ -352,17 +356,19 @@ void data<T>::change_processor_wait (comm_state& state,
     return;
   }
   
-  wtime_changeproc_wait.start();
+  static Timer total ("change_processor_wait");
+  total.start ();
   
   assert (vectorlength == 1);
   
   if (use_waitall) {
     if (not state.requests.empty()) {
       // wait for all requests at once
-      wtime_irecvwait.start();
+      static Timer timer ("irecvwait");
+      timer.start ();
       MPI_Waitall
         (state.requests.size(), &state.requests.front(), MPI_STATUSES_IGNORE);
-      wtime_irecvwait.stop();
+      timer.stop (0);
       state.requests.clear();
     }
   }
@@ -372,9 +378,10 @@ void data<T>::change_processor_wait (comm_state& state,
       // copy from other processor
       
       if (not use_waitall) {
-        wtime_irecvwait.start();
+        static Timer timer ("irecvwait");
+        timer.start ();
         MPI_Wait (&request, MPI_STATUS_IGNORE);
-        wtime_irecvwait.stop();
+        timer.stop (0);
       }
       
     } else if (dist::rank() == _proc) {
@@ -384,9 +391,10 @@ void data<T>::change_processor_wait (comm_state& state,
       assert (_memory);
       
       if (not use_waitall) {
-        wtime_isendwait.start();
+        static Timer timer ("isendwait");
+        timer.start ();
         MPI_Wait (&request, MPI_STATUS_IGNORE);
-        wtime_isendwait.stop();
+        timer.stop (0);
       }
       
       _memory->unregister_client (0);
@@ -401,7 +409,7 @@ void data<T>::change_processor_wait (comm_state& state,
   
   _proc = newproc;
   
-  wtime_changeproc_wait.stop();
+  total.stop (0);
 }
 
 
@@ -671,13 +679,15 @@ void data <T>
                           ibbox const & box,
                           int const order_space)
 {
-  wtime_prolongate.start();
+  static Timer total ("prolongate");
+  total.start ();
   
   switch (transport_operator) {
     
   case op_copy:
-  case op_Lagrange:
-    wtime_prolongate_Lagrange.start();
+  case op_Lagrange: {
+    static Timer timer ("prolongate_Lagrange");
+    timer.start ();
     switch (order_space) {
     case 1:
       prolongate_3d_o1_rf2 (static_cast <T const *> (src->storage()),
@@ -709,11 +719,13 @@ void data <T>
     default:
       assert (0);
     }
-    wtime_prolongate_Lagrange.stop();
+    timer.stop (0);
     break;
+  }
     
-  case op_ENO:
-    wtime_prolongate_ENO.start();
+  case op_ENO: {
+    static Timer timer ("prolongate_ENO");
+    timer.start ();
     switch (order_space) {
     case 1:
       CCTK_WARN (0, "There is no stencil for op=\"ENO\" with order_space=1");
@@ -730,11 +742,13 @@ void data <T>
     default:
       assert (0);
     }
-    wtime_prolongate_ENO.stop();
+    timer.stop (0);
     break;
+  }
     
-  case op_WENO:
-    wtime_prolongate_WENO.start();
+  case op_WENO: {
+    static Timer timer ("prolongate_WENO");
+    timer.start ();
     switch (order_space) {
     case 1:
       CCTK_WARN (0, "There is no stencil for op=\"WENO\" with order_space=1");
@@ -754,14 +768,15 @@ void data <T>
     default:
       assert (0);
     }
-    wtime_prolongate_WENO.stop();
+    timer.stop (0);
     break;
+  }
     
   default:
     assert(0);
   } // switch (transport_operator)
   
-  wtime_prolongate.stop();
+  total.stop (0);
 }
 
 template <>
@@ -781,7 +796,8 @@ void data <T>
                         ibbox const & box,
                         int const order_space)
 {
-  wtime_restrict.start();
+  static Timer total ("restrict");
+  total.start ();
   
   switch (transport_operator) {
     
@@ -818,7 +834,7 @@ void data <T>
     assert(0);
   }
   
-  wtime_restrict.stop();
+  total.stop (0);
 }
 
 template <>
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index 7180c2ce9..b07422a91 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -14,11 +14,13 @@
 #include "commstate.hh"
 #include "defs.hh"
 #include "dist.hh"
+#include "timestat.hh"
 #include "vect.hh"
 
 #include "gdata.hh"
 
 using namespace std;
+using namespace CarpetLib;
 
 
 
@@ -171,36 +173,41 @@ void gdata::copy_from (comm_state& state,
 void gdata::copy_from_post (comm_state& state,
                             const gdata* src, const ibbox& box)
 {
-  wtime_copyfrom_recv.start();
+  static Timer total ("copy_from_post");
+  total.start();
 
   if (dist::rank() == proc()) {
 
     // this processor receives data
 
-    wtime_copyfrom_recvinner_allocate.start();
+    static Timer alloc ("copy_from_post_receive_allocate");
+    alloc.start ();
     comm_state::gcommbuf * b = make_typed_commbuf (box);
     int typesize;
     MPI_Type_size (b->datatype(), & typesize);
-    wtime_copyfrom_recvinner_allocate.stop(b->size() * typesize);
+    alloc.stop (b->size() * typesize);
 
-    wtime_copyfrom_recvinner_recv.start();
+    static Timer timer ("copy_from_post_receive_irecv");
+    timer. start ();
     MPI_Irecv (b->pointer(), b->size(), b->datatype(), src->proc(),
                tag, dist::comm(), &b->request);
-    wtime_copyfrom_recvinner_recv.stop(b->size() * typesize);
+    timer.stop (b->size() * typesize);
     state.requests.push_back (b->request);
     state.recvbufs.push (b);
 
   } else {
     // this processor sends data
 
-    wtime_copyfrom_sendinner_allocate.start();
+    static Timer alloc ("copy_from_post_send_allocate");
+    alloc.start ();
     comm_state::gcommbuf * b = src->make_typed_commbuf (box);
     int typesize;
     MPI_Type_size (b->datatype(), & typesize);
-    wtime_copyfrom_sendinner_allocate.stop(b->size() * typesize);
+    alloc.stop (b->size() * typesize);
 
     // copy data into send buffer
-    wtime_copyfrom_sendinner_copy.start();
+    static Timer copy ("copy_from_post_send_memcpy");
+    copy.start ();
     const ibbox& ext = src->extent();
     ivect myshape = ext.shape() / ext.stride();
     ivect items = (box.upper() - box.lower()) / box.stride() + 1;
@@ -218,33 +225,36 @@ void gdata::copy_from_post (comm_state& state,
         bytes += datatypesize * items[0];
       }
     }
-    wtime_copyfrom_sendinner_copy.stop(bytes);
+    copy.stop (bytes);
 
-    wtime_copyfrom_sendinner_send.start();
+    static Timer timer ("copy_from_post_send_isend");
+    timer.start ();
     MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
                tag, dist::comm(), &b->request);
-    wtime_copyfrom_sendinner_send.stop(b->size() * typesize);
+    timer.stop (b->size() * typesize);
     state.requests.push_back (b->request);
     state.sendbufs.push (b);
   }
 
-  wtime_copyfrom_recv.stop();
+  total.stop (0);
 }
 
 
 void gdata::copy_from_wait (comm_state& state,
                             const gdata* src, const ibbox& box)
 {
-  wtime_copyfrom_wait.start();
+  static Timer total ("copy_from_wait");
+  total.start ();
 
-  wtime_copyfrom_recvwaitinner_wait.start();
+  static Timer wait ("copy_from_wait_wait");
+  wait.start ();
   if (not state.requests.empty()) {
     // wait for all requests at once
     MPI_Waitall (state.requests.size(), &state.requests.front(),
                  MPI_STATUSES_IGNORE);
     state.requests.clear();
   }
-  wtime_copyfrom_recvwaitinner_wait.stop();
+  wait.stop (0);
 
   queue<comm_state::gcommbuf*>* const bufs =
     dist::rank() == proc() ? &state.recvbufs : &state.sendbufs;
@@ -252,7 +262,8 @@ void gdata::copy_from_wait (comm_state& state,
 
   // copy data out of receive buffer
   if (bufs == &state.recvbufs) {
-    wtime_copyfrom_recvwaitinner_copy.start();
+    static Timer timer ("copy_from_wait_memcpy");
+    timer.start ();
     const ibbox& ext = extent();
     ivect myshape = ext.shape() / ext.stride();
     ivect items = (box.upper() - box.lower()) / box.stride() + 1;
@@ -268,15 +279,16 @@ void gdata::copy_from_wait (comm_state& state,
         recv_buffer += datatypesize * items[0];
       }
     }
-    wtime_copyfrom_recvwaitinner_copy.stop();
+    timer.stop (0);
   }
 
-  wtime_copyfrom_recvwaitinner_delete.start();
+  static Timer del ("copy_from_wait_delete");
+  del.start ();
   bufs->pop();
   delete b;
-  wtime_copyfrom_recvwaitinner_delete.stop();
+  del.stop (0);
 
-  wtime_copyfrom_wait.stop();
+  total.stop (0);
 }
 
 
@@ -322,12 +334,13 @@ void gdata::copy_into_sendbuffer (comm_state& state,
     if (not combine_sends) {
       // post the send if the buffer is full
       if (fillstate == (int)procbuf.sendbufsize * datatypesize) {
-        wtime_commstate_isend.start();
+        static Timer timer ("copy_into_sendbuffer_isend");
+        timer.start ();
         MPI_Isend (procbuf.sendbufbase, procbuf.sendbufsize,
                    state.typebufs.AT(c_datatype()).mpi_datatype,
                    proc(), c_datatype(), dist::comm(),
                    &state.srequests.at(dist::size()*c_datatype() + proc()));
-        wtime_commstate_isend.stop(procbuf.sendbufsize * datatypesize);
+        timer.stop (procbuf.sendbufsize * datatypesize);
       }
     }
   }
@@ -351,7 +364,8 @@ void gdata::copy_from_recvbuffer (comm_state& state,
   ivect items = (box.upper() - box.lower()) / box.stride() + 1;
   ivect offs  = (box.lower() - ext.lower()) / ext.stride();
 
-  wtime_commstate_memcpy.start();
+  static Timer timer ("copy_from_recvbuffer_memcpy");
+  timer.start ();
   double bytes = 0;
   assert (dim == 3);
   for (int k = 0; k < items[2]; k++) {
@@ -363,7 +377,7 @@ void gdata::copy_from_recvbuffer (comm_state& state,
       bytes += datatypesize * items[0];
     }
   }
-  wtime_commstate_memcpy.stop(bytes);
+  timer.stop (bytes);
 }
 
 
@@ -472,10 +486,11 @@ void gdata
     int typesize;
     MPI_Type_size (b->datatype(), & typesize);
 
-    wtime_commstate_interpolate_irecv.start();
+    static Timer timer ("interpolate_from_post_irecv");
+    timer.start ();
     MPI_Irecv (b->pointer(), b->size(), b->datatype(), src->proc(),
                tag, dist::comm(), &b->request);
-    wtime_commstate_interpolate_irecv.stop(b->size() * typesize);
+    timer.stop (b->size() * typesize);
     state.requests.push_back (b->request);
     state.recvbufs.push (b);
   } else {
@@ -491,10 +506,11 @@ void gdata
                                      order_space, order_time);
     delete tmp;
 
-    wtime_commstate_interpolate_from_isend.start();
+    static Timer timer ("interpolate_from_post_isend");
+    timer.start ();
     MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
                tag, dist::comm(), &b->request);
-    wtime_commstate_interpolate_from_isend.stop(b->size() * typesize);
+    timer.stop (b->size() * typesize);
     state.requests.push_back (b->request);
     state.sendbufs.push (b);
   }
@@ -545,12 +561,13 @@ void gdata
     if (not combine_sends) {
       // post the send if the buffer is full
       if (fillstate == (int)procbuf.sendbufsize*datatypesize) {
-        wtime_commstate_interpolate_to_isend.start();
+        static Timer timer ("interpolate_into_sendbuffer_isend");
+        timer.start ();
         MPI_Isend (procbuf.sendbufbase, procbuf.sendbufsize,
                    state.typebufs.at(c_datatype()).mpi_datatype,
                    proc(), c_datatype(), dist::comm(),
                    &state.srequests.at(dist::size()*c_datatype() + proc()));
-        wtime_commstate_interpolate_to_isend.stop(procbuf.sendbufsize*datatypesize);
+        timer.stop (procbuf.sendbufsize*datatypesize);
       }
     }
   }
diff --git a/Carpet/CarpetLib/src/ggf.cc b/Carpet/CarpetLib/src/ggf.cc
index 895cf941d..541c29427 100644
--- a/Carpet/CarpetLib/src/ggf.cc
+++ b/Carpet/CarpetLib/src/ggf.cc
@@ -9,10 +9,12 @@
 #include "defs.hh"
 #include "dh.hh"
 #include "th.hh"
+#include "timestat.hh"
 
 #include "ggf.hh"
 
 using namespace std;
+using namespace CarpetLib;
 
 
 
@@ -319,11 +321,14 @@ void ggf::copycat (comm_state& state,
   int const c2= c1;
   assert (ml2<h.mglevels());
   assert (tl2>=0 and tl2<timelevels(ml2,rl2));
+  static Timer copycat1 ("copycat_1");
+  copycat1.start ();
   ibbox const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_box;
   // copy the content
   gdata * const dst = storage.AT(ml1).AT(rl1).AT(c1).AT(tl1);
   gdata const * const src = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2);
   dst->copy_from(state, src, recv);
+  copycat1.stop (0);
 }
 
 // Copy regions
@@ -340,6 +345,8 @@ void ggf::copycat (comm_state& state,
   assert (rl2>=0 and rl2<h.reflevels());
   int const c2 = c1;
   assert (tl2>=0 and tl2<timelevels(ml2,rl2));
+  static Timer copycat1 ("copycat_list_1");
+  copycat1.start ();
   iblist const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
   // walk all boxes
   for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r) {
@@ -349,6 +356,7 @@ void ggf::copycat (comm_state& state,
     gdata const * const src = storage.AT(ml2).AT(rl2).AT(c2).AT(tl2);
     dst->copy_from(state, src, *r);
   }
+  copycat1.stop (0);
 }
 
 // Copy regions
@@ -365,6 +373,8 @@ void ggf::copycat (comm_state& state,
   assert (rl2>=0 and rl2<h.reflevels());
   assert (tl2>=0 and tl2<timelevels(ml2,rl2));
   // walk all components
+  static Timer copycat1 ("copycat_listvect_1");
+  copycat1.start ();
   for (int c2=0; c2<h.components(rl2); ++c2) {
     const iblist recv = (d.boxes.at(ml1).at(rl1).at(c1).*recv_listvect).at(c2);
     // walk all boxes
@@ -376,6 +386,7 @@ void ggf::copycat (comm_state& state,
       dst->copy_from(state, src, *r);
     }
   }
+  copycat1.stop (0);
 }
 
 // Interpolate a region
@@ -403,11 +414,14 @@ void ggf::intercat (comm_state& state,
     times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
   }
   
+  static Timer intercat1 ("intercat_1");
+  intercat1.start ();
   ibbox const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
   // interpolate the content
   storage.AT(ml1).AT(rl1).AT(c1).AT(tl1)->interpolate_from
     (state, gsrcs, times, recv, time,
      d.prolongation_order_space, prolongation_order_time);
+  intercat1.stop (0);
 }
 
 // Interpolate regions
@@ -435,6 +449,8 @@ void ggf::intercat (comm_state& state,
     times.AT(i) = t.time(tl2s.AT(i),rl2,ml2);
   }
   
+  static Timer intercat1 ("intercat_list_1");
+  intercat1.start ();
   iblist const & recv = d.boxes.AT(ml1).AT(rl1).AT(c1).*recv_list;
   // walk all boxes
   for (iblist::const_iterator r=recv.begin(); r!=recv.end(); ++r)
@@ -445,6 +461,7 @@ void ggf::intercat (comm_state& state,
       (state, gsrcs, times, *r, time,
        d.prolongation_order_space, prolongation_order_time);
   }
+  intercat1.stop (0);
 }
 
 // Interpolate regions
@@ -463,6 +480,8 @@ void ggf::intercat (comm_state& state,
     assert (tl2s.AT(i)>=0 and tl2s.AT(i)<timelevels(ml2,rl2));
   }
   // walk all components
+  static Timer intercat1 ("intercat_listvect_1");
+  intercat1.start ();
   for (int c2=0; c2<h.components(rl2); ++c2) {
     assert (ml2>=0 and ml2<h.mglevels());
     
@@ -487,6 +506,7 @@ void ggf::intercat (comm_state& state,
       	(state, gsrcs, times, *r, time, pos, pot);
     }
   }
+  intercat1.stop (0);
 }
 
 
@@ -495,18 +515,24 @@ void ggf::intercat (comm_state& state,
 void ggf::copy (comm_state& state, int tl, int rl, int c, int ml)
 {
   // Copy
+  static Timer timer ("copy");
+  timer.start ();
   copycat (state,
            tl  ,rl,c,ml, &dh::dboxes::exterior,
       	   tl+1,rl,  ml);
+  timer.stop (0);
 }
 
 // Synchronise the boundaries a component
 void ggf::sync (comm_state& state, int tl, int rl, int c, int ml)
 {
   // Copy
+  static Timer timer ("sync");
+  timer.start ();
   copycat (state,
            tl,rl,c,ml, &dh::dboxes::recv_sync,
       	   tl,rl,  ml);
+  timer.stop (0);
 }
 
 // Prolongate the boundaries of a component
@@ -518,6 +544,8 @@ void ggf::ref_bnd_prolongate (comm_state& state,
   assert (rl>=1);
   if (transport_operator == op_none) return;
   vector<int> tl2s;
+  static Timer timer ("ref_bnd_prolongate");
+  timer.start ();
   if (transport_operator != op_copy) {
     // Interpolation in time
     if (not (timelevels(ml,rl) >= prolongation_order_time+1)) {
@@ -539,6 +567,7 @@ void ggf::ref_bnd_prolongate (comm_state& state,
   intercat (state,
             tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_bnd_coarse,
             tl2s,rl-1,  ml, time);
+  timer.stop (0);
 }
 
 // Restrict a multigrid level
@@ -550,9 +579,12 @@ void ggf::mg_restrict (comm_state& state,
   assert (abs(t.get_time(rl,ml) - t.get_time(rl,ml-1))
 	  <= 1.0e-8 * abs(t.get_time(rl,ml)));
   const vector<int> tl2s(1,tl);
+  static Timer timer ("mg_restrict");
+  timer.start ();
   intercat (state,
             tl  ,rl,c,ml,   &dh::dboxes::recv_mg_coarse,
 	    tl2s,rl,  ml-1, time);
+  timer.stop (0);
 }
 
 // Prolongate a multigrid level
@@ -563,10 +595,13 @@ void ggf::mg_prolongate (comm_state& state,
   // Require same times
   assert (abs(t.get_time(rl,ml) - t.get_time(rl,ml+1))
 	  <= 1.0e-8 * abs(t.get_time(rl,ml)));
+  static Timer timer ("mg_prolongate");
+  timer.start ();
   const vector<int> tl2s(1,tl);
   intercat (state,
             tl  ,rl,c,ml,   &dh::dboxes::recv_mg_coarse,
 	    tl2s,rl,  ml+1, time);
+  timer.stop (0);
 }
 
 // Restrict a refinement level
@@ -579,9 +614,12 @@ void ggf::ref_restrict (comm_state& state,
 	  <= 1.0e-8 * abs(t.get_time(rl,ml)));
   if (transport_operator == op_none) return;
   const vector<int> tl2s(1,tl);
+  static Timer timer ("ref_restrict");
+  timer.start ();
   intercat (state,
             tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_fine,
 	    tl2s,rl+1,  ml, time);
+  timer.stop (0);
 }
 
 // Prolongate a refinement level
@@ -592,6 +630,8 @@ void ggf::ref_prolongate (comm_state& state,
   assert (rl>=1);
   if (transport_operator == op_none) return;
   vector<int> tl2s;
+  static Timer timer ("ref_prolongate");
+  timer.start ();
   // Interpolation in time
   assert (timelevels(ml,rl) >= prolongation_order_time+1);
   tl2s.resize(prolongation_order_time+1);
@@ -599,4 +639,5 @@ void ggf::ref_prolongate (comm_state& state,
   intercat (state,
             tl  ,rl  ,c,ml, &dh::dboxes::recv_ref_coarse,
 	    tl2s,rl-1,  ml, time);
+  timer.stop (0);
 }
diff --git a/Carpet/CarpetLib/src/timestat.cc b/Carpet/CarpetLib/src/timestat.cc
index addd1540d..0f2439fec 100644
--- a/Carpet/CarpetLib/src/timestat.cc
+++ b/Carpet/CarpetLib/src/timestat.cc
@@ -1,254 +1,357 @@
 #include <algorithm>
 #include <cassert>
 #include <cmath>
+#include <cstdio>
 #include <fstream>
 #include <iomanip>
 #include <iostream>
 #include <sstream>
 
+#include <unistd.h>
+
 #include <mpi.h>
 
 #include "cctk.h"
 #include "cctk_Arguments.h"
 #include "cctk_Parameters.h"
 
+#include "defs.hh"
 #include "dist.hh"
 #include "timestat.hh"
 
 
 
-using namespace std;
-
-
-
-timestat::timestat ()
-  : wtime(0.0), wtime2(0.0), wmin(0.0), wmax(0.0),
-    bytes(0.0), bytes2(0.0), bmin(0.0), bmax(0.0),
-    count(0.0),
-    running(false)
-{
-}
-
-void timestat::addstat (double const t, double const b)
-{
-  wtime += t;
-  wtime2 += t*t;
-  wmin = min (wmin, t);
-  wmax = max (wmax, t);
-  
-  bytes += b;
-  bytes2 += b*b;
-  bmin = min (bmin, b);
-  bmax = max (bmax, b);
-  
-  ++count;
-}
-
-void timestat::start ()
-{
-  assert (! running);
-  running = true;
-  starttime = MPI_Wtime();
-}
-
-void timestat::stop (double const b)
-{
-  assert (running);
-  running = false;
-  double const endtime = MPI_Wtime();
-  addstat (endtime - starttime, b);
-}
-
-
-
-ostream& operator<< (ostream& os, const timestat& wt)
-{
-  double avg, stddev, bavg, bstddev;
-  if (wt.count == 0.0) {
-    avg = 0.0;
-    stddev = 0.0;
-    bavg = 0.0;
-    bstddev = 0.0;
-  } else {
-    avg = wt.wtime / wt.count;
-    stddev = sqrt (max (0.0, wt.wtime2 / wt.count - pow (avg, 2)));
-    bavg = wt.bytes / wt.count;
-    bstddev = sqrt (max (0.0, wt.bytes2 / wt.count - pow (bavg, 2)));
-  }
-  os << "timestat[seconds]:"
-     << " cnt: " << wt.count
-     << "   time: sum: " << wt.wtime
-     << " avg: " << avg
-     << " stddev: " << stddev
-     << " min: " << wt.wmin
-     << " max: " << wt.wmax
-     << "   bytes: sum: " << wt.bytes
-     << " avg: " << bavg
-     << " stddev: " << bstddev
-     << " min: " << wt.bmin
-     << " max: " << wt.bmax;
-  return os;
-}
-
-
-
-timestat wtime_copyfrom_recv;
-timestat wtime_copyfrom_send;
-timestat wtime_copyfrom_wait;
-
-timestat wtime_copyfrom_recv_maketyped;
-timestat wtime_copyfrom_recv_allocate;
-timestat wtime_copyfrom_recv_changeproc_recv;
-timestat wtime_copyfrom_send_copyfrom_nocomm1;
-timestat wtime_copyfrom_send_copyfrom_nocomm2;
-timestat wtime_copyfrom_send_changeproc_send;
-timestat wtime_copyfrom_wait_changeproc_wait;
-timestat wtime_copyfrom_wait_copyfrom_nocomm;
-timestat wtime_copyfrom_wait_delete;
-
-timestat wtime_copyfrom_recvinner_allocate;
-timestat wtime_copyfrom_recvinner_recv;
-timestat wtime_copyfrom_sendinner_allocate;
-timestat wtime_copyfrom_sendinner_copy;
-timestat wtime_copyfrom_sendinner_send;
-timestat wtime_copyfrom_recvwaitinner_wait;
-timestat wtime_copyfrom_recvwaitinner_copy;
-timestat wtime_copyfrom_recvwaitinner_delete;
-timestat wtime_copyfrom_sendwaitinner_wait;
-timestat wtime_copyfrom_sendwaitinner_delete;
-
-timestat wtime_changeproc_recv;
-timestat wtime_changeproc_send;
-timestat wtime_changeproc_wait;
-
-timestat wtime_irecv;
-timestat wtime_isend;
-timestat wtime_isendwait;
-timestat wtime_irecvwait;
-
-timestat wtime_commstate_sizes_irecv;
-timestat wtime_commstate_waitall_final;
-timestat wtime_commstate_waitall;
-timestat wtime_commstate_waitsome;
-timestat wtime_commstate_send;
-timestat wtime_commstate_ssend;
-timestat wtime_commstate_isend;
-timestat wtime_commstate_memcpy;
-timestat wtime_commstate_interpolate_irecv;
-timestat wtime_commstate_interpolate_from_isend;
-timestat wtime_commstate_interpolate_to_isend;
-
-
-
-timestat wtime_restrict;
-timestat wtime_prolongate;
-timestat wtime_prolongate_copy;
-timestat wtime_prolongate_Lagrange;
-timestat wtime_prolongate_ENO;
-timestat wtime_prolongate_WENO;
-
-
-
-extern "C" void CarpetLib_printtimestats (CCTK_ARGUMENTS);
-
-void CarpetLib_printtimestats (CCTK_ARGUMENTS)
-{
-  DECLARE_CCTK_ARGUMENTS;
-  DECLARE_CCTK_PARAMETERS;
-  if (print_timestats or
-      (print_timestats_every and
-       cctk_iteration % print_timestats_every == 0))
-  {
-    ostringstream filenamebuf;
-    filenamebuf << out_dir << "/" << timestat_file
-                << "." << setw(4) << setfill('0') << dist::rank()
-                << ".txt";
-    string const filename = filenamebuf.str();
-    
-    ofstream file;
-    static bool do_truncate = true;
-    if (do_truncate) {
-      if (not IO_TruncateOutputFiles (cctkGH)) {
-        do_truncate = false;
+namespace CarpetLib {
+  
+  using namespace std;
+  
+  
+  
+  // A faster timing routine:
+  // Read the Intel CPU time stamp counter
+  static inline
+  double
+  rdtsc ()
+  {
+#if defined(__i386__)
+    unsigned long long int val;
+    __asm__ __volatile__("rdtsc" : "=A" (val) : );
+    return val;
+#else
+    return 0;
+#endif
+  }
+  static
+  double rdtsc_cputick = 0.0;
+  
+  static
+  void
+  init_rdtsc ()
+  {
+    double const rstart = rdtsc ();
+    double const wstart = MPI_Wtime ();
+    int const ierr = usleep (1000 * 1000);
+    double const rend = rdtsc ();
+    double const wend = MPI_Wtime ();
+    if (ierr) {
+      CCTK_WARN (1, "Could not determine a reliable rdtsc timer resolution");
+    }
+    rdtsc_cputick = (wend - wstart) / (rend - rstart);
+  }
+  
+  
+  
+  
+  // Call a timer
+  static
+  double
+  call_timer ()
+  {
+    DECLARE_CCTK_PARAMETERS;
+    enum timer_type { timer_unset, timer_MPI_Wtime, timer_rdtsc, timer_none };
+    static timer_type timer = timer_unset;
+    if (timer == timer_unset) {
+      if (CCTK_EQUALS (timestat_timer, "MPI_Wtime")) {
+        timer = timer_MPI_Wtime;
+      } else if (CCTK_EQUALS (timestat_timer, "rdtsc")) {
+        timer = timer_rdtsc;
+        init_rdtsc ();
+      } else if (CCTK_EQUALS (timestat_timer, "none")) {
+        timer = timer_none;
+      } else {
+        assert (0);
       }
     }
-    if (do_truncate) {
-      do_truncate = false;
-      file.open (filename.c_str(), ios::out | ios::trunc);
+    switch (timer) {
+    case timer_MPI_Wtime:
+      return MPI_Wtime ();
+    case timer_rdtsc:
+      return rdtsc_cputick * rdtsc ();
+    case timer_none:
+      return 0.0;
+    default:
+      assert (0);
+    }
+  }
+  
+  
+  
+  // A global timer set
+  static
+  TimerSet timerSet;
+  
+  
+  
+  // Add a timer
+  void
+  TimerSet::add (Timer * const timer)
+  {
+    timers.push_back (timer);
+  }
+  
+  
+  
+  // Remove a timer
+  void
+  TimerSet::remove (Timer * const timer)
+  {
+    timers.remove (timer);
+  }
+  
+  
+  
+  // Output all timer names
+  void
+  TimerSet::outputNames (ostream & os)
+    const
+  {
+    os << "Timer names:" << eol;
+    int n = 0;
+    for (list <Timer *>::const_iterator
+           itimer = timers.begin(); itimer != timers.end(); ++ itimer)
+    {
+      os << "   [" << setw (4) << setfill ('0') << n << "] "
+         << (* itimer)->name() << eol;
+      ++ n;
+    }
+  }
+  
+  
+  
+  // Output all timer data
+  void
+  TimerSet::outputData (ostream & os)
+    const
+  {
+    for (list <Timer *>::const_iterator
+           itimer = timers.begin(); itimer != timers.end(); ++ itimer)
+    {
+      os << * (* itimer);
+    }
+  }
+  
+  
+  
+  // Create a new timer with the given name
+  Timer::Timer (char const * const timername_)
+    : timername (timername_)
+  {
+    assert (timername_);
+    resetstats ();
+    timerSet.add (this);
+  }
+  
+  
+  
+  // Destroy a timer
+  Timer::~Timer ()
+  {
+    timerSet.remove (this);
+  }
+  
+  
+  
+  // Reset the statistics
+  void
+  Timer::resetstats ()
+  {
+    wtime  = 0.0;
+    wtime2 = 0.0;
+    wmin   = 0.0;
+    wmax   = 0.0;
+    
+    bytes  = 0.0;
+    bytes2 = 0.0;
+    bmin   = 0.0;
+    bmax   = 0.0;
+    
+    count  = 0.0;
+    
+    running = false;
+  }
+  
+  
+  
+  // Add statistics of a timing operation
+  void
+  Timer::addstat (double const t,
+                  double const b)
+  {
+    wtime  += t;
+    wtime2 += pow (t, 2);
+    wmin   = min (wmin, t);
+    wmax   = max (wmax, t);
+    
+    bytes  += b;
+    bytes2 += pow (b, 2);
+    bmin   = min (bmin, b);
+    bmax   = max (bmax, b);
+    
+    ++ count;
+  }
+  
+  
+  
+  // Start the timer
+  void
+  Timer::start ()
+  {
+    DECLARE_CCTK_PARAMETERS;
+    if (timestat_disable) return;
+    assert (not running);
+    running = true;
+    starttime = call_timer ();
+  }
+  
+  
+  
+  // Stop the timer
+  void
+  Timer::stop (double const b)
+  {
+    DECLARE_CCTK_PARAMETERS;
+    if (timestat_disable) return;
+    assert (running);
+    running = false;
+    double const endtime = call_timer ();
+    addstat (endtime - starttime, b);
+  }
+  
+  
+  
+  // Reset the timer
+  void
+  Timer::reset ()
+  {
+    resetstats ();
+  }
+  
+  
+  
+  // Timer name
+  string
+  Timer::name ()
+    const
+  {
+    return timername;
+  }
+  
+  
+  
+  // Output timer data
+  void
+  Timer::outputData (ostream & os)
+    const
+  {
+    double avg, stddev, bavg, bstddev;
+    if (count == 0.0) {
+      avg     = 0.0;
+      stddev  = 0.0;
+      bavg    = 0.0;
+      bstddev = 0.0;
     } else {
-      file.open (filename.c_str(), ios::out | ios::app);
+      avg     = wtime / count;
+      stddev  = sqrt (max (0.0, wtime2 / count - pow (avg, 2)));
+      bavg    = bytes / count;
+      bstddev = sqrt (max (0.0, bytes2 / count - pow (bavg, 2)));
     }
     
-    static bool do_print_info = true;
-    if (do_print_info) {
-      do_print_info = false;
-      if (CCTK_IsFunctionAliased ("UniqueBuildID")) {
-        char const * const build_id
-          = static_cast<char const *> (UniqueBuildID (cctkGH));
-        file << "Build ID: " << build_id << endl;
+    os << timername << ":"
+       << " cnt: " << count
+       << "   time: sum: " << wtime
+       << " avg: " << avg
+       << " stddev: " << stddev
+       << " min: " << wmin
+       << " max: " << wmax
+       << "   bytes: sum: " << bytes
+       << " avg: " << bavg
+       << " stddev: " << bstddev
+       << " min: " << bmin
+       << " max: " << bmax
+       << eol;
+  }
+  
+  
+  
+  extern "C" {
+    void
+    CarpetLib_printtimestats (CCTK_ARGUMENTS);
+  }
+  
+  void
+  CarpetLib_printtimestats (CCTK_ARGUMENTS)
+  {
+    DECLARE_CCTK_ARGUMENTS;
+    DECLARE_CCTK_PARAMETERS;
+    
+    if (print_timestats or
+        (print_timestats_every and
+         cctk_iteration % print_timestats_every == 0))
+    {
+      ostringstream filenamebuf;
+      filenamebuf << out_dir << "/" << timestat_file
+                  << "." << setw(4) << setfill('0') << dist::rank()
+                  << ".txt";
+      string const filename = filenamebuf.str();
+      
+      ofstream file;
+      static bool do_truncate = true;
+      if (do_truncate) {
+        if (not IO_TruncateOutputFiles (cctkGH)) {
+          do_truncate = false;
+        }
       }
-      if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
-        char const * const job_id
-          = static_cast<char const *> (UniqueSimulationID (cctkGH));
-        file << "Simulation ID: " << job_id << endl;
+      if (do_truncate) {
+        do_truncate = false;
+        file.open (filename.c_str(), ios::out | ios::trunc);
+      } else {
+        file.open (filename.c_str(), ios::out | ios::app);
       }
-      file << "Running on " << dist::size() << " processors" << endl;
-    }
+      
+      static bool do_print_info = true;
+      if (do_print_info) {
+        do_print_info = false;
+        if (CCTK_IsFunctionAliased ("UniqueBuildID")) {
+          char const * const build_id =
+            static_cast <char const *> (UniqueBuildID (cctkGH));
+          file << "Build ID: " << build_id << eol;
+        }
+        if (CCTK_IsFunctionAliased ("UniqueSimulationID")) {
+          char const * const job_id =
+            static_cast <char const *> (UniqueSimulationID (cctkGH));
+          file << "Simulation ID: " << job_id << eol;
+        }
+        file << "Running on " << dist::size() << " processors" << eol;
+      } // if do_print_info
+      
+      file << "********************************************************************************" << eol
+           << "CarpetLib timing information at iteration " << cctkGH->cctk_iteration << " time " << cctkGH->cctk_time << ":" << eol
+           << timerSet;
+      
+      file.close ();
+      
+    } // if print_timestats
     
-    file << endl
-         << "********************************************************************************" << endl
-         << "Timing statistics from CarpetLib at iteration " << cctkGH->cctk_iteration << " time  " << cctkGH->cctk_time << ":" << endl
-         << "   wtime_copyfrom_recv:                    " << wtime_copyfrom_recv                    << endl
-         << "   wtime_copyfrom_send:                    " << wtime_copyfrom_send                    << endl
-         << "   wtime_copyfrom_wait:                    " << wtime_copyfrom_wait                    << endl
-         << endl
-         << "   wtime_copyfrom_recv_maketyped:          " << wtime_copyfrom_recv_maketyped          << endl   
-         << "   wtime_copyfrom_recv_allocate:           " << wtime_copyfrom_recv_allocate           << endl
-         << "   wtime_copyfrom_recv_changeproc_recv:    " << wtime_copyfrom_recv_changeproc_recv    << endl
-         << "   wtime_copyfrom_send_copyfrom_nocomm1:   " << wtime_copyfrom_send_copyfrom_nocomm1   << endl
-         << "   wtime_copyfrom_send_copyfrom_nocomm2:   " << wtime_copyfrom_send_copyfrom_nocomm2   << endl
-         << "   wtime_copyfrom_send_changeproc_send:    " << wtime_copyfrom_send_changeproc_send    << endl
-         << "   wtime_copyfrom_wait_changeproc_wait:    " << wtime_copyfrom_wait_changeproc_wait    << endl
-         << "   wtime_copyfrom_wait_copyfrom_nocomm2:   " << wtime_copyfrom_wait_copyfrom_nocomm    << endl
-         << "   wtime_copyfrom_wait_delete:             " << wtime_copyfrom_wait_delete             << endl
-         << endl
-         << "   wtime_copyfrom_recvinner_allocate:      " << wtime_copyfrom_recvinner_allocate      << endl
-         << "   wtime_copyfrom_recvinner_recv:          " << wtime_copyfrom_recvinner_recv          << endl
-         << "   wtime_copyfrom_sendinner_allocate:      " << wtime_copyfrom_sendinner_allocate      << endl
-         << "   wtime_copyfrom_sendinner_copy:          " << wtime_copyfrom_sendinner_copy          << endl
-         << "   wtime_copyfrom_sendinner_send:          " << wtime_copyfrom_sendinner_send          << endl
-         << "   wtime_copyfrom_recvwaitinner_wait:      " << wtime_copyfrom_recvwaitinner_wait      << endl
-         << "   wtime_copyfrom_recvwaitinner_copy:      " << wtime_copyfrom_recvwaitinner_copy      << endl
-         << "   wtime_copyfrom_recvwaitinner_delete:    " << wtime_copyfrom_recvwaitinner_delete    << endl
-         << "   wtime_copyfrom_sendwaitinner_wait:      " << wtime_copyfrom_sendwaitinner_wait      << endl
-         << "   wtime_copyfrom_sendwaitinner_delete:    " << wtime_copyfrom_sendwaitinner_delete    << endl
-         << endl
-         << "   wtime_changeproc_recv:                  " << wtime_changeproc_recv                  << endl
-         << "   wtime_changeproc_send:                  " << wtime_changeproc_send                  << endl
-         << "   wtime_changeproc_wait:                  " << wtime_changeproc_wait                  << endl
-         << endl
-         << "   wtime_irecv:                            " << wtime_irecv                            << endl
-         << "   wtime_isend:                            " << wtime_isend                            << endl
-         << "   wtime_isendwait:                        " << wtime_isendwait                        << endl
-         << "   wtime_irecvwait:                        " << wtime_irecvwait                        << endl
-         << endl
-         << "   wtime_commstate_sizes_irecv:            " << wtime_commstate_sizes_irecv            << endl
-         << "   wtime_commstate_waitall_final:          " << wtime_commstate_waitall_final          << endl
-         << "   wtime_commstate_waitall:                " << wtime_commstate_waitall                << endl
-         << "   wtime_commstate_waitsome:               " << wtime_commstate_waitsome               << endl
-         << "   wtime_commstate_send:                   " << wtime_commstate_send                   << endl
-         << "   wtime_commstate_ssend:                  " << wtime_commstate_ssend                  << endl
-         << "   wtime_commstate_isend:                  " << wtime_commstate_isend                  << endl
-         << "   wtime_commstate_memcpy:                 " << wtime_commstate_memcpy                 << endl
-         << "   wtime_commstate_interpolate_irecv:      " << wtime_commstate_interpolate_irecv      << endl
-         << "   wtime_commstate_interpolate_from_isend: " << wtime_commstate_interpolate_from_isend << endl
-         << "   wtime_commstate_interpolate_to_isend:   " << wtime_commstate_interpolate_to_isend   << endl
-         << endl
-         << "   wtime_restrict:                         " << wtime_restrict                         << endl
-         << "   wtime_prolongate:                       " << wtime_prolongate                       << endl
-         << "   wtime_prolongate_Lagrange:              " << wtime_prolongate_Lagrange              << endl
-         << "   wtime_prolongate_ENO:                   " << wtime_prolongate_ENO                   << endl
-         << "   wtime_prolongate_WENO:                  " << wtime_prolongate_WENO                  << endl
-         << endl;
-  }
-}
+  }
+  
+} // namespace CarpetLib
diff --git a/Carpet/CarpetLib/src/timestat.hh b/Carpet/CarpetLib/src/timestat.hh
index 811369306..49eb4fab2 100644
--- a/Carpet/CarpetLib/src/timestat.hh
+++ b/Carpet/CarpetLib/src/timestat.hh
@@ -2,98 +2,134 @@
 #define TIMESTAT_HH
 
 #include <iostream>
+#include <list>
+#include <string>
 
-using namespace std;
 
 
-
-// Time (in seconds) spend during various operations
-class timestat {
+namespace CarpetLib {
   
-public:
-  double wtime;
-  double wtime2;
-  double wmin;
-  double wmax;
+  using namespace std;
   
-  double bytes;
-  double bytes2;
-  double bmin;
-  double bmax;
   
-  double count;
   
-public:
-  timestat ();
+  class Timer;
   
-private:
-  void addstat (double t, double b);
   
-private:
-  bool running;
-  double starttime;
   
-public:
-  void start();
-  void stop(double bytes = 0.0);
-};
-
-ostream& operator<< (ostream& os, const timestat& wt);
-
-
-
-extern timestat wtime_copyfrom_recv;
-extern timestat wtime_copyfrom_send;
-extern timestat wtime_copyfrom_wait;
-
-extern timestat wtime_copyfrom_recv_maketyped;
-extern timestat wtime_copyfrom_recv_allocate;
-extern timestat wtime_copyfrom_recv_changeproc_recv;
-extern timestat wtime_copyfrom_send_copyfrom_nocomm1;
-extern timestat wtime_copyfrom_send_copyfrom_nocomm2;
-extern timestat wtime_copyfrom_send_changeproc_send;
-extern timestat wtime_copyfrom_wait_changeproc_wait;
-extern timestat wtime_copyfrom_wait_copyfrom_nocomm;
-extern timestat wtime_copyfrom_wait_delete;
-
-extern timestat wtime_copyfrom_recvinner_allocate;
-extern timestat wtime_copyfrom_recvinner_recv;
-extern timestat wtime_copyfrom_sendinner_allocate;
-extern timestat wtime_copyfrom_sendinner_copy;
-extern timestat wtime_copyfrom_sendinner_send;
-extern timestat wtime_copyfrom_recvwaitinner_wait;
-extern timestat wtime_copyfrom_recvwaitinner_copy;
-extern timestat wtime_copyfrom_recvwaitinner_delete;
-extern timestat wtime_copyfrom_sendwaitinner_wait;
-extern timestat wtime_copyfrom_sendwaitinner_delete;
-
-extern timestat wtime_changeproc_recv;
-extern timestat wtime_changeproc_send;
-extern timestat wtime_changeproc_wait;
-
-extern timestat wtime_irecv;
-extern timestat wtime_isend;
-extern timestat wtime_irecvwait;
-extern timestat wtime_isendwait;
-
-extern timestat wtime_commstate_sizes_irecv;
-extern timestat wtime_commstate_waitall_final;
-extern timestat wtime_commstate_waitall;
-extern timestat wtime_commstate_waitsome;
-extern timestat wtime_commstate_send;
-extern timestat wtime_commstate_ssend;
-extern timestat wtime_commstate_isend;
-extern timestat wtime_commstate_memcpy;
-extern timestat wtime_commstate_interpolate_irecv;
-extern timestat wtime_commstate_interpolate_from_isend;
-extern timestat wtime_commstate_interpolate_to_isend;
-
-
-
-extern timestat wtime_restrict;
-extern timestat wtime_prolongate;
-extern timestat wtime_prolongate_Lagrange;
-extern timestat wtime_prolongate_ENO;
-extern timestat wtime_prolongate_WENO;
+  // A set of timers
+  class TimerSet {
+    
+    list <Timer *> timers;
+    
+  public:
+    
+    // Add a timer
+    void
+    add (Timer * timer);
+    
+    // Remove a timer
+    void
+    remove (Timer * timer);
+    
+    // Output all timer names
+    void
+    outputNames (ostream & os)
+      const;
+    
+    // Output all timer data
+    void
+    outputData (ostream & os)
+      const;
+    
+  }; // class TimerSet
+  
+  inline
+  ostream & operator << (ostream & os,
+                         TimerSet const & timerSet)
+  {
+    timerSet.outputData (os);
+    return os;
+  }
+  
+  
+  
+  // A timer, which counts time (in seconds) spent in and amount (in
+  // bytes) used in various operations
+  class Timer {
+    
+    string timername;
+    
+  public:
+    
+    // Create a new timer with the given name
+    Timer (char const * timername_);
+    
+    // Destroy a timer
+    ~Timer ();
+    
+  private:
+    
+    // Reset the statistics
+    void
+    resetstats ();
+    
+    // Add statistics of a timing operation
+    void
+    addstat (double t,
+             double b);
+    
+  private:
+    
+    double wtime;
+    double wtime2;
+    double wmin;
+    double wmax;
+    
+    double bytes;
+    double bytes2;
+    double bmin;
+    double bmax;
+    
+    double count;
+    
+    bool running;
+    double starttime;
+    
+  public:
+    
+    // Start the timer
+    void
+    start ();
+    
+    // Stop the timer
+    void
+    stop (double b);
+    
+    // Reset the timer
+    void
+    reset ();
+    
+    // Timer name
+    string
+    name ()
+      const;
+    
+    // Print timer data
+    void
+    outputData (ostream & os)
+      const;
+    
+  };
+  
+  inline
+  ostream & operator << (ostream & os,
+                         Timer const & timer)
+  {
+    timer.outputData (os);
+    return os;
+  }
+  
+} // namespace CarpetLib
 
 #endif  // TIMESTAT_HH