CarpetLib, CycleClock: Move cycle counting clock into its own thorn

New thorn CycleClock, mostly taken out of CarpetLib. This thorn provides an (almost) cycle-accurate clock, taken from FFTW. This clock is both directly accessible, and is also wrapped in a Cactus clock.

Note: Applying this commit will require adding thorn CycleClock to thorn lists and parameter files.

9 files changed, 854 insertions, 0 deletions

diff --git a/Carpet/CycleClock/configuration.ccl b/Carpet/CycleClock/configuration.ccl
new file mode 100644
index 000000000..a79586e82
--- /dev/null
+++ b/Carpet/CycleClock/configuration.ccl
@@ -0,0 +1,5 @@
+# Configuration definitions for thorn CycleClock
+
+PROVIDES CycleClock
+{
+}
diff --git a/Carpet/CycleClock/interface.ccl b/Carpet/CycleClock/interface.ccl
new file mode 100644
index 000000000..c52a14e8f
--- /dev/null
+++ b/Carpet/CycleClock/interface.ccl
@@ -0,0 +1,5 @@
+# Interface definition for thorn CycleClock
+
+IMPLEMENTS: CycleClock
+
+INCLUDES HEADER: cycleclock.h IN cycleclock.h
diff --git a/Carpet/CycleClock/param.ccl b/Carpet/CycleClock/param.ccl
new file mode 100644
index 000000000..887db2384
--- /dev/null
+++ b/Carpet/CycleClock/param.ccl
@@ -0,0 +1 @@
+# Parameter definitions for thorn CycleClock
diff --git a/Carpet/CycleClock/schedule.ccl b/Carpet/CycleClock/schedule.ccl
new file mode 100644
index 000000000..cde207d25
--- /dev/null
+++ b/Carpet/CycleClock/schedule.ccl
@@ -0,0 +1,6 @@
+# Schedule definitions for thorn CycleClock
+
+SCHEDULE CycleClock_Setup AT startup BEFORE Driver_Startup
+{
+  LANG: C
+} "Set up CycleClock"
diff --git a/Carpet/CycleClock/src/clock.cc b/Carpet/CycleClock/src/clock.cc
new file mode 100644
index 000000000..26e251ca1
--- /dev/null
+++ b/Carpet/CycleClock/src/clock.cc
@@ -0,0 +1,177 @@
+#include "cycleclock.h"
+
+#include <cctk.h>
+
+#include <cmath>
+
+
+
+namespace CycleClock {
+  
+  using namespace std;
+  
+  
+  
+  class cycleclock_t {
+    double sum;
+    double sum2;
+    double min;
+    double max;
+    double count;
+    
+    ticks last;
+    
+  public:
+    cycleclock_t()
+    {
+      reset();
+    }
+    
+    ~cycleclock_t()
+    {
+    }
+    
+    void start()
+    {
+      last = getticks();
+    }
+    
+    void stop()
+    {
+      ticks const current = getticks();
+      double const difference = elapsed(current, last);
+      sum += difference;
+      sum2 += pow(difference, 2.0);
+      min = min == 0.0 ? difference : fmin(min, difference);
+      max = fmax(min, difference);
+      count += 1.0;
+    }
+    
+    void reset()
+    {
+      sum   = 0.0;
+      sum2  = 0.0;
+      min   = 0.0;              // numeric_limits<double>::max();
+      max   = 0.0;
+      count = 0.0;
+    }
+    
+    void get(cTimerVal* restrict const vals) const
+    {
+      double const tick = seconds_per_tick();
+      
+      // Sum time
+      vals[0].type       = val_double;
+      vals[0].heading    = "cycle";
+      vals[0].units      = "secs";
+      vals[0].val.d      = sum;
+      vals[0].seconds    = tick * vals[0].val.d;
+      vals[0].resolution = tick;
+      
+      // Average
+      vals[1].type       = val_double;
+      vals[1].heading    = "cycle[avg]";
+      vals[1].units      = "secs";
+      vals[1].val.d      = count == 0.0 ? 0.0 : sum / count;
+      vals[1].seconds    = tick * vals[1].val.d;
+      vals[1].resolution = tick;
+      
+      // Standard deviation
+      vals[2].type       = val_double;
+      vals[2].heading    = "cycle[sdv]";
+      vals[2].units      = "secs";
+      vals[2].val.d      = (count == 0.0 ?
+                            0.0 :
+                            sqrt(fabs(sum2 * count - pow(sum, 2.0)) / count));
+      vals[2].seconds    = tick * vals[2].val.d;
+      vals[2].resolution = tick;
+      
+      // Minimum
+      vals[3].type       = val_double;
+      vals[3].heading    = "cycle[min]";
+      vals[3].units      = "secs";
+      vals[3].val.d      = min;
+      vals[3].seconds    = tick * vals[3].val.d;
+      vals[3].resolution = tick;
+      
+      // Maximum
+      vals[4].type       = val_double;
+      vals[4].heading    = "cycle[max]";
+      vals[4].units      = "secs";
+      vals[4].val.d      = max;
+      vals[4].seconds    = tick * vals[4].val.d;
+      vals[4].resolution = tick;
+    }
+    
+    void set(cTimerVal const* restrict const vals)
+    {
+      reset();                  // punt
+      sum = vals[0].val.d;
+    }
+  };
+  
+  
+  
+  void* cycleclock_create(int const timernum)
+  {
+    return new cycleclock_t;
+  }
+  
+  void cycleclock_destroy(int const timernum, void* const data)
+  {
+    if (!data) return;
+    delete static_cast<cycleclock_t*>(data);
+  }
+  
+  void cycleclock_start(int const timernum, void* const data)
+  {
+    static_cast<cycleclock_t*>(data)->start();
+  }
+  
+  void cycleclock_stop(int const timernum, void* const data)
+  {
+    static_cast<cycleclock_t*>(data)->stop();
+  }
+  
+  void cycleclock_reset(int const timernum, void* const data)
+  {
+    static_cast<cycleclock_t*>(data)->reset();
+  }
+  
+  void cycleclock_get(int const timernum, void* const data,
+                      cTimerVal* const vals)
+  {
+    static_cast<cycleclock_t const*>(data)->get(vals);
+  }
+  
+  void cycleclock_set(int const timernum, void* const data,
+                      cTimerVal* const vals)
+  {
+    static_cast<cycleclock_t*>(data)->set(vals);
+  }
+  
+  void cycleclock_register()
+  {
+    cClockFuncs functions;
+    functions.n_vals  = 5;
+    functions.create  = cycleclock_create;
+    functions.destroy = cycleclock_destroy;
+    functions.start   = cycleclock_start;
+    functions.stop    = cycleclock_stop;
+    functions.reset   = cycleclock_reset;
+    functions.get     = cycleclock_get;
+    functions.set     = cycleclock_set;
+    CCTK_ClockRegister("cycle", &functions);
+  }
+  
+  
+  
+  extern "C"
+  int CycleClock_Setup()
+  {
+    measure_tick();
+    cycleclock_register();
+    return 0;
+  }
+  
+} // namespace CycleClock
diff --git a/Carpet/CycleClock/src/cycle.cc b/Carpet/CycleClock/src/cycle.cc
new file mode 100644
index 000000000..3a6b48364
--- /dev/null
+++ b/Carpet/CycleClock/src/cycle.cc
@@ -0,0 +1,108 @@
+#include "cycleclock.h"
+
+#include <cctk.h>
+
+
+
+// Find a good wall clock timer
+#ifdef _OPENMP
+#  include <omp.h>
+#endif
+#ifdef HAVE_CAPABILITY_MPI
+#  include <mpi.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+#endif
+
+
+
+static double cycleclock_tick = -1.0; // uninitialised
+
+
+
+#ifdef HAVE_SYS_TIME_H
+static double get_sys_time()
+{
+  timeval tp;
+  gettimeofday(&tp, NULL);
+  return tp.tv_sec + 1.0e-6 * tp.tv_usec;
+}
+#endif
+
+
+
+void measure_tick()
+{
+  // Make a few warm-up measurements
+  getticks();
+  getticks();
+  getticks();
+  
+#ifdef _OPENMP
+  if (cycleclock_tick < 0.0) {
+    // Use omp_get_sys_time to calibrate the timer
+    CCTK_INFO("Measuring CycleClock tick via OpenMP...");
+    ticks const rstart = getticks();
+    double const wstart = omp_get_wtime();
+    while (omp_get_wtime() < wstart + 1.0) {
+      // do nothing, just wait
+    }
+    ticks const rend = getticks();
+    double const wend = omp_get_wtime();
+    cycleclock_tick = (wend - wstart) / elapsed(rend, rstart);
+  }
+#endif
+  
+#ifdef HAVE_CAPABILITY_MPI
+  if (cycleclock_tick < 0.0) {
+    // Use MPI_Wtime to calibrate the timer
+    CCTK_INFO("Measuring CycleClock tick via MPI...");
+    ticks const rstart = getticks();
+    double const wstart = MPI_Wtime();
+    while (MPI_Wtime() < wstart + 1.0) {
+        // do nothing, just wait
+    }
+    ticks const rend = getticks();
+    double const wend = MPI_Wtime();
+    cycleclock_tick = (wend - wstart) / elapsed(rend, rstart);
+  }
+#endif
+  
+#ifdef HAVE_SYS_TIME_H
+  if (cycleclock_tick < 0.0) {
+    // Use gettimeofday to calibrate the timer
+    CCTK_INFO("Measuring CycleClock tick via gettimeofday...");
+    ticks const rstart = getticks();
+    double const wstart = get_sys_time();
+    while (get_sys_time() < wstart + 1.0) {
+      // do nothing, just wait
+    }
+    ticks const rend = getticks();
+    double const wend = get_sys_time();
+    cycleclock_tick = (wend - wstart) / elapsed(rend, rstart);
+  }
+#endif
+  
+  if (cycleclock_tick < 0.0) {
+    // Give up; just set the time scale to 1
+    CCTK_INFO("Could not measure CycleClock tick");
+    cycleclock_tick = 1.0;
+  }
+  
+  CCTK_VInfo(CCTK_THORNSTRING,
+             "Calibrated CycleClock: %g ns per clock tick (%g GHz)",
+             1.0e9 * cycleclock_tick, 1.0e-9 / cycleclock_tick);
+}
+
+
+
+double seconds_per_tick()
+{
+  // if (CCTK_BUILTIN_EXPECT(cycleclock_tick < 0.0, false)) {
+  //   CCTK_WARN(CCTK_WARN_ALERT,
+  //             "Thorn CycleClock has not been activated; the first measurements may be wrong");
+  //   measure_tick();
+  // }
+  return cycleclock_tick;
+}
diff --git a/Carpet/CycleClock/src/cycle.h b/Carpet/CycleClock/src/cycle.h
new file mode 100644
index 000000000..cbcf67f43
--- /dev/null
+++ b/Carpet/CycleClock/src/cycle.h
@@ -0,0 +1,513 @@
+/*
+ * Copyright (c) 2003, 2007-8 Matteo Frigo
+ * Copyright (c) 2003, 2007-8 Massachusetts Institute of Technology
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+
+/* machine-dependent cycle counters code. Needs to be inlined. */
+
+/***************************************************************************/
+/* To use the cycle counters in your code, simply #include "cycle.h" (this
+   file), and then use the functions/macros:
+
+                 ticks getticks(void);
+
+   ticks is an opaque typedef defined below, representing the current time.
+   You extract the elapsed time between two calls to gettick() via:
+
+                 double elapsed(ticks t1, ticks t0);
+
+   which returns a double-precision variable in arbitrary units.  You
+   are not expected to convert this into human units like seconds; it
+   is intended only for *comparisons* of time intervals.
+
+   (In order to use some of the OS-dependent timer routines like
+   Solaris' gethrtime, you need to paste the autoconf snippet below
+   into your configure.ac file and #include "config.h" before cycle.h,
+   or define the relevant macros manually if you are not using autoconf.)
+*/
+
+/***************************************************************************/
+/* This file uses macros like HAVE_GETHRTIME that are assumed to be
+   defined according to whether the corresponding function/type/header
+   is available on your system.  The necessary macros are most
+   conveniently defined if you are using GNU autoconf, via the tests:
+   
+   dnl ---------------------------------------------------------------------
+
+   AC_C_INLINE
+   AC_HEADER_TIME
+   AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h mach/mach_time.h])
+
+   AC_CHECK_TYPE([hrtime_t],[AC_DEFINE(HAVE_HRTIME_T, 1, [Define to 1 if hrtime_t is defined in <sys/time.h>])],,[#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif])
+
+   AC_CHECK_FUNCS([gethrtime read_real_time time_base_to_time clock_gettime mach_absolute_time])
+
+   dnl Cray UNICOS _rtc() (real-time clock) intrinsic
+   AC_MSG_CHECKING([for _rtc intrinsic])
+   rtc_ok=yes
+   AC_TRY_LINK([#ifdef HAVE_INTRINSICS_H
+#include <intrinsics.h>
+#endif], [_rtc()], [AC_DEFINE(HAVE__RTC,1,[Define if you have the UNICOS _rtc() intrinsic.])], [rtc_ok=no])
+   AC_MSG_RESULT($rtc_ok)
+
+   dnl ---------------------------------------------------------------------
+*/
+
+/***************************************************************************/
+
+#if TIME_WITH_SYS_TIME
+# include <sys/time.h>
+# include <time.h>
+#else
+# if HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# else
+#  include <time.h>
+# endif
+#endif
+
+#define INLINE_ELAPSED(INL) static INL double elapsed(ticks t1, ticks t0) \
+{									  \
+     return (double)t1 - (double)t0;					  \
+}
+
+/*----------------------------------------------------------------*/
+/* Solaris */
+#if defined(HAVE_GETHRTIME) && defined(HAVE_HRTIME_T) && !defined(HAVE_TICK_COUNTER)
+typedef hrtime_t ticks;
+
+#define getticks gethrtime
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* AIX v. 4+ routines to read the real-time clock or time-base register */
+#if defined(HAVE_READ_REAL_TIME) && defined(HAVE_TIME_BASE_TO_TIME) && !defined(HAVE_TICK_COUNTER)
+typedef timebasestruct_t ticks;
+
+static __inline ticks getticks(void)
+{
+     ticks t;
+     read_real_time(&t, TIMEBASE_SZ);
+     return t;
+}
+
+static __inline double elapsed(ticks t1, ticks t0) /* time in nanoseconds */
+{
+     time_base_to_time(&t1, TIMEBASE_SZ);
+     time_base_to_time(&t0, TIMEBASE_SZ);
+     return (((double)t1.tb_high - (double)t0.tb_high) * 1.0e9 + 
+	     ((double)t1.tb_low - (double)t0.tb_low));
+}
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * PowerPC ``cycle'' counter using the time base register.
+ */
+#if ((((defined(__GNUC__) && (defined(__powerpc__) || defined(__ppc__))) || (defined(__MWERKS__) && defined(macintosh)))) || (defined(__IBM_GCC_ASM) && (defined(__powerpc__) || defined(__ppc__))))  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     unsigned int tbl, tbu0, tbu1;
+
+     do {
+	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu0));
+	  __asm__ __volatile__ ("mftb %0" : "=r"(tbl));
+	  __asm__ __volatile__ ("mftbu %0" : "=r"(tbu1));
+     } while (tbu0 != tbu1);
+
+     return (((unsigned long long)tbu0) << 32) | tbl;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* MacOS/Mach (Darwin) time-base register interface (unlike UpTime,
+   from Carbon, requires no additional libraries to be linked). */
+#if defined(HAVE_MACH_ABSOLUTE_TIME) && defined(HAVE_MACH_MACH_TIME_H) && !defined(HAVE_TICK_COUNTER)
+#include <mach/mach_time.h>
+typedef uint64_t ticks;
+#define getticks mach_absolute_time
+INLINE_ELAPSED(__inline__)
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * Pentium cycle counter 
+ */
+#if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__)  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__("rdtsc": "=A" (ret));
+     /* no input, nothing else clobbered */
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#define TIME_MIN 5000.0   /* unreliable pentium IV cycle counter */
+#endif
+
+/* Visual C++ -- thanks to Morten Nissov for his help with this */
+#if _MSC_VER >= 1200 && _M_IX86 >= 500 && !defined(HAVE_TICK_COUNTER)
+#include <windows.h>
+typedef LARGE_INTEGER ticks;
+#define RDTSC __asm __emit 0fh __asm __emit 031h /* hack for VC++ 5.0 */
+
+static __inline ticks getticks(void)
+{
+     ticks retval;
+
+     __asm {
+	  RDTSC
+	  mov retval.HighPart, edx
+	  mov retval.LowPart, eax
+     }
+     return retval;
+}
+
+static __inline double elapsed(ticks t1, ticks t0)
+{  
+     return (double)t1.QuadPart - (double)t0.QuadPart;
+}  
+
+#define HAVE_TICK_COUNTER
+#define TIME_MIN 5000.0   /* unreliable pentium IV cycle counter */
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * X86-64 cycle counter
+ */
+#if (defined(__GNUC__) || defined(__ICC) || defined(__SUNPRO_C)) && defined(__x86_64__)  && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks a, d;
+     asm volatile("rdtsc" : "=a" (a), "=d" (d)); 
+     return a | (d << 32); 
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* PGI compiler, courtesy Cristiano Calonaci, Andrea Tarsi, & Roberto Gori.
+   NOTE: this code will fail to link unless you use the -Masmkeyword compiler
+   option (grrr). */
+#if defined(__PGI) && defined(__x86_64__) && !defined(HAVE_TICK_COUNTER) 
+typedef unsigned long long ticks;
+static ticks getticks(void)
+{
+    asm(" rdtsc; shl    $0x20,%rdx; mov    %eax,%eax; or     %rdx,%rax;    ");
+}
+INLINE_ELAPSED(__inline__)
+#define HAVE_TICK_COUNTER
+#endif
+
+/* Visual C++, courtesy of Dirk Michaelis */
+#if _MSC_VER >= 1400 && (defined(_M_AMD64) || defined(_M_X64)) && !defined(HAVE_TICK_COUNTER)
+
+#include <intrin.h>
+#pragma intrinsic(__rdtsc)
+typedef unsigned __int64 ticks;
+#define getticks __rdtsc
+INLINE_ELAPSED(__inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * IA64 cycle counter
+ */
+
+/* intel's icc/ecc compiler */
+#if (defined(__EDG_VERSION) || defined(__ECC)) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+#include <ia64intrin.h>
+
+static __inline__ ticks getticks(void)
+{
+     return __getReg(_IA64_REG_AR_ITC);
+}
+ 
+INLINE_ELAPSED(__inline__)
+ 
+#define HAVE_TICK_COUNTER
+#endif
+
+/* gcc */
+#if defined(__GNUC__) && defined(__ia64__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__ ("mov %0=ar.itc" : "=r"(ret));
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* HP/UX IA64 compiler, courtesy Teresa L. Johnson: */
+#if defined(__hpux) && defined(__ia64) && !defined(HAVE_TICK_COUNTER)
+#include <machine/sys/inline.h>
+typedef unsigned long ticks;
+
+static inline ticks getticks(void)
+{
+     ticks ret;
+
+     ret = _Asm_mov_from_ar (_AREG_ITC);
+     return ret;
+}
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/* Microsoft Visual C++ */
+#if defined(_MSC_VER) && defined(_M_IA64) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned __int64 ticks;
+
+#  ifdef __cplusplus
+extern "C"
+#  endif
+ticks __getReg(int whichReg);
+#pragma intrinsic(__getReg)
+
+static __inline ticks getticks(void)
+{
+     volatile ticks temp;
+     temp = __getReg(3116);
+     return temp;
+}
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/*
+ * PA-RISC cycle counter 
+ */
+#if defined(__hppa__) || defined(__hppa) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+#  ifdef __GNUC__
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+
+     __asm__ __volatile__("mfctl 16, %0": "=r" (ret));
+     /* no input, nothing else clobbered */
+     return ret;
+}
+#  else
+#  include <machine/inline.h>
+static inline unsigned long getticks(void)
+{
+     register ticks ret;
+     _MFCTL(16, ret);
+     return ret;
+}
+#  endif
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* S390, courtesy of James Treacy */
+#if defined(__GNUC__) && defined(__s390__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks cycles;
+     __asm__("stck 0(%0)" : : "a" (&(cycles)) : "memory", "cc");
+     return cycles;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+/*----------------------------------------------------------------*/
+#if defined(__GNUC__) && defined(__alpha__) && !defined(HAVE_TICK_COUNTER)
+/*
+ * The 32-bit cycle counter on alpha overflows pretty quickly, 
+ * unfortunately.  A 1GHz machine overflows in 4 seconds.
+ */
+typedef unsigned int ticks;
+
+static __inline__ ticks getticks(void)
+{
+     unsigned long cc;
+     __asm__ __volatile__ ("rpcc %0" : "=r"(cc));
+     return (cc & 0xFFFFFFFF);
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+#if defined(__GNUC__) && defined(__sparc_v9__) && !defined(HAVE_TICK_COUNTER)
+typedef unsigned long ticks;
+
+static __inline__ ticks getticks(void)
+{
+     ticks ret;
+     __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
+     return ret;
+}
+
+INLINE_ELAPSED(__inline__)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+#if (defined(__DECC) || defined(__DECCXX)) && defined(__alpha) && defined(HAVE_C_ASM_H) && !defined(HAVE_TICK_COUNTER)
+#  include <c_asm.h>
+typedef unsigned int ticks;
+
+static __inline ticks getticks(void)
+{
+     unsigned long cc;
+     cc = asm("rpcc %v0");
+     return (cc & 0xFFFFFFFF);
+}
+
+INLINE_ELAPSED(__inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+/*----------------------------------------------------------------*/
+/* SGI/Irix */
+#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) && !defined(HAVE_TICK_COUNTER)
+typedef struct timespec ticks;
+
+static inline ticks getticks(void)
+{
+     struct timespec t;
+     clock_gettime(CLOCK_SGI_CYCLE, &t);
+     return t;
+}
+
+static inline double elapsed(ticks t1, ticks t0)
+{
+     return ((double)t1.tv_sec - (double)t0.tv_sec) * 1.0E9 +
+	  ((double)t1.tv_nsec - (double)t0.tv_nsec);
+}
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* Cray UNICOS _rtc() intrinsic function */
+#if defined(HAVE__RTC) && !defined(HAVE_TICK_COUNTER)
+#ifdef HAVE_INTRINSICS_H
+#  include <intrinsics.h>
+#endif
+
+typedef long long ticks;
+
+#define getticks _rtc
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+
+/*----------------------------------------------------------------*/
+/* MIPS ZBus */
+#if HAVE_MIPS_ZBUS_TIMER
+#if defined(__mips__) && !defined(HAVE_TICK_COUNTER)
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+typedef uint64_t ticks;
+
+static inline ticks getticks(void)
+{
+  static uint64_t* addr = 0;
+
+  if (addr == 0)
+  {
+    uint32_t rq_addr = 0x10030000;
+    int fd;
+    int pgsize;
+
+    pgsize = getpagesize();
+    fd = open ("/dev/mem", O_RDONLY | O_SYNC, 0);
+    if (fd < 0) {
+      perror("open");
+      return NULL;
+    }
+    addr = mmap(0, pgsize, PROT_READ, MAP_SHARED, fd, rq_addr);
+    close(fd);
+    if (addr == (uint64_t *)-1) {
+      perror("mmap");
+      return NULL;
+    }
+  }
+
+  return *addr;
+}
+
+INLINE_ELAPSED(inline)
+
+#define HAVE_TICK_COUNTER
+#endif
+#endif /* HAVE_MIPS_ZBUS_TIMER */
diff --git a/Carpet/CycleClock/src/cycleclock.h b/Carpet/CycleClock/src/cycleclock.h
new file mode 100644
index 000000000..c985609f1
--- /dev/null
+++ b/Carpet/CycleClock/src/cycleclock.h
@@ -0,0 +1,32 @@
+#ifndef CYCLECLOCK_H
+#define CYCLECLOCK_H
+
+/* This defines:
+ *    typedef XXX ticks;
+ *    ticks getticks();
+ *    double elapsed(ticks t1, ticks t0);
+ * Use as:
+ *    #include <cycleclock.h>
+ *    ticks t0 = getticks();
+ *    ...
+ *    ticks t1 = getticks();
+ *    double elapsed_ticks = elapsed(t1, t0);
+ *    double elapsed_seconds = seconds_per_tick() * elapsed_ticks;
+*/
+
+//#include <cctk.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  
+#include "cycle.h"
+  
+  double seconds_per_tick();
+  void measure_tick();
+  
+#ifdef __cplusplus
+}
+#endif
+
+#endif  /* CYCLECLOCK_H */
diff --git a/Carpet/CycleClock/src/make.code.defn b/Carpet/CycleClock/src/make.code.defn
new file mode 100644
index 000000000..eb3a889e6
--- /dev/null
+++ b/Carpet/CycleClock/src/make.code.defn
@@ -0,0 +1,7 @@
+# Main make.code.defn file for thorn CycleClock
+
+# Source files in this directory
+SRCS = clock.cc cycle.cc
+
+# Subdirectories containing source files
+SUBDIRS =