CarpetLib: Collect more timing information

Extend the CarpetLib timers so that they also count the number of
bytes transmitted in addition to the wall time.

darcs-hash:20060904020649-dae7b-7a18c2fbe2de09d662b4154a3ae2e553ad57c12b.gz

4 files changed, 53 insertions, 20 deletions

diff --git a/Carpet/CarpetLib/src/data.cc b/Carpet/CarpetLib/src/data.cc
index 8b12ea4cc..ae4249048 100644
--- a/Carpet/CarpetLib/src/data.cc
+++ b/Carpet/CarpetLib/src/data.cc
@@ -165,7 +165,7 @@ void data<T>::change_processor_recv (comm_state& state,
       MPI_Irecv (_memory->storage(0),
                  _size, dist::datatype(dummy), proc(),
                  tag, dist::comm(), &request);
-      wtime_irecv.stop();
+      wtime_irecv.stop(_size * sizeof(T));
       if (use_waitall) {
         state.requests.push_back (request);
       }
@@ -217,7 +217,7 @@ void data<T>::change_processor_send (comm_state& state,
       MPI_Isend (_memory->storage(0),
                  _size, dist::datatype(dummy), newproc,
                  tag, dist::comm(), &request);
-      wtime_isend.stop();
+      wtime_isend.stop(_size * sizeof(T));
       if (use_waitall) {
         state.requests.push_back (request);
       }
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index 428eb6871..d31d917d2 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -177,12 +177,14 @@ void gdata::copy_from_post (comm_state& state,
 
     wtime_copyfrom_recvinner_allocate.start();
     comm_state::gcommbuf * b = make_typed_commbuf (box);
-    wtime_copyfrom_recvinner_allocate.stop();
+    int typesize;
+    MPI_Type_size (b->datatype(), & typesize);
+    wtime_copyfrom_recvinner_allocate.stop(b->size() * typesize);
 
     wtime_copyfrom_recvinner_recv.start();
     MPI_Irecv (b->pointer(), b->size(), b->datatype(), src->proc(),
                tag, dist::comm(), &b->request);
-    wtime_copyfrom_recvinner_recv.stop();
+    wtime_copyfrom_recvinner_recv.stop(b->size() * typesize);
     state.requests.push_back (b->request);
     state.recvbufs.push (b);
 
@@ -191,7 +193,9 @@ void gdata::copy_from_post (comm_state& state,
 
     wtime_copyfrom_sendinner_allocate.start();
     comm_state::gcommbuf * b = src->make_typed_commbuf (box);
-    wtime_copyfrom_sendinner_allocate.stop();
+    int typesize;
+    MPI_Type_size (b->datatype(), & typesize);
+    wtime_copyfrom_sendinner_allocate.stop(b->size() * typesize);
 
     // copy data into send buffer
     wtime_copyfrom_sendinner_copy.start();
@@ -202,20 +206,22 @@ void gdata::copy_from_post (comm_state& state,
     char* send_buffer = (char*) b->pointer();
     int& datatypesize = state.typebufs.at(c_datatype()).datatypesize;
 
+    double bytes = 0;
     for (int k = 0; k < items[2]; k++) {
       for (int j = 0; j < items[1]; j++) {
         int i = offs[0] + myshape[0]*((j+offs[1]) + myshape[1]*(k+offs[2]));
         memcpy (send_buffer, ((char*) src->storage()) + datatypesize*i,
                 datatypesize * items[0]);
         send_buffer += datatypesize * items[0];
+        bytes += datatypesize * items[0];
       }
     }
-    wtime_copyfrom_sendinner_copy.stop();
+    wtime_copyfrom_sendinner_copy.stop(bytes);
 
     wtime_copyfrom_sendinner_send.start();
     MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
                tag, dist::comm(), &b->request);
-    wtime_copyfrom_sendinner_send.stop();
+    wtime_copyfrom_sendinner_send.stop(b->size() * typesize);
     state.requests.push_back (b->request);
     state.sendbufs.push (b);
   }
@@ -316,7 +322,7 @@ void gdata::copy_into_sendbuffer (comm_state& state,
                  state.typebufs.at(c_datatype()).mpi_datatype,
                  proc(), c_datatype(), dist::comm(),
                  &state.srequests.at(dist::size()*c_datatype() + proc()));
-      wtime_commstate_isend.stop();
+      wtime_commstate_isend.stop(procbuf.sendbufsize * datatypesize);
     }
   }
 }
@@ -340,6 +346,7 @@ void gdata::copy_from_recvbuffer (comm_state& state,
   ivect offs  = (box.lower() - ext.lower()) / ext.stride();
 
   wtime_commstate_memcpy.start();
+  double bytes = 0;
   assert (dim == 3);
   for (int k = 0; k < items[2]; k++) {
     for (int j = 0; j < items[1]; j++) {
@@ -347,9 +354,10 @@ void gdata::copy_from_recvbuffer (comm_state& state,
       memcpy (((char*) storage()) + datatypesize*i,
               procbuf.recvbuf, datatypesize * items[0]);
       procbuf.recvbuf += datatypesize * items[0];
+      bytes += datatypesize * items[0];
     }
   }
-  wtime_commstate_memcpy.stop();
+  wtime_commstate_memcpy.stop(bytes);
 }
 
 
@@ -455,17 +463,21 @@ void gdata
     // this processor receives data
 
     comm_state::gcommbuf * b = make_typed_commbuf (box);
+    int typesize;
+    MPI_Type_size (b->datatype(), & typesize);
 
     wtime_commstate_interpolate_irecv.start();
     MPI_Irecv (b->pointer(), b->size(), b->datatype(), src->proc(),
                tag, dist::comm(), &b->request);
-    wtime_commstate_interpolate_irecv.stop();
+    wtime_commstate_interpolate_irecv.stop(b->size() * typesize);
     state.requests.push_back (b->request);
     state.recvbufs.push (b);
   } else {
     // this processor sends data
 
     comm_state::gcommbuf * b = src->make_typed_commbuf (box);
+    int typesize;
+    MPI_Type_size (b->datatype(), & typesize);
 
     gdata * tmp = src->make_typed (varindex, transport_operator, tag);
     tmp->allocate (box, src->proc(), b->pointer());
@@ -476,7 +488,7 @@ void gdata
     wtime_commstate_interpolate_from_isend.start();
     MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
                tag, dist::comm(), &b->request);
-    wtime_commstate_interpolate_from_isend.stop();
+    wtime_commstate_interpolate_from_isend.stop(b->size() * typesize);
     state.requests.push_back (b->request);
     state.sendbufs.push (b);
   }
@@ -529,7 +541,7 @@ void gdata
                  state.typebufs.at(c_datatype()).mpi_datatype,
                  proc(), c_datatype(), dist::comm(),
                  &state.srequests.at(dist::size()*c_datatype() + proc()));
-      wtime_commstate_interpolate_to_isend.stop();
+      wtime_commstate_interpolate_to_isend.stop(procbuf.sendbufsize*datatypesize);
     }
   }
 }
diff --git a/Carpet/CarpetLib/src/timestat.cc b/Carpet/CarpetLib/src/timestat.cc
index f5116481f..88987742f 100644
--- a/Carpet/CarpetLib/src/timestat.cc
+++ b/Carpet/CarpetLib/src/timestat.cc
@@ -18,17 +18,25 @@ using namespace std;
 
 
 timestat::timestat ()
-  : wtime(0.0), wtime2(0.0), wmin(0.0), wmax(0.0), count(0.0),
+  : wtime(0.0), wtime2(0.0), wmin(0.0), wmax(0.0),
+    bytes(0.0), bytes2(0.0), bmin(0.0), bmax(0.0),
+    count(0.0),
     running(false)
 {
 }
 
-void timestat::addstat (double const t)
+void timestat::addstat (double const t, double const b)
 {
   wtime += t;
   wtime2 += t*t;
   wmin = min (wmin, t);
   wmax = max (wmax, t);
+  
+  bytes += b;
+  bytes2 += b*b;
+  bmin = min (bmin, b);
+  bmax = max (bmax, b);
+  
   ++count;
 }
 
@@ -39,12 +47,12 @@ void timestat::start ()
   starttime = MPI_Wtime();
 }
 
-void timestat::stop ()
+void timestat::stop (double const b)
 {
   assert (running);
   running = false;
   double const endtime = MPI_Wtime();
-  addstat (endtime - starttime);
+  addstat (endtime - starttime, b);
 }
 
 
@@ -53,13 +61,20 @@ ostream& operator<< (ostream& os, const timestat& wt)
 {
   double const avg = wt.wtime / wt.count;
   double const stddev = sqrt (max (0.0, wt.wtime2 / wt.count - pow (avg, 2)));
+  double const bavg = wt.bytes / wt.count;
+  double const bstddev = sqrt (max (0.0, wt.bytes2 / wt.count - pow (bavg, 2)));
   os << "timestat[seconds]:"
      << " cnt: " << wt.count
-     << " sum: " << wt.wtime
+     << "   time: sum: " << wt.wtime
      << " avg: " << avg
      << " stddev: " << stddev
      << " min: " << wt.wmin
-     << " max: " << wt.wmax;
+     << " max: " << wt.wmax
+     << "   bytes: sum: " << wt.bytes
+     << " avg: " << bavg
+     << " stddev: " << bstddev
+     << " min: " << wt.bmin
+     << " max: " << wt.bmax;
   return os;
 }
 
diff --git a/Carpet/CarpetLib/src/timestat.hh b/Carpet/CarpetLib/src/timestat.hh
index 896b94236..a93e5717a 100644
--- a/Carpet/CarpetLib/src/timestat.hh
+++ b/Carpet/CarpetLib/src/timestat.hh
@@ -15,13 +15,19 @@ public:
   double wtime2;
   double wmin;
   double wmax;
+  
+  double bytes;
+  double bytes2;
+  double bmin;
+  double bmax;
+  
   double count;
   
 public:
   timestat ();
   
 private:
-  void addstat (double const t);
+  void addstat (double t, double b);
   
 private:
   bool running;
@@ -29,7 +35,7 @@ private:
   
 public:
   void start();
-  void stop();
+  void stop(double bytes = 0.0);
 };
 
 ostream& operator<< (ostream& os, const timestat& wt);