CarpetLib: Restructure lightweight communication buffers

Restructure the lightweight communication buffers.
Use lightweight communication buffers for interpolation as well.

darcs-hash:20050103200712-891bb-7e42816d3b8d667916084e3f32527c8f35327d7f.gz

6 files changed, 407 insertions, 67 deletions

diff --git a/Carpet/CarpetLib/src/commstate.cc b/Carpet/CarpetLib/src/commstate.cc
index 8b81ce428..865ab6c7f 100644
--- a/Carpet/CarpetLib/src/commstate.cc
+++ b/Carpet/CarpetLib/src/commstate.cc
@@ -1,6 +1,11 @@
 #include "cctk.h"
 #include "cctk_Parameters.h"
 
+#include "bbox.hh"
+#include "defs.hh"
+#include "dist.hh"
+#include "vect.hh"
+
 #include "commstate.hh"
 
 
@@ -69,3 +74,68 @@ comm_state::~comm_state ()
   assert (tmps2.empty());
   assert (requests.empty());
 }
+
+
+
+comm_state::gcommbuf::gcommbuf ()
+{
+}
+
+comm_state::gcommbuf::~gcommbuf ()
+{
+}
+
+
+
+
+template<typename T>
+comm_state::commbuf<T>::commbuf (ibbox const & box)
+{
+  data.resize (prod (box.shape() / box.stride()));
+}
+
+template<typename T>
+comm_state::commbuf<T>::~commbuf ()
+{
+}
+
+template<typename T>
+void const *
+comm_state::commbuf<T>::pointer ()
+  const
+{
+  return &data.front();
+}
+
+template<typename T>
+void *
+comm_state::commbuf<T>::pointer ()
+{
+  return &data.front();
+}
+
+template<typename T>
+int
+comm_state::commbuf<T>::size ()
+  const
+{
+  return data.size();
+}
+
+template<typename T>
+MPI_Datatype
+comm_state::commbuf<T>::datatype ()
+  const
+{
+  T dummy;
+  return dist::datatype (dummy);
+}
+
+
+
+#define INSTANTIATE(T)                          \
+  template class comm_state::commbuf<T>;
+
+#include "instantiate"
+
+#undef INSTANTIATE
diff --git a/Carpet/CarpetLib/src/commstate.hh b/Carpet/CarpetLib/src/commstate.hh
index 5a9570c7c..16981e767 100644
--- a/Carpet/CarpetLib/src/commstate.hh
+++ b/Carpet/CarpetLib/src/commstate.hh
@@ -41,11 +41,22 @@ public:
   
   // Lists of communication buffers for use_lightweight_buffers
   struct gcommbuf {
-    bool am_receiver, am_sender;
+    gcommbuf ();
+    virtual ~gcommbuf ();
     MPI_Request request;
+    virtual void const * pointer () const = 0;
+    virtual void * pointer () = 0;
+    virtual int size () const = 0;
+    virtual MPI_Datatype datatype () const = 0;
   };
   template<typename T>
   struct commbuf : gcommbuf {
+    commbuf (ibbox const & box);
+    virtual ~commbuf ();
+    virtual void const * pointer () const;
+    virtual void * pointer ();
+    virtual int size () const;
+    virtual MPI_Datatype datatype () const;
     vector<T> data;
   };
   queue<gcommbuf*> recvbufs, sendbufs;
diff --git a/Carpet/CarpetLib/src/data.cc b/Carpet/CarpetLib/src/data.cc
index 5ff5c28d8..717589a31 100644
--- a/Carpet/CarpetLib/src/data.cc
+++ b/Carpet/CarpetLib/src/data.cc
@@ -38,8 +38,9 @@ static const CCTK_REAL eps = 1.0e-10;
 template<typename T>
 data<T>::data (const int varindex_, const operator_type transport_operator_,
                const int vectorlength_, const int vectorindex_,
-               data* const vectorleader_)
-  : gdata(varindex_, transport_operator_),
+               data* const vectorleader_,
+               const int tag_)
+  : gdata(varindex_, transport_operator_, tag_),
     _storage(NULL), _allocated_bytes(0),
     vectorlength(vectorlength_), vectorindex(vectorindex_),
     vectorleader(vectorleader_)
@@ -83,10 +84,11 @@ data<T>::~data ()
 // Pseudo constructors
 template<typename T>
 data<T>* data<T>::make_typed (const int varindex_,
-                              const operator_type transport_operator_)
+                              const operator_type transport_operator_,
+                              const int tag_)
   const
 {
-  return new data(varindex_, transport_operator_);
+  return new data(varindex_, transport_operator_, 1, 0, NULL, tag_);
 }
 
 
@@ -398,6 +400,7 @@ void data<T>::change_processor_wait (comm_state& state,
 
 
 
+#if 0
 template<typename T>
 void
 data<T>::copy_from_recv_inner (comm_state& state,
@@ -424,9 +427,10 @@ data<T>::copy_from_recv_inner (comm_state& state,
   }
   state.recvbufs.push (b);
 }
+#endif
 
 
-
+#if 0
 template<typename T>
 void
 data<T>::copy_from_send_inner (comm_state& state,
@@ -435,10 +439,9 @@ data<T>::copy_from_send_inner (comm_state& state,
   DECLARE_CCTK_PARAMETERS;
   
   wtime_copyfrom_sendinner_allocate.start();
-  comm_state::commbuf<T> * b = new comm_state::commbuf<T>;
+  comm_state::gcommbuf * b = gsrc->make_typed_commbuf (box);
   b->am_receiver = false;
   b->am_sender = true;
-  b->data.resize (prod (box.shape() / box.stride()));
   wtime_copyfrom_sendinner_allocate.stop();
   
   wtime_copyfrom_sendinner_copy.start();
@@ -446,6 +449,7 @@ data<T>::copy_from_send_inner (comm_state& state,
   assert (src->_has_storage);
   assert (src->_owns_storage);
   // copy src to b
+#if 0
   {
     T * restrict p = & b->data.front();
     T const * restrict const q = src->_storage;
@@ -461,12 +465,19 @@ data<T>::copy_from_send_inner (comm_state& state,
       }
     }
   }
+#endif
+  {
+    data<T> * tmp = src->make_typed (varindex, transport_operator, tag);
+    tmp->allocate (box, src->proc(), &b->data.front());
+    tmp->copy_from_innerloop (src, box);
+    delete tmp;
+  }
   wtime_copyfrom_sendinner_copy.stop();
   
   wtime_copyfrom_sendinner_send.start();
   assert (dist::rank() == src->proc());
   T dummy;
-  MPI_Isend (&b->data.front(), b->data.size(), dist::datatype(dummy), proc(),
+  MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
              tag, dist::comm, &b->request);
   wtime_copyfrom_sendinner_send.stop();
   if (use_waitall) {
@@ -474,9 +485,11 @@ data<T>::copy_from_send_inner (comm_state& state,
   }
   state.sendbufs.push (b);
 }
+#endif
 
 
 
+#if 0
 template<typename T>
 void
 data<T>::copy_from_recv_wait_inner (comm_state& state,
@@ -531,9 +544,11 @@ data<T>::copy_from_recv_wait_inner (comm_state& state,
   delete b;
   wtime_copyfrom_recvwaitinner_delete.stop();
 }
+#endif
 
 
 
+#if 0
 template<typename T>
 void
 data<T>::copy_from_send_wait_inner (comm_state& state,
@@ -567,11 +582,23 @@ data<T>::copy_from_send_wait_inner (comm_state& state,
   delete b;
   wtime_copyfrom_sendwaitinner_delete.stop();
 }
+#endif
 
 
 
 // Data manipulators
 template<typename T>
+comm_state::gcommbuf *
+data<T>::
+make_typed_commbuf (const ibbox & box)
+  const
+{
+  return new comm_state::commbuf<T> (box);
+}
+
+
+
+template<typename T>
 void data<T>
 ::copy_from_innerloop (const gdata* gsrc, const ibbox& box)
 {
@@ -609,6 +636,35 @@ void data<T>
 
 
 
+template<class T>
+void data<T>
+::fill_bbox_arrays (int srcshp[dim],
+                    int dstshp[dim],
+                    int srcbbox[dim][dim],
+                    int dstbbox[dim][dim],
+                    int regbbox[dim][dim],
+                    const ibbox & box,
+                    const ibbox & sext,
+                    const ibbox & dext)
+{
+  for (int d=0; d<dim; ++d) {
+    srcshp[d] = (sext.shape() / sext.stride())[d];
+    dstshp[d] = (dext.shape() / dext.stride())[d];
+    
+    srcbbox[0][d] = sext.lower()[d];
+    srcbbox[1][d] = sext.upper()[d];
+    srcbbox[2][d] = sext.stride()[d];
+    
+    dstbbox[0][d] = dext.lower()[d];
+    dstbbox[1][d] = dext.upper()[d];
+    dstbbox[2][d] = dext.stride()[d];
+    
+    regbbox[0][d] = box.lower()[d];
+    regbbox[1][d] = box.upper()[d];
+    regbbox[2][d] = box.stride()[d];
+  }
+}
+
 template<typename T>
 void data<T>
 ::interpolate_from_innerloop (const vector<const gdata*> gsrcs,
@@ -676,35 +732,6 @@ extern "C" {
      const int regbbox[3][3]);
 }
 
-template<class T>
-void data<T>
-::fill_bbox_arrays (int srcshp[dim],
-                    int dstshp[dim],
-                    int srcbbox[dim][dim],
-                    int dstbbox[dim][dim],
-                    int regbbox[dim][dim],
-                    const ibbox & box,
-                    const ibbox & sext,
-                    const ibbox & dext)
-{
-  for (int d=0; d<dim; ++d) {
-    srcshp[d] = (sext.shape() / sext.stride())[d];
-    dstshp[d] = (dext.shape() / dext.stride())[d];
-    
-    srcbbox[0][d] = sext.lower()[d];
-    srcbbox[1][d] = sext.upper()[d];
-    srcbbox[2][d] = sext.stride()[d];
-    
-    dstbbox[0][d] = dext.lower()[d];
-    dstbbox[1][d] = dext.upper()[d];
-    dstbbox[2][d] = dext.stride()[d];
-    
-    regbbox[0][d] = box.lower()[d];
-    regbbox[1][d] = box.upper()[d];
-    regbbox[2][d] = box.stride()[d];
-  }
-}
-
 template<>
 void data<CCTK_INT4>
 ::copy_from_innerloop (const gdata* gsrc, const ibbox& box)
diff --git a/Carpet/CarpetLib/src/data.hh b/Carpet/CarpetLib/src/data.hh
index 9f0d9b6dd..7d7eed414 100644
--- a/Carpet/CarpetLib/src/data.hh
+++ b/Carpet/CarpetLib/src/data.hh
@@ -43,7 +43,8 @@ public:
   data (const int varindex = -1,
         const operator_type transport_operator = op_error,
         const int vectorlength = 1, const int vectorindex = 0,
-        data* const vectorleader = NULL);
+        data* const vectorleader = NULL,
+        const int tag = -1);
   data (const int varindex, const operator_type transport_operator,
         const int vectorlength, const int vectorindex,
         data* const vectorleader,
@@ -54,7 +55,8 @@ public:
 
   // Pseudo constructors
   virtual data* make_typed (const int varindex,
-                            const operator_type transport_operator) const;
+                            const operator_type transport_operator,
+                            const int tag) const;
 
   // Storage management
 private:
@@ -108,6 +110,7 @@ public:
     return _storage[offset(index)];
   }
 
+#if 0
 protected:
   virtual void
   copy_from_recv_inner (comm_state& state,
@@ -125,9 +128,14 @@ protected:
   copy_from_send_wait_inner (comm_state& state,
                              const gdata* src,
                              const ibbox& box);
+#endif
   
   // Data manipulators
 private:
+  virtual comm_state::gcommbuf *
+  make_typed_commbuf (const ibbox & box)
+    const;
+  
   static void
   fill_bbox_arrays (int srcshp[dim],
                     int dstshp[dim],
diff --git a/Carpet/CarpetLib/src/gdata.cc b/Carpet/CarpetLib/src/gdata.cc
index afd56a5dd..a02dc7d58 100644
--- a/Carpet/CarpetLib/src/gdata.cc
+++ b/Carpet/CarpetLib/src/gdata.cc
@@ -2,6 +2,8 @@
 #include <cstdlib>
 #include <iostream>
 
+#include <mpi.h>
+
 #include "cctk.h"
 #include "cctk_Parameters.h"
 
@@ -35,11 +37,12 @@ static int nexttag ()
 
 
 // Constructors
-gdata::gdata (const int varindex_, const operator_type transport_operator_)
+gdata::gdata (const int varindex_, const operator_type transport_operator_,
+              const int tag_)
   : varindex(varindex_), transport_operator(transport_operator_),
     _has_storage(false),
     comm_active(false),
-    tag(nexttag())
+    tag(tag_ >= 0 ? tag_ : nexttag())
 {
   DECLARE_CCTK_PARAMETERS;
   if (barriers) {
@@ -177,8 +180,8 @@ void gdata::copy_from_recv (comm_state& state,
     // copy on same processor
     
   } else {
-    
     // copy to different processor
+    
     if (! use_lightweight_buffers) {
       
       wtime_copyfrom_recv_maketyped.start();
@@ -196,7 +199,21 @@ void gdata::copy_from_recv (comm_state& state,
       
       if (dist::rank() == proc()) {
         // this processor receives data
-        copy_from_recv_inner (state, src, box);
+        
+        wtime_copyfrom_recvinner_allocate.start();
+        comm_state::gcommbuf * b = make_typed_commbuf (box);
+        wtime_copyfrom_recvinner_allocate.stop();
+        
+        wtime_copyfrom_recvinner_recv.start();
+        assert (dist::rank() == proc());
+        MPI_Irecv (b->pointer(), b->size(), b->datatype(), src->proc(),
+                   tag, dist::comm, &b->request);
+        wtime_copyfrom_recvinner_recv.stop();
+        if (use_waitall) {
+          state.requests.push_back (b->request);
+        }
+        state.recvbufs.push (b);
+        
       }
       
     }
@@ -235,8 +252,8 @@ void gdata::copy_from_send (comm_state& state,
     wtime_copyfrom_send_copyfrom_nocomm1.stop();
     
   } else {
-    
     // copy to different processor
+    
     if (! use_lightweight_buffers) {
       
       gdata* const tmp = state.tmps1.front();
@@ -254,7 +271,30 @@ void gdata::copy_from_send (comm_state& state,
       
       if (dist::rank() == src->proc()) {
         // this processor sends data
-        copy_from_send_inner (state, src, box);
+        
+        wtime_copyfrom_sendinner_allocate.start();
+        comm_state::gcommbuf * b = src->make_typed_commbuf (box);
+        wtime_copyfrom_sendinner_allocate.stop();
+        
+        wtime_copyfrom_sendinner_copy.start();
+        assert (src->_has_storage);
+        assert (src->_owns_storage);
+        gdata * tmp = src->make_typed (varindex, transport_operator, tag);
+        tmp->allocate (box, src->proc(), b->pointer());
+        tmp->copy_from_innerloop (src, box);
+        delete tmp;
+        wtime_copyfrom_sendinner_copy.stop();
+        
+        wtime_copyfrom_sendinner_send.start();
+        assert (dist::rank() == src->proc());
+        MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
+                   tag, dist::comm, &b->request);
+        wtime_copyfrom_sendinner_send.stop();
+        if (use_waitall) {
+          state.requests.push_back (b->request);
+        }
+        state.sendbufs.push (b);
+        
       }
     }
     
@@ -288,8 +328,8 @@ void gdata::copy_from_wait (comm_state& state,
     // copy on same processor
     
   } else {
-    
     // copy to different processor
+    
     if (! use_lightweight_buffers) {
       
       gdata* const tmp = state.tmps2.front();
@@ -309,11 +349,67 @@ void gdata::copy_from_wait (comm_state& state,
       
       if (dist::rank() == proc()) {
         // this processor receives data
-        copy_from_recv_wait_inner (state, src, box);
+        
+        comm_state::gcommbuf * b = state.recvbufs.front();
+        state.recvbufs.pop();
+        
+        wtime_copyfrom_recvwaitinner_wait.start();
+        if (use_waitall) {
+          if (! state.requests.empty()) {
+            // wait for all requests at once
+            MPI_Waitall (state.requests.size(), &state.requests.front(),
+                         MPI_STATUSES_IGNORE);
+            state.requests.clear();
+          }
+        }
+        
+        if (! use_waitall) {
+          MPI_Status status;
+          MPI_Wait (&b->request, &status);
+        }
+        wtime_copyfrom_recvwaitinner_wait.stop();
+        
+        wtime_copyfrom_recvwaitinner_copy.start();
+        assert (_has_storage);
+        assert (_owns_storage);
+        gdata * tmp = make_typed (varindex, transport_operator, tag);
+        tmp->allocate (box, proc(), b->pointer());
+        copy_from_innerloop (tmp, box);
+        delete tmp;
+        wtime_copyfrom_recvwaitinner_copy.stop();
+        
+        wtime_copyfrom_recvwaitinner_delete.start();
+        delete b;
+        wtime_copyfrom_recvwaitinner_delete.stop();
+        
       }
+      
       if (dist::rank() == src->proc()) {
         // this processor sends data
-        copy_from_send_wait_inner (state, src, box);
+        
+        comm_state::gcommbuf * b = state.sendbufs.front();
+        state.sendbufs.pop();
+        
+        wtime_copyfrom_sendwaitinner_wait.start();
+        if (use_waitall) {
+          if (! state.requests.empty()) {
+            // wait for all requests at once
+            MPI_Waitall (state.requests.size(), &state.requests.front(),
+                         MPI_STATUSES_IGNORE);
+            state.requests.clear();
+          }
+        }
+        
+        if (! use_waitall) {
+          MPI_Status status;
+          MPI_Wait (&b->request, &status);
+        }
+        wtime_copyfrom_sendwaitinner_wait.stop();
+        
+        wtime_copyfrom_sendwaitinner_delete.start();
+        delete b;
+        wtime_copyfrom_sendwaitinner_delete.stop();
+        
       }
       
     }
@@ -407,6 +503,8 @@ void gdata
                          const int order_space,
                          const int order_time)
 {
+  DECLARE_CCTK_PARAMETERS;
+  
   assert (has_storage());
   assert (all(box.lower()>=extent().lower()));
   assert (all(box.upper()<=extent().upper()));
@@ -428,10 +526,31 @@ void gdata
   } else {
     // interpolate from other processor
     
-    gdata* const tmp = make_typed(varindex, transport_operator);
-    state.tmps1.push (tmp);
-    tmp->allocate (box, srcs.at(0)->proc());
-    tmp->change_processor_recv (state, proc());
+    if (! use_lightweight_buffers) {
+      
+      gdata* const tmp = make_typed(varindex, transport_operator);
+      state.tmps1.push (tmp);
+      tmp->allocate (box, srcs.at(0)->proc());
+      tmp->change_processor_recv (state, proc());
+      
+    } else {
+      
+      if (dist::rank() == proc()) {
+        // this processor receives data
+        
+        comm_state::gcommbuf * b = make_typed_commbuf (box);
+        
+        assert (dist::rank() == proc());
+        MPI_Irecv (b->pointer(), b->size(), b->datatype(), srcs.at(0)->proc(),
+                   tag, dist::comm, &b->request);
+        if (use_waitall) {
+          state.requests.push_back (b->request);
+        }
+        state.recvbufs.push (b);
+        
+      }
+      
+    }
     
   }
 }
@@ -446,6 +565,8 @@ void gdata
                          const int order_space,
                          const int order_time)
 {
+  DECLARE_CCTK_PARAMETERS;
+  
   assert (has_storage());
   assert (all(box.lower()>=extent().lower()));
   assert (all(box.upper()<=extent().upper()));
@@ -469,12 +590,43 @@ void gdata
   } else {
     // interpolate from other processor
     
-    gdata* const tmp = state.tmps1.front();
-    state.tmps1.pop();
-    state.tmps2.push (tmp);
-    assert (tmp);
-    tmp->interpolate_from_nocomm (srcs, times, box, time, order_space, order_time);
-    tmp->change_processor_send (state, proc());
+    if (! use_lightweight_buffers) {
+      
+      gdata* const tmp = state.tmps1.front();
+      state.tmps1.pop();
+      state.tmps2.push (tmp);
+      assert (tmp);
+      tmp->interpolate_from_nocomm
+        (srcs, times, box, time, order_space, order_time);
+      tmp->change_processor_send (state, proc());
+      
+    } else {
+      
+      if (dist::rank() == srcs.at(0)->proc()) {
+        // this processor sends data
+        
+        comm_state::gcommbuf * b = srcs.at(0)->make_typed_commbuf (box);
+        
+        assert (srcs.at(0)->_has_storage);
+        assert (srcs.at(0)->_owns_storage);
+        gdata * tmp
+          = srcs.at(0)->make_typed (varindex, transport_operator, tag);
+        tmp->allocate (box, srcs.at(0)->proc(), b->pointer());
+        tmp->interpolate_from_innerloop
+          (srcs, times, box, time, order_space, order_time);
+        delete tmp;
+        
+        assert (dist::rank() == srcs.at(0)->proc());
+        MPI_Isend (b->pointer(), b->size(), b->datatype(), proc(),
+                   tag, dist::comm, &b->request);
+        if (use_waitall) {
+          state.requests.push_back (b->request);
+        }
+        state.sendbufs.push (b);
+
+      }
+      
+    }
     
   }
 }
@@ -489,6 +641,8 @@ void gdata
                          const int order_space,
                          const int order_time)
 {
+  DECLARE_CCTK_PARAMETERS;
+  
   assert (has_storage());
   assert (all(box.lower()>=extent().lower()));
   assert (all(box.upper()<=extent().upper()));
@@ -510,12 +664,73 @@ void gdata
   } else {
     // interpolate from other processor
     
-    gdata* const tmp = state.tmps2.front();
-    state.tmps2.pop();
-    assert (tmp);
-    tmp->change_processor_wait (state, proc());
-    copy_from_nocomm (tmp, box);
-    delete tmp;
+    if (! use_lightweight_buffers) {
+      
+      gdata* const tmp = state.tmps2.front();
+      state.tmps2.pop();
+      assert (tmp);
+      tmp->change_processor_wait (state, proc());
+      copy_from_nocomm (tmp, box);
+      delete tmp;
+      
+    } else {
+      
+      if (dist::rank() == proc()) {
+        // this processor receives data
+        
+        comm_state::gcommbuf * b = state.recvbufs.front();
+        state.recvbufs.pop();
+        
+        if (use_waitall) {
+          if (! state.requests.empty()) {
+            // wait for all requests at once
+            MPI_Waitall (state.requests.size(), &state.requests.front(),
+                         MPI_STATUSES_IGNORE);
+            state.requests.clear();
+          }
+        }
+        
+        if (! use_waitall) {
+          MPI_Status status;
+          MPI_Wait (&b->request, &status);
+        }
+        
+        assert (_has_storage);
+        assert (_owns_storage);
+        gdata * tmp = make_typed (varindex, transport_operator, tag);
+        tmp->allocate (box, proc(), b->pointer());
+        copy_from_innerloop (tmp, box);
+        delete tmp;
+        
+        delete b;
+        
+      }
+      
+      if (dist::rank() == srcs.at(0)->proc()) {
+        // this processor sends data
+        
+        comm_state::gcommbuf * b = state.sendbufs.front();
+        state.sendbufs.pop();
+        
+        if (use_waitall) {
+          if (! state.requests.empty()) {
+            // wait for all requests at once
+            MPI_Waitall (state.requests.size(), &state.requests.front(),
+                         MPI_STATUSES_IGNORE);
+            state.requests.clear();
+          }
+        }
+        
+        if (! use_waitall) {
+          MPI_Status status;
+          MPI_Wait (&b->request, &status);
+        }
+        
+        delete b;
+        
+      }
+      
+    }
     
   }
 }
diff --git a/Carpet/CarpetLib/src/gdata.hh b/Carpet/CarpetLib/src/gdata.hh
index e6c9434f9..7315bc763 100644
--- a/Carpet/CarpetLib/src/gdata.hh
+++ b/Carpet/CarpetLib/src/gdata.hh
@@ -52,7 +52,8 @@ public:
 
   // Constructors
   gdata (const int varindex,
-         const operator_type transport_operator = op_error);
+         const operator_type transport_operator = op_error,
+         const int tag = -1);
 
   // Destructors
   virtual ~gdata ();
@@ -60,7 +61,8 @@ public:
   // Pseudo constructors
   virtual gdata*
   make_typed (const int varindex,
-              const operator_type transport_operator = op_error) const = 0;
+              const operator_type transport_operator = op_error,
+              const int tag = -1) const = 0;
   
   // Assignment
   gdata & operator= (gdata const & from);
@@ -140,6 +142,11 @@ public:
   }
   
   // Data manipulators
+private:
+  virtual comm_state::gcommbuf *
+  make_typed_commbuf (const ibbox & box)
+    const = 0;
+  
  public:
     void copy_from (comm_state& state,
                     const gdata* src, const ibbox& box);
@@ -151,6 +158,7 @@ public:
                        const gdata* src, const ibbox& box);
   void copy_from_wait (comm_state& state,
                        const gdata* src, const ibbox& box);
+#if 0
  protected:
   virtual void
   copy_from_recv_inner (comm_state& state,
@@ -172,6 +180,7 @@ public:
                              const gdata* src,
                              const ibbox& box)
     = 0;
+#endif
   
  public:
   void interpolate_from (comm_state& state,