CarpetLib: Implement using MPI_Waitall for communicating

Add a parameter CarpetLib::use_waitall that switches from using a
series of MPI_Wait statements to using a single MPI_Waitall
statement.  This might improve performance on many processors.

darcs-hash:20041124235118-891bb-034efea054db236a187022b1858e4574da867fa3.gz

1 files changed, 18 insertions, 8 deletions

diff --git a/Carpet/CarpetLib/src/gdata.hh b/Carpet/CarpetLib/src/gdata.hh
index 0fb1882d4..26dd33251 100644
--- a/Carpet/CarpetLib/src/gdata.hh
+++ b/Carpet/CarpetLib/src/gdata.hh
@@ -21,7 +21,6 @@
 using namespace std;
 
 
-  
 
 template<int D>
 class gdata;
@@ -40,12 +39,13 @@ struct comm_state {
   ~comm_state ();
   
 private:
-  // Forbit copying and passing by value
+  // Forbid copying and passing by value
   comm_state (comm_state const &);
   comm_state& operator= (comm_state const &);
 public:
   
   vector<gdata<D>*> tmps;
+  vector<MPI_Request> requests; // for use_waitall
   size_t current;
 };
 
@@ -80,8 +80,8 @@ protected:                      // should be readonly
   
   ibbox _extent;		// bbox for all data
   
-  bool comm_active;
-  MPI_Request request;
+  bool comm_active;             // a communication is going on
+  MPI_Request request;          // outstanding MPI request
   
   int tag;                      // MPI tag for this object
   
@@ -101,11 +101,21 @@ public:
   
   // Processor management
   virtual void change_processor (comm_state<D>& state,
-                                 const int newproc, void* const mem=0) = 0;
+                                 const int newproc,
+                                 void* const mem=0) = 0;
  protected:
-  virtual void change_processor_recv (const int newproc, void* const mem=0) = 0;
-  virtual void change_processor_send (const int newproc, void* const mem=0) = 0;
-  virtual void change_processor_wait (const int newproc, void* const mem=0) = 0;
+  virtual void change_processor_recv (comm_state<D>& state,
+                                      const int newproc,
+                                      void* const mem=0)
+    = 0;
+  virtual void change_processor_send (comm_state<D>& state,
+                                      const int newproc,
+                                      void* const mem=0)
+    = 0;
+  virtual void change_processor_wait (comm_state<D>& state,
+                                      const int newproc,
+                                      void* const mem=0)
+    = 0;
  public:
   
   // Storage management