Carpet: Make host/process association publicly available

Determine and store host/process association.
Provide aliased function API to query this.

Improve performance of multi-model API.

7 files changed, 266 insertions, 22 deletions

diff --git a/Carpet/Carpet/interface.ccl b/Carpet/Carpet/interface.ccl
index 0415465de..506f9e8d4 100644
--- a/Carpet/Carpet/interface.ccl
+++ b/Carpet/Carpet/interface.ccl
@@ -48,6 +48,18 @@ PROVIDES FUNCTION GetMPICommWorld WITH Carpet_GetMPICommWorld LANGUAGE C
 
 
 
+# Get access to process->host mapping
+CCTK_INT FUNCTION CCTK_MyHost(CCTK_POINTER_TO_CONST IN cctkGH)
+CCTK_INT FUNCTION CCTK_nHosts(CCTK_POINTER_TO_CONST IN cctkGH)
+CCTK_INT FUNCTION CCTK_nProcsOnHost(CCTK_POINTER_TO_CONST IN cctkGH, CCTK_INT IN host)
+CCTK_INT FUNCTION CCTK_ProcsOnHost(CCTK_POINTER_TO_CONST IN cctkGH, CCTK_INT IN host, CCTK_INT ARRAY OUT procs, CCTK_INT IN nprocs)
+PROVIDES FUNCTION CCTK_MyHost       WITH Carpet_MyHost       LANGUAGE C
+PROVIDES FUNCTION CCTK_nHosts       WITH Carpet_nHosts       LANGUAGE C
+PROVIDES FUNCTION CCTK_nProcsOnHost WITH Carpet_nProcsOnHost LANGUAGE C
+PROVIDES FUNCTION CCTK_ProcsOnHost  WITH Carpet_ProcsOnHost  LANGUAGE C
+
+
+
 # Register and unregister routines which are called before and after
 # every scheduled routine
 CCTK_INT \
diff --git a/Carpet/Carpet/src/Hosts.cc b/Carpet/Carpet/src/Hosts.cc
new file mode 100644
index 000000000..a9f01031c
--- /dev/null
+++ b/Carpet/Carpet/src/Hosts.cc
@@ -0,0 +1,164 @@
+#include <cctk.h>
+
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#ifdef CCTK_MPI
+#  include <mpi.h>
+#else
+#  include "nompi.h"
+#endif
+
+#include <dist.hh>
+#include <functions.hh>
+#include <mpi_string.hh>
+
+
+
+namespace Carpet
+{
+  
+  using namespace std;
+  
+  
+  
+  vector <string> hosts;              // Host id to host name
+  std::map <string, int> host_map;    // Host name to host id
+  vector <int> host_ids;              // Process to host id
+  vector <vector <int> > host_procs;  // Host id to processes
+  
+  
+  
+  vector <string> const & Hosts () { return hosts; }
+  std::map <string, int> const & HostMap () { return host_map; }
+  vector <int> const & HostIds () { return host_ids; }
+  vector <vector <int> > const & HostProcs () { return host_procs; }
+  
+  string Host (int const id)
+  {
+    return hosts.at (id);
+  }
+
+  int HostMap (string const name)
+  {
+    if (host_map.find (name) != host_map.end())
+    {
+      return host_map[name];
+    }
+    else
+    {
+      return -1;
+    }
+  }
+  
+  int HostId (int const proc)
+  {
+    return host_ids.at (proc);
+  }
+
+  vector <int> const & HostProcs (int const id)
+  {
+    return host_procs.at (id);
+  }
+  
+  
+  
+  void
+  DetermineHosts (string const host, bool const verbose)
+  {
+    // Get the total number of processes
+    int num_procs;
+    MPI_Comm_size (dist::comm(), & num_procs);
+    int my_proc;
+    MPI_Comm_rank (dist::comm(), & my_proc);
+    
+    // Gather all host names
+    hosts = allgather_string (dist::comm(), host);
+    
+    // Map host strings to small integers
+    int num_hosts = 0;
+    host_ids.resize (num_procs);
+    host_map.clear ();
+    for (int n = 0; n < num_procs; ++ n)
+    {
+      if (host_map.find (hosts.AT(n)) != host_map.end())
+      {
+        host_ids.AT(n) = host_map[hosts.AT(n)];
+      }
+      else
+      {
+        host_map[hosts.AT(n)] = num_hosts;
+        host_ids.AT(n) = num_hosts;
+        ++ num_hosts;
+      }
+    }
+    
+    // Determine processes per host
+    vector <int> num_host_procs (num_hosts, 0);
+    for (int n = 0; n < num_procs; ++ n)
+    {
+      ++ num_host_procs.at (host_ids.AT(n));
+    }
+    
+    host_procs.resize (num_hosts);
+    for (int m = 0; m < num_hosts; ++ m)
+    {
+      host_procs.AT(m).reserve (num_host_procs.AT(m));
+    }
+    for (int n = 0; n < num_procs; ++ n)
+    {
+      host_procs.at (host_ids.AT(n)).push_back (n);
+    }
+    for (int m = 0; m < num_hosts; ++ m)
+    {
+      assert (static_cast<int> (host_procs.AT(m).size())
+              == num_host_procs.AT(m));
+    }
+    
+    if (verbose)
+    {
+      CCTK_INFO ("Host listing:");
+      for (int m = 0; m < num_hosts; ++ m)
+      {
+        cout << "   host " << m << ": \"" << hosts.AT(m) << "\"" << endl;
+      }
+      CCTK_INFO ("Host/process mapping:");
+      for (int n = 0; n < num_procs; ++ n)
+      {
+        int const m = host_ids.AT(n);
+        bool const same_host_as_prev =
+          n-1 >= 0 and host_ids.AT(n-1) == m;
+        bool const same_host_as_next =
+          n+1 < num_procs and host_ids.AT(n+1) == m;
+        if (same_host_as_next) {
+          if (same_host_as_prev) {
+            // Output nothing
+          } else {
+            // This process has the same host as the next one:
+            // output only a partial line
+            cout << "   processes " << n << "-";
+          }
+        } else {
+          if (same_host_as_prev) {
+            // This process has the same host as the previous one:
+            // finish a partial line
+            cout << n << ": "
+                 << "host " << m << " \"" << hosts.AT(m) << "\"" << endl;
+          } else {
+            cout << "   process " << n << ": "
+                 << "host " << m << " \"" << hosts.AT(m) << "\"" << endl;
+          }
+        }
+      }
+      int const my_host = host_ids.AT(my_proc);
+      CCTK_VInfo (CCTK_THORNSTRING,
+                  "Host mapping: This is process %d, host %d \"%s\"",
+                  my_proc, my_host, host.c_str());
+    }
+  }
+  
+} // namespace Carpet
diff --git a/Carpet/Carpet/src/MultiModel.cc b/Carpet/Carpet/src/MultiModel.cc
index 7f4db58bc..0de480051 100644
--- a/Carpet/Carpet/src/MultiModel.cc
+++ b/Carpet/Carpet/src/MultiModel.cc
@@ -32,12 +32,12 @@ namespace Carpet
   
   
   
-  vector <string> Models () { return models; }
-  std::map <string, int> ModelMap () { return model_map; }
-  vector <int> ModelIds () { return model_ids; }
-  vector <vector <int> > ModelProcs () { return model_procs; }
+  vector <string> const & Models () { return models; }
+  std::map <string, int> const & ModelMap () { return model_map; }
+  vector <int> const & ModelIds () { return model_ids; }
+  vector <vector <int> > const & ModelProcs () { return model_procs; }
   
-  string Models (int const id)
+  string Model (int const id)
   {
     return models.at (id);
   }
@@ -59,9 +59,9 @@ namespace Carpet
     return model_ids.at (proc);
   }
 
-  vector <int> ModelProcs (int const proc)
+  vector <int> const & ModelProcs (int const id)
   {
-    return model_procs.at (proc);
+    return model_procs.at (id);
   }
   
   
diff --git a/Carpet/Carpet/src/SetupGH.cc b/Carpet/Carpet/src/SetupGH.cc
index c5279c3f7..87ae212cb 100644
--- a/Carpet/Carpet/src/SetupGH.cc
+++ b/Carpet/Carpet/src/SetupGH.cc
@@ -254,16 +254,19 @@ namespace Carpet {
       MPI_Get_processor_name (hostnamebuf, &hostnamelen);
       string const hostname (hostnamebuf);
 #endif
+      DetermineHosts (hostname, true);
+#if HAVE_GETPID
       int const mypid = static_cast<int> (getpid ());
+#else
+      int const mypid = -1;
+#endif
       // Output
       CCTK_VInfo (CCTK_THORNSTRING,
                   "This process runs on host %s, pid=%d",
                   hostname.c_str(), mypid);
       if (verbose or veryverbose) {
-        // Collect host names
-        vector <string> hostnames = allgather_string (dist::comm(), hostname);
-        vector <int> pids (nprocs);
         // Collect process ids
+        vector <int> pids (nprocs);
         MPI_Allgather (const_cast <int *> (& mypid), 1, MPI_INT,
                        & pids.front(), 1, MPI_INT,
                        dist::comm());
@@ -276,9 +279,11 @@ namespace Carpet {
         CCTK_VInfo (CCTK_THORNSTRING,
                     "Running on the following hosts:");
         for (int n = 0; n < nprocs; ++ n) {
+          int const host_id = HostId(n);
           CCTK_VInfo (CCTK_THORNSTRING,
-                      "   %6d: %s, pid=%d, num_threads=%d",
-                      n, hostnames.AT(n).c_str(), pids.AT(n), nthreads.AT(n));
+                      "   %6d: hid=%d (%s), pid=%d, num_threads=%d",
+                      n, host_id, Host(host_id).c_str(),
+                      pids.AT(n), nthreads.AT(n));
         }
       }
       
diff --git a/Carpet/Carpet/src/functions.hh b/Carpet/Carpet/src/functions.hh
index 6c1d3c84a..fb9afb76b 100644
--- a/Carpet/Carpet/src/functions.hh
+++ b/Carpet/Carpet/src/functions.hh
@@ -57,20 +57,45 @@ namespace Carpet {
                  bool verbose);
   
   // Model id to model name
-  vector <string> Models ();
-  string Model (int id) CCTK_ATTRIBUTE_PURE;
+  vector <string> const & Models ();
+  string Model (int id);
   
   // Model name to model id
-  std::map <string, int> ModelMap ();
-  int ModelMap (string name) CCTK_ATTRIBUTE_PURE;
+  std::map <string, int> const & ModelMap ();
+  int ModelMap (string name);
   
   // Processor to model id
-  vector <int> ModelIds ();
-  int ModelId (int proc) CCTK_ATTRIBUTE_PURE;
+  vector <int> const & ModelIds ();
+  int ModelId (int proc);
+  
+  // Model id to processes
+  vector <vector <int> > const & ModelProcs ();
+  vector <int> const & ModelProcs (int id);
+  
+  
+  
+  // Host mapping
+  void
+  DetermineHosts (string host,
+                  bool verbose);
+  
+  // Host id to host name
+  vector <string> const & Hosts ();
+  string Host (int id);
+  
+  // Host name to host id
+  std::map <string, int> const & HostMap ();
+  int HostMap (string name);
+  
+  // Processor to host id
+  vector <int> const & HostIds ();
+  int HostId (int proc);
+  
+  // Host id to processes
+  vector <vector <int> > const & HostProcs ();
+  vector <int> const & HostProcs (int id);
+  
   
-  // Model id to processors
-  vector <vector <int> > ModelProcs () CCTK_ATTRIBUTE_PURE;
-  vector <int> ModelProcs (int proc) CCTK_ATTRIBUTE_PURE;
   
   extern "C" {
     CCTK_POINTER_TO_CONST
diff --git a/Carpet/Carpet/src/helpers.cc b/Carpet/Carpet/src/helpers.cc
index afacfc802..ccf5f71cc 100644
--- a/Carpet/Carpet/src/helpers.cc
+++ b/Carpet/Carpet/src/helpers.cc
@@ -149,7 +149,44 @@ namespace Carpet {
   }
   
 
-
+  
+  // Hosts
+  extern "C"
+  CCTK_INT Carpet_MyHost(CCTK_POINTER_TO_CONST const cctkGH_)
+  {
+    return HostId(dist::rank());
+  }
+  
+  extern "C"
+  CCTK_INT Carpet_nHosts(CCTK_POINTER_TO_CONST const cctkGH_)
+  {
+    return Hosts().size();
+  }
+  
+  extern "C"
+  CCTK_INT Carpet_nProcsOnHost(CCTK_POINTER_TO_CONST const cctkGH_,
+                               CCTK_INT const host)
+  {
+    return HostProcs(host).size();
+  }
+  
+  extern "C"
+  CCTK_INT Carpet_ProcsOnHost(CCTK_POINTER_TO_CONST const cctkGH_,
+                              CCTK_INT const host,
+                              CCTK_INT procs[], CCTK_INT const nprocs)
+  {
+    int const nprocs1 = HostProcs(host).size();
+    for (int p=0; p<nprocs1; ++p) {
+      if (p >= nprocs) break;
+      procs[p] = HostProcs(host).AT(p);
+    }
+    return nprocs1;
+  }
+  
+  
+  
+  // Coordinates
+  
   CCTK_INT
   Carpet_GetCoordRange (CCTK_POINTER_TO_CONST const cctkGH_,
                         CCTK_INT              const m,
diff --git a/Carpet/Carpet/src/make.code.defn b/Carpet/Carpet/src/make.code.defn
index 92c781faf..28cb37f05 100644
--- a/Carpet/Carpet/src/make.code.defn
+++ b/Carpet/Carpet/src/make.code.defn
@@ -10,6 +10,7 @@ SRCS =	CactusTimer.cc				\
 	Comm.cc					\
 	Cycle.cc				\
 	Evolve.cc				\
+	Hosts.cc				\
 	Initialise.cc				\
 	MultiModel.cc				\
 	OutputGH.cc				\