From 4c972cfc352ae5ba851cae142ca6fe594d88bc04 Mon Sep 17 00:00:00 2001
From: Anton Khirnov <anton@khirnov.net>
Date: Thu, 23 May 2019 11:01:00 +0200
Subject: egs: add support for MPI-based multi-component solves

---
 ell_grid_solve.h | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

(limited to 'ell_grid_solve.h')

diff --git a/ell_grid_solve.h b/ell_grid_solve.h
index 7ce61a0..2893ce5 100644
--- a/ell_grid_solve.h
+++ b/ell_grid_solve.h
@@ -40,6 +40,9 @@
 #include <stdint.h>
 #include <threadpool.h>
 
+#include <mpi.h>
+
+#include "components.h"
 #include "log.h"
 #include "mg2d_boundary.h"
 #include "mg2d_constants.h"
@@ -122,7 +125,8 @@ typedef struct EGSContext {
     int cpuflags;
 
     /**
-     * Size of the solver grid, set by mg2di_egs_alloc().
+     * Size of the solver grid, set by mg2di_egs_alloc[_mpi](). For
+     * multi-component runs, this contains the size of this component only.
      * Read-only for the caller.
      */
     size_t domain_size[2];
@@ -142,6 +146,9 @@ typedef struct EGSContext {
     /**
      * Boundary specification, indexed by MG2DBoundaryLoc.
      * To be filled by the caller before mg2di_egs_init().
+     *
+     * For multi-component runs, only the outer (not inter-component) boundary
+     * specifications are accessed by the solver.
      */
     MG2DBoundary *boundaries[4];
 
@@ -156,9 +163,9 @@ typedef struct EGSContext {
     NDArray *u;
 
     /**
-     * u including the ghost zones.
+     * u including the outer boundary ghost zones.
      */
-    NDArray *u_base;
+    NDArray *u_exterior;
 
     /**
      * Values of the right-hand side.
@@ -199,6 +206,7 @@ typedef struct EGSContext {
     Timer   timer_res_calc;
     Timer   timer_init;
     Timer   timer_solve;
+    Timer   timer_mpi_sync;
 } EGSContext;
 
 #define EGS_INIT_FLAG_SAME_DIFF_COEFFS (1 << 0)
@@ -211,6 +219,18 @@ typedef struct EGSContext {
  * @return The solver context on success, NULL on failure.
  */
 EGSContext *mg2di_egs_alloc(size_t domain_size[2]);
+
+/**
+ * Allocate a solver component in a multi-component MPI-based solve.
+ *
+ * @param comm The MPI communicator used to communicate with the other
+ *             components
+ * @dg The geometry of the full computational domain. This component is indexed
+ *     by its MPI rank in this geometry.
+ *
+ * @return The solver context on success, NULL on failure.
+ */
+EGSContext *mg2di_egs_alloc_mpi(MPI_Comm comm, const DomainGeometry *dg);
 /**
  * Initialize the solver for use, after all the required fields are filled by
  * the caller.
-- 
cgit v1.2.3