From 771217689609c434b69771ecb8ffd7bbeda4153b Mon Sep 17 00:00:00 2001
From: Anton Khirnov <anton@khirnov.net>
Date: Sat, 29 Jun 2019 11:24:10 +0200
Subject: egs: sync the residual maximum across components.

---
 ell_grid_solve.c |  7 +++++++
 mg2d.c           | 20 ++++----------------
 relax_mpi_test.c |  4 ++--
 3 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/ell_grid_solve.c b/ell_grid_solve.c
index 97b8688..c3ce3a7 100644
--- a/ell_grid_solve.c
+++ b/ell_grid_solve.c
@@ -186,6 +186,13 @@ static void residual_calc(EGSContext *ctx, int export_res)
 
     mg2di_timer_stop(&ctx->timer_res_calc);
 
+    if (priv->dg->nb_components > 1) {
+        mg2di_timer_start(&ctx->timer_mpi_sync);
+        MPI_Allreduce(MPI_IN_PLACE, &ctx->residual_max, 1,
+                      MPI_DOUBLE, MPI_MAX, priv->comm);
+        mg2di_timer_stop(&ctx->timer_mpi_sync);
+    }
+
     priv->reflect_skip = ~0;
 }
 
diff --git a/mg2d.c b/mg2d.c
index 853e9ab..5560681 100644
--- a/mg2d.c
+++ b/mg2d.c
@@ -227,18 +227,6 @@ static int mg_prolong_u(MG2DContext *ctx, MG2DLevel *l)
     return 0;
 }
 
-static double mg_resmax_get(MG2DLevel *l)
-{
-    double ret = l->solver->residual_max;
-
-    if (l->dg->nb_components > 1) {
-        MPI_Allreduce(MPI_IN_PLACE, &ret, 1,
-                      MPI_DOUBLE, MPI_MAX, l->mpi_comm);
-    }
-
-    return ret;
-}
-
 static int mg_solve_subgrid(MG2DContext *ctx, MG2DLevel *level, int allow_exact)
 {
     enum EGSType solve_type = (allow_exact && level->dg->nb_components == 1 &&
@@ -263,7 +251,7 @@ static int mg_solve_subgrid(MG2DContext *ctx, MG2DLevel *level, int allow_exact)
         level->egs_init_flags |= EGS_INIT_FLAG_SAME_DIFF_COEFFS;
     }
 
-    res_old = mg_resmax_get(level);
+    res_old = level->solver->residual_max;
 
     /* handle exact solve */
     if (solve_type == EGS_SOLVE_EXACT) {
@@ -334,7 +322,7 @@ static int mg_solve_subgrid(MG2DContext *ctx, MG2DLevel *level, int allow_exact)
     }
 
 finish:
-    res_new = mg_resmax_get(level);
+    res_new = level->solver->residual_max;
     if (!isfinite(res_new) ||
         (res_new > 1e2 * ctx->tol && res_old / res_new <= 1e-1)) {
         mg2di_log(&ctx->priv->logger, MG2D_LOG_ERROR,
@@ -1131,7 +1119,7 @@ int mg2d_solve(MG2DContext *ctx)
 
     root->egs_init_flags |= EGS_INIT_FLAG_SAME_DIFF_COEFFS;
 
-    res_orig = mg_resmax_get(root);
+    res_orig = s_root->residual_max;
     res_prev = res_orig;
 
     for (int i = 0; i < ctx->maxiter; i++) {
@@ -1139,7 +1127,7 @@ int mg2d_solve(MG2DContext *ctx)
         if (ret < 0)
             goto fail;
 
-        res_cur = mg_resmax_get(root);
+        res_cur = s_root->residual_max;
 
         if (res_cur < ctx->tol) {
             mg2di_log(&priv->logger, MG2D_LOG_INFO, "converged on iteration %d, residual %g\n",
diff --git a/relax_mpi_test.c b/relax_mpi_test.c
index 75e76ef..5e69b49 100644
--- a/relax_mpi_test.c
+++ b/relax_mpi_test.c
@@ -190,7 +190,7 @@ int main(int argc, char **argv)
         goto fail;
     }
 
-    MPI_Reduce(&ctx->residual_max, &res_old, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    res_old = ctx->residual_max;
 
     for (int i = 0; i < maxiter; i++) {
         ret = mg2di_egs_solve(ctx, EGS_SOLVE_RELAXATION, 0);
@@ -199,7 +199,7 @@ int main(int argc, char **argv)
             ret = 1;
             goto fail;
         }
-        MPI_Reduce(&ctx->residual_max, &res_new, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+        res_new = ctx->residual_max;
         if (rank == 0) {
             if (res_new > 1e3) {
                 fprintf(stderr, "Diverged at step %d: %g -> %g\n", i, res_old, res_new);
-- 
cgit v1.2.3