CarpetLib: New parameter use_loopcontrol_in_operators

When this parameters is set, then (some) prolongation/restriction operators are parallelized via LoopControl instead of in dh.cc.

1 files changed, 32 insertions, 33 deletions

diff --git a/Carpet/CarpetLib/src/restrict_3d_cc_o5_rf2.cc b/Carpet/CarpetLib/src/restrict_3d_cc_o5_rf2.cc
index 2cd1b1f1f..5c2f88642 100644
--- a/Carpet/CarpetLib/src/restrict_3d_cc_o5_rf2.cc
+++ b/Carpet/CarpetLib/src/restrict_3d_cc_o5_rf2.cc
@@ -1,6 +1,8 @@
 #include <cctk.h>
 #include <cctk_Parameters.h>
 
+#include <loopcontrol.h>
+
 #include <algorithm>
 #include <cassert>
 #include <cmath>
@@ -124,8 +126,6 @@ namespace CarpetLib {
     
     DECLARE_CCTK_PARAMETERS;
     
-    typedef typename typeprops<T>::real RT;
-    
     
     
     if (any (srcbbox.stride() >= regbbox.stride() or
@@ -205,41 +205,40 @@ namespace CarpetLib {
     
     
     // Loop over coarse region
-#pragma omp parallel for collapse(3)
-    for (int k=0; k<regkext; ++k) {
-      for (int j=0; j<regjext; ++j) {
-        for (int i=0; i<regiext; ++i) {
+#pragma omp parallel
+    CCTK_LOOP3(restrict_3d_cc_o5_rf2,
+               i,j,k, 0,0,0, regiext,regjext,regkext,
+               dstipadext,dstjpadext,dstkpadext)
+    {
           
 #ifdef CARPET_DEBUG
-    if(not (2 * k + 3 + srckoff < srckext and
-            2 * j + 3 + srcjoff < srcjext and
-            2 * i + 3 + srcioff < srciext and
-            2 * k - 2 + srckoff >= 0 and
-            2 * j - 2 + srcjoff >= 0 and
-            2 * i - 2 + srcioff >= 0))
-    {
-      cout << "restrict_3d_cc_o3_rf2.cc\n";
-      cout << "regext " << regext << "\n";
-      cout << "srcext " << srcext << "\n";
-      cout << "srcbbox=" << srcbbox << "\n";
-      cout << "dstbbox=" << dstbbox << "\n";
-      cout << "regbbox=" << regbbox << "\n";
-      cout << "i,j,k=" << i << " " << j << " " << k << "\n";
-      assert(2 * k + 2 + srckoff < srckext);
-      assert(2 * j + 2 + srcjoff < srcjext);
-      assert(2 * i + 2 + srcioff < srciext);
-      assert(2 * k - 1 + srckoff >= 0);
-      assert(2 * j - 1 + srcjoff >= 0);
-      assert(2 * i - 1 + srcioff >= 0);
-    }
+      if(not (2 * k + 3 + srckoff < srckext and
+              2 * j + 3 + srcjoff < srcjext and
+              2 * i + 3 + srcioff < srciext and
+              2 * k - 2 + srckoff >= 0 and
+              2 * j - 2 + srcjoff >= 0 and
+              2 * i - 2 + srcioff >= 0))
+      {
+        cout << "restrict_3d_cc_o3_rf2.cc\n";
+        cout << "regext " << regext << "\n";
+        cout << "srcext " << srcext << "\n";
+        cout << "srcbbox=" << srcbbox << "\n";
+        cout << "dstbbox=" << dstbbox << "\n";
+        cout << "regbbox=" << regbbox << "\n";
+        cout << "i,j,k=" << i << " " << j << " " << k << "\n";
+        assert(2 * k + 2 + srckoff < srckext);
+        assert(2 * j + 2 + srcjoff < srcjext);
+        assert(2 * i + 2 + srcioff < srciext);
+        assert(2 * k - 1 + srckoff >= 0);
+        assert(2 * j - 1 + srcjoff >= 0);
+        assert(2 * i - 1 + srcioff >= 0);
+      }
 #endif    
-          dst [DSTIND3(i, j, k)] =
-            CarpetLib::restrict3
-            (& src[SRCIND3(2*i, 2*j, 2*k)], srcdi, srcdj, srcdk);
+      dst [DSTIND3(i, j, k)] =
+        CarpetLib::restrict3
+        (& src[SRCIND3(2*i, 2*j, 2*k)], srcdi, srcdj, srcdk);
 
-        }
-      }
-    }
+    } CCTK_ENDLOOP3(restrict_3d_cc_o5_rf2);
     
   }