print_state/printCell: Make it work without managed memory (#3543)

AMReX-Codes · Sep 16, 2023 · 64d04be · 64d04be
1 parent 6eb91be
commit 64d04be
Showing 1 changed file with 29 additions and 5 deletions.
diff --git a/Src/Base/AMReX_FabArrayUtility.H b/Src/Base/AMReX_FabArrayUtility.H
@@ -1055,18 +1055,42 @@ printCell (FabArray<FAB> const& mf, const IntVect& cell, int comp = -1,
     {
         const Box& bx = amrex::grow(mfi.validbox(), ng);
         if (bx.contains(cell)) {
+            int n = (comp >= 0) ? 1 : mf.nComp();
+            auto const& fab = mf.const_array(mfi);
+            Gpu::PinnedVector<typename FAB::value_type> pv(n);
+            auto* dp = pv.data();
+            auto f = [=] AMREX_GPU_HOST_DEVICE ()
+                {
+                    if (comp >= 0) {
+                        *dp = fab(cell, comp);
+                    } else {
+                        for (int i = 0; i < n; ++i) {
+                            dp[i] = fab(cell,i);
+                        }
+                    }
+                };
+
+#ifdef AMREX_USE_GPU
+            if (mf.arena()->isManaged() || mf.arena()->isDevice()) {
+                amrex::single_task(f);
+                Gpu::streamSynchronize();
+            } else
+#endif
+            {
+                f();
+            }
+
             if (comp >= 0) {
                 amrex::AllPrint().SetPrecision(17) << " At cell " << cell << " in Box " << bx
-                                                   << ": " << mf[mfi](cell, comp) << std::endl;
+                                                   << ": " << *dp << std::endl;
             } else {
                 std::ostringstream ss;
                 ss.precision(17);
-                const int ncomp = mf.nComp();
-                for (int i = 0; i < ncomp-1; ++i)
+                for (int i = 0; i < n-1; ++i)
                 {
-                    ss << mf[mfi](cell,i) << ", ";
+                    ss << dp[i] << ", ";
                 }
-                ss << mf[mfi](cell,ncomp-1);
+                ss << dp[n-1];
                 amrex::AllPrint() << " At cell " << cell << " in Box " << bx
                                   << ": " << ss.str() << std::endl;
             }