Merge branch 'development' into cvode

AMReX-Codes · Sep 18, 2023 · 770c9db · 770c9db
2 parents ddc9f6c + 48b3ec7
commit 770c9db
Show file tree

Hide file tree

Showing 34 changed files with 993 additions and 165 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -46,7 +46,7 @@ jobs:
             -DCMAKE_CXX_COMPILER=$(which g++)            \
             -DCMAKE_CUDA_HOST_COMPILER=$(which g++)      \
             -DCMAKE_Fortran_COMPILER=$(which gfortran)   \
-            -DAMReX_CUDA_ARCH=7.0                        \
+            -DAMReX_CUDA_ARCH="7.0"                      \
             -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
             -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
             -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache        \
@@ -68,8 +68,8 @@ jobs:
       CUDAHOSTCXX: clang++-15
       CUDAFLAGS: "-stdlib=libc++"
       LDFLAGS: "-stdlib=libc++"
-      CUDAARCHS: 70
-      AMReX_CUDA_ARCH: 7.0
+      CUDAARCHS: "70"
+      AMReX_CUDA_ARCH: "7.0"
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -199,7 +199,7 @@ jobs:
         # /home/runner/work/amrex/amrex/Src/Base/AMReX_GpuLaunchGlobal.H:16:41: error: unused parameter ‘f0’ [-Werror=unused-parameter]
         #    16 |     AMREX_GPU_GLOBAL void launch_global (L f0) { f0(); }
         #
-        make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS="-fno-operator-names -Wno-unused-parameter" CCACHE=ccache
+        make -j2 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS="-fno-operator-names -Wno-unused-parameter" CCACHE=ccache CUDA_ARCH="7.0 7.2"
         make install
 
         ccache -s

diff --git a/CHANGES b/CHANGES
@@ -1,3 +1,112 @@
+# 23.09
+
+  -- Fix InitRandomPerBox for 1D & 2D (#3527)
+
+  -- Add ability for GCC 8 in CMake to build fgradient which uses std::filesystem (#3523)
+
+  -- Fix: Include Guard 1D for MLPoisson (#3528)
+
+  -- Workaround for Intel compiler classic (#3526)
+
+  -- Constness in buildInfo: num_modules (#3522)
+
+  -- MLABecLaplacian: Tweak kernel fusing (#3521)
+
+  -- UniqueString tweak (#3520)
+
+  -- faverage fix (#3519)
+
+  -- Empty level fix (#3517)
+
+  -- Simplify filterParticles Kernel (#3510)
+
+  -- Generatize particle-to-cell assignment function (#3499)
+     Follow-on to 3499 (#3514)
+     ParticleLocator: Make Assignor optional template parameter (#3515)
+
+  -- GPU Launch Min Blocks (#3503)
+
+  -- Remove unsafe std::forward (#3513)
+
+  -- Fix Bug in FaceLinear Interpolater (#3483)
+
+  -- Add test on whether we are in the valid box when updating drho_as_crs… (#3506)
+
+  -- Fix backtrace nullptrs (#3505)
+
+  -- Skip ubsan lib on OSX (#3504)
+
+  -- SYCL: Add table of GNU Make config vars to docs (#3400)
+
+  -- Fix: Backtraces will NULL Addresses (#3502)
+
+  -- Update gpu elixir and async array synchronization for sycl (#3498)
+
+  -- SundialsIntegrator: Fix memory deallocation (#3501)
+
+  -- FillRandom & FillRandomNormal (#3500)
+
+  -- Update position in 2nd pass for tracer advect with Ucc. (#3496)
+
+  -- Move use of bcrec back onto GPU (#3494)
+
+  -- fix bug in StateRedist and add multi-level functionality for FluxRedi… (#3491)
+
+  -- fgradient: A new plotfile tool for computing gradient (#3490)
+
+  -- CI: Remove redundancy in nvcc dependencies files (#3387)
+
+  -- CMake: HIP Debug with -O1 (#3487)
+
+  -- Fix overflow in ParallelFor(Long n, ...) (#3489)
+
+  -- Adjust build for upstream Mpich and OpenMPI modules on Perlmutter. (#3486)
+
+  -- CArena: Implement alloc_in_place (#3426)
+
+  -- Clang-Tidy: Add more checks (#3466)
+     Fix a bug in #3466 (#3485)
+
+  -- Option to force regrid level zero (#3484)
+
+  -- Fix typo introduced in PR 3452 (#3482)
+
+  -- Clang-Tidy changes in Src/LinearSolvers (#3475)
+
+  -- Update FluxRedist and comments for StateItracker (#3481)
+
+  -- Clang-Tidy changes in Src/EB (#3473)
+
+  -- Clang-Tidy changes in Src/Boundary (#3471)
+
+  -- Clang-Tidy changes in Src/F_Interfaces (#3474)
+
+  -- Clang-Tidy changes in Src/Particle (#3476)
+
+  -- Clang-Tidy changes in Src/Extern (#3472)
+
+  -- Clang-Tidy changes in Tests (#3477)
+
+  -- Clang-Tidy changes in Tools (#3478)
+
+  -- Change the default value of The_Pinned_Arena release threshold (#3479)
+
+  -- Handle ThetaGPU in GNU make system (#3469)
+
+  -- Clang-Tidy changes in Src/AmrCore (#3468)
+
+  -- Clang-Tidy changes in Src/Amr (#3467)
+
+  -- Fix a bug in EB tensor solver's cross term (#3465)
+
+  -- Add misc-misplaced-const check to clang-tidy CI test (#3464)
+
+  -- Suppress implicit return warnings with nvcc (#3463)
+
+  -- Silence "unreachable loop" warning for GPU builds (#3462)
+
+  -- PlotFileUtil: Add Direct Includes (#3446)
+
 # 23.08
 
   -- Fix a bug in AmrMesh::ChopGrids (#3460)

diff --git a/CITATION b/CITATION
@@ -1,7 +1,7 @@
 If you use AMReX, we ask you to cite the JOSS paper:
 
     @ARTICLE{AMReX_JOSS,
-       author = {Zhang et al.},
+       author = {Weiqun Zhang and Ann Almgren and Vince Beckner and John Bell and Johannes Blaschke and Cy Chan and Marcus Day and Brian Friesen and Kevin Gott and Daniel Graves and Max Katz and Andrew Myers and Tan Nguyen and Andrew Nonaka and Michele Rosso and Samuel Williams and Michael Zingale},
 	title = "{AMReX: A Framework for Block-Structured Adaptive Mesh Refinement}", 
       journal = {Journal of Open Source Software},
 	 year = 2019,

diff --git a/Src/Amr/AMReX_Amr.cpp b/Src/Amr/AMReX_Amr.cpp
@@ -2205,7 +2205,8 @@ Amr::coarseTimeStep (Real stop_time)
                << " DT = "   << dt_level[0] << '\n';
     }
     if (record_run_info_terse && ParallelDescriptor::IOProcessor()) {
-        runlog_terse << level_steps[0] << " " << cumtime << " " << dt_level[0] << '\n';
+        runlog_terse << level_steps[0] << " " << cumtime << " " << dt_level[0];
+        runlog_terse << std::endl; // Make sure we flush!
     }
 
     int check_test = 0;

diff --git a/Src/Base/AMReX_FabArrayUtility.H b/Src/Base/AMReX_FabArrayUtility.H
@@ -1055,18 +1055,42 @@ printCell (FabArray<FAB> const& mf, const IntVect& cell, int comp = -1,
     {
         const Box& bx = amrex::grow(mfi.validbox(), ng);
         if (bx.contains(cell)) {
+            int n = (comp >= 0) ? 1 : mf.nComp();
+            auto const& fab = mf.const_array(mfi);
+            Gpu::PinnedVector<typename FAB::value_type> pv(n);
+            auto* dp = pv.data();
+            auto f = [=] AMREX_GPU_HOST_DEVICE ()
+                {
+                    if (comp >= 0) {
+                        *dp = fab(cell, comp);
+                    } else {
+                        for (int i = 0; i < n; ++i) {
+                            dp[i] = fab(cell,i);
+                        }
+                    }
+                };
+
+#ifdef AMREX_USE_GPU
+            if (mf.arena()->isManaged() || mf.arena()->isDevice()) {
+                amrex::single_task(f);
+                Gpu::streamSynchronize();
+            } else
+#endif
+            {
+                f();
+            }
+
             if (comp >= 0) {
                 amrex::AllPrint().SetPrecision(17) << " At cell " << cell << " in Box " << bx
-                                                   << ": " << mf[mfi](cell, comp) << std::endl;
+                                                   << ": " << *dp << std::endl;
             } else {
                 std::ostringstream ss;
                 ss.precision(17);
-                const int ncomp = mf.nComp();
-                for (int i = 0; i < ncomp-1; ++i)
+                for (int i = 0; i < n-1; ++i)
                 {
-                    ss << mf[mfi](cell,i) << ", ";
+                    ss << dp[i] << ", ";
                 }
-                ss << mf[mfi](cell,ncomp-1);
+                ss << dp[n-1];
                 amrex::AllPrint() << " At cell " << cell << " in Box " << bx
                                   << ": " << ss.str() << std::endl;
             }

diff --git a/Src/Base/AMReX_GpuComplex.H b/Src/Base/AMReX_GpuComplex.H
@@ -2,6 +2,7 @@
 #define AMREX_GPUCOMPLEX_H_
 #include <AMReX_Config.H>
 
+#include <AMReX_Algorithm.H>
 #include <AMReX_Math.H>
 #include <cmath>
 #include <iostream>

diff --git a/Src/Base/AMReX_NFiles.cpp b/Src/Base/AMReX_NFiles.cpp
@@ -346,8 +346,6 @@ NFilesIter &NFilesIter::operator++() {
 
 #ifdef BL_USE_MPI
 
-  ParallelDescriptor::Message rmess;
-
   if(isReading) {
     fileStream.close();
 
@@ -451,7 +449,7 @@ NFilesIter &NFilesIter::operator++() {
 
             ParallelDescriptor::Send(&nextFileNumberToWrite, 1, nextProcToWrite, writeTag);
 
-            rmess = ParallelDescriptor::Recv(&nextFileNumberAvailable, 1, MPI_ANY_SOURCE, doneTag);
+            ParallelDescriptor::Recv(&nextFileNumberAvailable, 1, MPI_ANY_SOURCE, doneTag);
             availableFileNumbers.insert(nextFileNumberAvailable);
             --remainingWriters;
             }

diff --git a/Src/Base/AMReX_RealBox.H b/Src/Base/AMReX_RealBox.H
@@ -8,6 +8,7 @@
 #include <AMReX_REAL.H>
 #include <AMReX_Box.H>
 #include <AMReX_RealVect.H>
+#include <AMReX_SPACE.H>
 
 #include <iosfwd>
 #include <array>

diff --git a/Src/Base/AMReX_Reduce.H b/Src/Base/AMReX_Reduce.H
@@ -258,6 +258,7 @@ public:
                                                     * m_max_blocks * sizeof(Type)))),
           m_fn_value([&reduce_op,this] () -> Type { return this->value(reduce_op); })
     {
+        reduce_op.resetResultReadiness();
         static_assert(std::is_trivially_copyable<Type>(),
                       "ReduceData::Type must be trivially copyable");
         static_assert(std::is_trivially_destructible<Type>(),
@@ -750,6 +751,9 @@ public:
 
 private:
     bool m_result_is_ready = false;
+
+public:
+    void resetResultReadiness () { m_result_is_ready = false; }
 };
 
 namespace Reduce {
@@ -993,6 +997,7 @@ public:
         : m_tuple(OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads()),
           m_fn_value([&reduce_op,this] () -> Type { return this->value(reduce_op); })
     {
+        reduce_op.resetResultReadiness();
         for (auto& t : m_tuple) {
             Reduce::detail::for_each_init<0, Type, Ps...>(t);
         }
@@ -1158,6 +1163,8 @@ public:
     }
 
     bool m_result_is_ready = false;
+
+    void resetResultReadiness () { m_result_is_ready = false; }
 };
 
 namespace Reduce {

diff --git a/Src/Base/Parser/AMReX_Parser_Y.H b/Src/Base/Parser/AMReX_Parser_Y.H
@@ -45,7 +45,13 @@ enum parser_f1_t {  // Built-in functions with one argument
     PARSER_COMP_ELLINT_2
 };
 
-static constexpr std::string_view parser_f1_s[] =
+static
+#if defined(__INTEL_COMPILER) && defined(__EDG__)
+    const
+#else
+    constexpr
+#endif
+std::string_view parser_f1_s[] =
 {
     "sqrt",
     "exp",
@@ -88,7 +94,13 @@ enum parser_f2_t {  // Built-in functions with two arguments
     PARSER_FMOD
 };
 
-static constexpr std::string_view parser_f2_s[] =
+static
+#if defined(__INTEL_COMPILER) && defined(__EDG__)
+    const
+#else
+    constexpr
+#endif
+std::string_view parser_f2_s[] =
 {
     "pow",
     "atan2",
@@ -111,7 +123,13 @@ enum parser_f3_t { // functions with three arguments
     PARSER_IF
 };
 
-static constexpr std::string_view parser_f3_s[] =
+static
+#if defined(__INTEL_COMPILER) && defined(__EDG__)
+    const
+#else
+    constexpr
+#endif
+std::string_view parser_f3_s[] =
 {
     "if"
 };
@@ -130,7 +148,13 @@ enum parser_node_t {
     PARSER_LIST
 };
 
-static constexpr std::string_view parser_node_s[] =
+static
+#if defined(__INTEL_COMPILER) && defined(__EDG__)
+    const
+#else
+    constexpr
+#endif
+std::string_view parser_node_s[] =
 {
     "number",
     "symbol",

diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H
@@ -62,6 +62,8 @@ public:
     const Geometry& Geom () const noexcept { return m_geom; }
     IndexSpace const* getEBIndexSpace () const noexcept { return m_parent; }
 
+    IntVect const& nGrowVect () const noexcept { return m_ngrow; }
+
     void write_to_chkpt_file (const std::string& fname, bool extend_domain_face, int max_grid_size) const;
 
     bool hasEBInfo () const noexcept { return m_has_eb_info; }

diff --git a/Src/EB/AMReX_EB2_Level.cpp b/Src/EB/AMReX_EB2_Level.cpp
@@ -468,13 +468,7 @@ Level::fillEBCellFlag (FabArray<EBCellFlagFab>& cellflag, const Geometry& geom)
             }
 
             // fix type for each fab
-            fab.setType(FabType::undefined);
-            auto typ = fab.getType(bx);
-            fab.setType(typ);
-            for (int nshrink = 1; nshrink < ng; ++nshrink) {
-                const Box& b = amrex::grow(bx,-nshrink);
-                fab.getType(b);
-            }
+            fab.resetType(ng);
         }
     }
 }

diff --git a/Src/EB/AMReX_EBCellFlag.H b/Src/EB/AMReX_EBCellFlag.H
@@ -317,6 +317,11 @@ public:
      */
     FabType getType (const Box& bx) const noexcept;
 
+    /**
+     * \brief Reset FabType
+     */
+    void resetType (int ng);
+
     /**
      * \brief Returns the number of regular cells in the given Box.
      */

diff --git a/Src/EB/AMReX_EBCellFlag.cpp b/Src/EB/AMReX_EBCellFlag.cpp
@@ -128,6 +128,21 @@ EBCellFlagFab::getType (const Box& bx_in) const noexcept
     }
 }
 
+void
+EBCellFlagFab::resetType (int ng)
+{
+    this->setType(FabType::undefined);
+    m_typemap.clear();
+
+    Box const& bx = this->box();
+    auto typ = this->getType(bx);
+    this->setType(typ);
+    for (int nshrink = 1; nshrink < ng; ++nshrink) {
+        const Box& b = amrex::grow(bx,-nshrink);
+        this->getType(b);
+    }
+}
+
 int
 EBCellFlagFab::getNumRegularCells (const Box& bx_in) const noexcept
 {