Add cmake flags for using external amrex library and minor tweaks for…

… multiblock (erf-model#1785)

* add internal resuse of amrex library

* Updates for multiblock

---------

Co-authored-by: Aaron M. Lattanzi <103702284+AMLattanzi@users.noreply.github.com>

CMake/BuildERFExe.cmake

@@ -255,7 +255,7 @@ function(build_erf_lib erf_lib_name)
   endif()
 
   #Link to amrex library
-  target_link_libraries_system(${erf_lib_name} PUBLIC amrex)
+  target_link_libraries_system(${erf_lib_name} PUBLIC AMReX::amrex)
   if(ERF_ENABLE_CUDA)
     set(pctargets "${erf_lib_name}")
     foreach(tgt IN LISTS pctargets)

CMake/ERFConfig.cmake.in

@@ -0,0 +1,17 @@
+@PACKAGE_INIT@
+
+find_package(AMReX QUIET REQUIRED @AMREX_COMPONENTS@)
+
+include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
+
+set(@PROJECT_NAME@_INCLUDE_DIRS "${PROJECT_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@")
+set(@PROJECT_NAME@_LIBRARY_DIRS "${PROJECT_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@")
+set(@PROJECT_NAME@_LIBRARIES "@PROJECT_NAME@::erf_srclib")
+
+# CMake targets aliases: last dimension built will be our legacy target
+if (NOT TARGET @PROJECT_NAME@::erf_srclib")  # protection in case of multiple inclusions
+    add_library(@PROJECT_NAME@::erf_srclib ALIAS erf_srclib)
+endif()
+
+set(@PROJECT_NAME@_FOUND TRUE)
+check_required_components(@PROJECT_NAME@)

CMakeLists.txt

@@ -13,6 +13,7 @@ option(ERF_ENABLE_DOCUMENTATION "Build documentation" OFF)
 option(ERF_ENABLE_ALL_WARNINGS "Enable all compiler warnings" OFF)
 option(ERF_ENABLE_TESTS "Enable regression and unit tests" OFF)
 option(ERF_ENABLE_REGRESSION_TESTS_ONLY "Enable only regression tests" OFF)
+option(ERF_USE_INTERNAL_AMREX "Add AMReX as subproject" ON)
 option(ERF_ENABLE_NETCDF "Enable NetCDF IO" OFF)
 option(ERF_ENABLE_HDF5 "Enable HDF5 IO" ${ERF_ENABLE_NETCDF})
 option(ERF_ENABLE_PARTICLES "Enable Lagrangian particles" OFF)
@@ -66,13 +67,51 @@ endif()
 
 ########################### AMReX #####################################
 
-set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/AMReX")
-include(${CMAKE_SOURCE_DIR}/CMake/SetAmrexOptions.cmake)
-list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake")
+if (${ERF_USE_INTERNAL_AMREX})
+    set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/AMReX")
+    include(${CMAKE_SOURCE_DIR}/CMake/SetAmrexOptions.cmake)
+    list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake")
+#    if (ERF_ENABLE_CUDA AND (CMAKE_VERSION VERSION_LESS 3.20))
+#      include(AMReX_SetupCUDA)
+#    endif()
 
 ########################### AMReX #####################################
 
-add_subdirectory(${AMREX_SUBMOD_LOCATION})
+    add_subdirectory(${AMREX_SUBMOD_LOCATION})
+    set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/amrex_fcompare
+      CACHE INTERNAL "Path to fcompare executable for regression tests")
+else()
+    set(CMAKE_PREFIX_PATH ${AMREX_DIR} ${CMAKE_PREFIX_PATH})
+    list(APPEND AMREX_COMPONENTS
+      "3D" "PIC" "PARTICLES" "PDOUBLE" "DOUBLE" "LSOLVERS")
+    if (ERF_ENABLE_MPI)
+      list(APPEND AMREX_COMPONENTS "MPI")
+    endif()
+    if (ERF_ENABLE_OPENMP)
+      list(APPEND AMREX_COMPONENTS "OMP")
+    endif()
+    if (ERF_ENABLE_CUDA)
+      list(APPEND AMREX_COMPONENTS "CUDA")
+    endif()
+    if (ERF_ENABLE_SYCL)
+      list(APPEND AMREX_COMPONENTS "SYCL")
+    endif()
+    if (ERF_ENABLE_ROCM)
+      list(APPEND AMREX_COMPONENTS "HIP")
+    endif()
+    if (ERF_ENABLE_HYPRE)
+      list(APPEND AMREX_COMPONENTS "HYPRE")
+    endif()
+    if (ERF_ENABLE_TINY_PROFILE)
+      list(APPEND AMREX_COMPONENTS "TINY_PROFILE")
+    endif()
+    separate_arguments(AMREX_COMPONENTS)
+    find_package(AMReX CONFIG REQUIRED
+      COMPONENTS ${AMREX_COMPONENTS})
+    message(STATUS "Found AMReX = ${AMReX_DIR}")
+    set(FCOMPARE_EXE ${AMReX_DIR}/../../../bin/amrex_fcompare
+      CACHE INTERNAL "Path to fcompare executable for regression tests")
+endif()
 
 ########################## NETCDF ##################################
 
@@ -145,6 +184,12 @@ configure_file(
   @ONLY
 )
 
+configure_package_config_file(
+  CMake/${PROJECT_NAME}Config.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
+  INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
+  )
+
 # General information about machine, compiler, and build type
 message(STATUS "ERF Information:")
 message(STATUS "CMAKE_SYSTEM_NAME = ${CMAKE_SYSTEM_NAME}")

Exec/DevTests/MultiBlock/prob.cpp

@@ -45,7 +45,7 @@ Problem::init_custom_pert(
     Array4<Real const> const& /*mf_m*/,
     Array4<Real const> const& /*mf_u*/,
     Array4<Real const> const& /*mf_v*/,
-    const SolverChoice&)
+    const SolverChoice& sc)
 {
   const int khi = geomdata.Domain().bigEnd()[2];
 

Source/ERF.H

@@ -375,7 +375,7 @@ public:
 
     // Velocity time averaged field
     amrex::Vector<std::unique_ptr<amrex::MultiFab>> vel_t_avg;
-    amrex::Vector<int> t_avg_cnt;
+    amrex::Vector<amrex::Real> t_avg_cnt;
 
 #endif
     std::string pp_prefix {"erf"};