Skip to content

Commit

Permalink
move linear solvers into separate directory
Browse files Browse the repository at this point in the history
  • Loading branch information
asalmgren committed Nov 22, 2024
1 parent a701d04 commit 89aa1d7
Show file tree
Hide file tree
Showing 15 changed files with 595 additions and 83 deletions.
15 changes: 9 additions & 6 deletions CMake/BuildERFExe.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,15 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/EB/ERF_eb_cylinder.cpp
${SRC_DIR}/EB/ERF_eb_regular.cpp
${SRC_DIR}/EB/ERF_initEB.cpp
${SRC_DIR}/EB/ERF_writeEBsurface.cpp)
${SRC_DIR}/EB/ERF_writeEBsurface.cpp
${SRC_DIR}/LinearSolvers/ERF_solve_with_EB_mlmg.cpp)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/EB>)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_EB)
endif()

if(ERF_ENABLE_FFT)
target_sources(${erf_lib_name} PRIVATE
${SRC_DIR}/Utils/ERF_solve_with_fft.cpp)
${SRC_DIR}/LinearSolvers/ERF_solve_with_fft.cpp)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_FFT)
endif()

Expand Down Expand Up @@ -189,10 +190,11 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/Utils/ERF_AverageDown.cpp
${SRC_DIR}/Utils/ERF_ChopGrids.cpp
${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp
${SRC_DIR}/Utils/ERF_solve_with_gmres.cpp
${SRC_DIR}/Utils/ERF_solve_with_mlmg.cpp
${SRC_DIR}/LinearSolvers/ERF_PoissonSolve.cpp
${SRC_DIR}/LinearSolvers/ERF_PoissonSolve_tb.cpp
${SRC_DIR}/LinearSolvers/ERF_solve_with_gmres.cpp
${SRC_DIR}/LinearSolvers/ERF_solve_with_mlmg.cpp
${SRC_DIR}/LinearSolvers/ERF_TerrainPoisson.cpp
${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
${SRC_DIR}/Utils/ERF_VelocityToMomentum.cpp
${SRC_DIR}/Utils/ERF_InteriorGhostCells.cpp
Expand Down Expand Up @@ -244,6 +246,7 @@ endif()
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Diffusion>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Initialization>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/IO>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LinearSolvers>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/PBL>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/SourceTerms>)
target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/TimeIntegration>)
Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ target_link_libraries(erf_api PUBLIC erf_srclib)
add_library(${PROJECT_NAME}::erf_api ALIAS erf_srclib)

# Collect all headers and make them installable with the target
set(ERF_INCLUDES "Source/ERF.H;Source/ERF_Constants.H;Source/WindFarmParametrization/SimpleActuatorDisk/ERF_SimpleAD.H;Source/WindFarmParametrization/EWP/ERF_EWP.H;Source/WindFarmParametrization/Null/ERF_NullWindFarm.H;Source/WindFarmParametrization/ERF_WindFarm.H;Source/WindFarmParametrization/Fitch/ERF_Fitch.H;Source/BoundaryConditions/ERF_PhysBCFunct.H;Source/BoundaryConditions/ERF_MOSTAverage.H;Source/BoundaryConditions/ERF_MOSTRoughness.H;Source/BoundaryConditions/ERF_ABLMost.H;Source/BoundaryConditions/ERF_FillPatcher.H;Source/BoundaryConditions/ERF_MOSTStress.H;Source/BoundaryConditions/ERF_TimeInterpolatedData.H;Source/Utils/ERF_Interpolation.H;Source/Utils/ERF_TileNoZ.H;Source/Utils/ERF_PlaneAverage.H;Source/Utils/ERF_Interpolation_WENO.H;Source/Utils/ERF_DirectionSelector.H;Source/Utils/ERF_ParFunctions.H;Source/Utils/ERF_Wstar.H;Source/Utils/ERF_Microphysics_Utils.H;Source/Utils/ERF_Sat_methods.H;Source/Utils/ERF_Interpolation_1D.H;Source/Utils/ERF_Interpolation_UPW.H;Source/Utils/ERF_TerrainMetrics.H;Source/Utils/ERF_Interpolation_WENO_Z.H;Source/Utils/ERF_Thetav.H;Source/Utils/ERF_Water_vapor_saturation.H;Source/Utils/ERF_Utils.H;Source/Utils/ERF_Orbit.H;Source/Utils/ERF_EOS.H;Source/Utils/ERF_HSE_utils.H;Source/EB/ERF_TerrainIF.H;Source/EB/ERF_FlowerIF.H;Source/EB/ERF_eb_if.H;Source/Particles/ERFPC.H;Source/Particles/ERF_ParticleData.H;Source/Prob/ERF_init_density_hse_dry.H;Source/Prob/ERF_init_rayleigh_damping.H;Source/Prob/ERF_init_constant_density_hse.H;Source/ERF_prob_common.H;Source/ERF_Derive.H;Source/Radiation/ERF_Mam4_constituents.H;Source/Radiation/ERF_Mam4_aero.H;Source/Radiation/ERF_Optics.H;Source/Radiation/ERF_Modal_aero_wateruptake.H;Source/Radiation/ERF_Cloud_rad_props.H;Source/Radiation/ERF_Phys_prop.H;Source/Radiation/ERF_Radiation.H;Source/Radiation/ERF_Albedo.H;Source/Radiation/ERF_Parameterizations.H;Source/Radiation/ERF_Rad_constants.H;Source/Radiation/ERF_Aero_rad_props.H;Source/Radiation/ERF_m2005_effradius.H;Source/Radiation/ERF_Linear_interpolate.H;Source/Radiation/ERF_Slingo.H;Source/Radiation/ERF_Rrtmgp.H;Source/Radiation/ERF_Ebert_curry.H;Source/SourceTerms/ERF_NumericalDiffusion.H;Source/SourceTerms/ERF_Src_headers.H;Source/IO/ERF_SampleData.H;Source/IO/ERF_NCInterface.H;Source/IO/ERF_NCWpsFile.H;Source/IO/ERF_NCPlotFile.H;Source/IO/ERF_ReadBndryPlanes.H;Source/IO/ERF_WriteBndryPlanes.H;Source/PBL/ERF_MYNNStruct.H;Source/PBL/ERF_PBLModels.H;Source/PBL/ERF_PBLHeight.H;Source/TimeIntegration/ERF_TI_substep_fun.H;Source/TimeIntegration/ERF_TI_slow_headers.H;Source/TimeIntegration/ERF_TI_slow_rhs_fun.H;Source/TimeIntegration/ERF_TI_fast_headers.H;Source/TimeIntegration/ERF_TI_utils.H;Source/TimeIntegration/ERF_MRI.H;Source/TimeIntegration/ERF_TI_no_substep_fun.H;Source/LandSurfaceModel/Null/ERF_NullSurf.H;Source/LandSurfaceModel/ERF_LandSurface.H;Source/LandSurfaceModel/MM5/ERF_MM5.H;Source/LandSurfaceModel/SLM/ERF_SLM.H;Source/ERF_IndexDefines.H;Source/Advection/ERF_AdvectionSrcForMom_N.H;Source/Advection/ERF_AdvectionSrcForScalars.H;Source/Advection/ERF_AdvectionSrcForMom_T.H;Source/Advection/ERF_Advection.H;Source/MultiBlock/ERF_MultiBlockContainer.H;Source/Initialization/ERF_Metgrid_utils.H;Source/Diffusion/ERF_EddyViscosity.H;Source/Diffusion/ERF_Diffusion.H;Source/Microphysics/Null/ERF_NullMoistLagrangian.H;Source/Microphysics/Null/ERF_NullMoist.H;Source/Microphysics/ERF_Microphysics.H;Source/Microphysics/ERF_LagrangianMicrophysics.H;Source/Microphysics/ERF_EulerianMicrophysics.H;Source/Microphysics/Kessler/ERF_Kessler.H;Source/Microphysics/SAM/ERF_SAM.H;Source/DataStructs/ERF_InputSpongeData.H;Source/DataStructs/ERF_TurbPertStruct.H;Source/DataStructs/ERF_SpongeStruct.H;Source/DataStructs/ERF_AdvStruct.H;Source/DataStructs/ERF_DataStruct.H;Source/DataStructs/ERF_InputSoundingData.H;Source/DataStructs/ERF_DiffStruct.H;Source/DataStructs/ERF_TurbStruct.H")
set(ERF_INCLUDES "Source/ERF.H;Source/ERF_Constants.H;Source/WindFarmParametrization/SimpleActuatorDisk/ERF_SimpleAD.H;Source/WindFarmParametrization/EWP/ERF_EWP.H;Source/WindFarmParametrization/Null/ERF_NullWindFarm.H;Source/WindFarmParametrization/ERF_WindFarm.H;Source/WindFarmParametrization/Fitch/ERF_Fitch.H;Source/BoundaryConditions/ERF_PhysBCFunct.H;Source/BoundaryConditions/ERF_MOSTAverage.H;Source/BoundaryConditions/ERF_MOSTRoughness.H;Source/BoundaryConditions/ERF_ABLMost.H;Source/BoundaryConditions/ERF_FillPatcher.H;Source/BoundaryConditions/ERF_MOSTStress.H;Source/BoundaryConditions/ERF_TimeInterpolatedData.H;Source/Utils/ERF_Interpolation.H;Source/Utils/ERF_TileNoZ.H;Source/Utils/ERF_PlaneAverage.H;Source/Utils/ERF_Interpolation_WENO.H;Source/Utils/ERF_DirectionSelector.H;Source/Utils/ERF_ParFunctions.H;Source/Utils/ERF_Wstar.H;Source/Utils/ERF_Microphysics_Utils.H;Source/Utils/ERF_Sat_methods.H;Source/Utils/ERF_Interpolation_1D.H;Source/Utils/ERF_Interpolation_UPW.H;Source/Utils/ERF_TerrainMetrics.H;Source/Utils/ERF_Interpolation_WENO_Z.H;Source/Utils/ERF_Thetav.H;Source/Utils/ERF_Water_vapor_saturation.H;Source/Utils/ERF_Utils.H;Source/Utils/ERF_Orbit.H;Source/Utils/ERF_EOS.H;Source/Utils/ERF_HSE_utils.H;Source/EB/ERF_TerrainIF.H;Source/EB/ERF_FlowerIF.H;Source/EB/ERF_eb_if.H;Source/Particles/ERFPC.H;Source/Particles/ERF_ParticleData.H;Source/Prob/ERF_init_density_hse_dry.H;Source/Prob/ERF_init_rayleigh_damping.H;Source/Prob/ERF_init_constant_density_hse.H;Source/ERF_prob_common.H;Source/ERF_Derive.H;Source/Radiation/ERF_Mam4_constituents.H;Source/Radiation/ERF_Mam4_aero.H;Source/Radiation/ERF_Optics.H;Source/Radiation/ERF_Modal_aero_wateruptake.H;Source/Radiation/ERF_Cloud_rad_props.H;Source/Radiation/ERF_Phys_prop.H;Source/Radiation/ERF_Radiation.H;Source/Radiation/ERF_Albedo.H;Source/Radiation/ERF_Parameterizations.H;Source/Radiation/ERF_Rad_constants.H;Source/Radiation/ERF_Aero_rad_props.H;Source/Radiation/ERF_m2005_effradius.H;Source/Radiation/ERF_Linear_interpolate.H;Source/Radiation/ERF_Slingo.H;Source/Radiation/ERF_Rrtmgp.H;Source/Radiation/ERF_Ebert_curry.H;Source/SourceTerms/ERF_NumericalDiffusion.H;Source/SourceTerms/ERF_Src_headers.H;Source/IO/ERF_SampleData.H;Source/IO/ERF_NCInterface.H;Source/IO/ERF_NCWpsFile.H;Source/IO/ERF_NCPlotFile.H;Source/IO/ERF_ReadBndryPlanes.H;Source/IO/ERF_WriteBndryPlanes.H;Source/PBL/ERF_MYNNStruct.H;Source/PBL/ERF_PBLModels.H;Source/PBL/ERF_PBLHeight.H;Source/TimeIntegration/ERF_TI_substep_fun.H;Source/TimeIntegration/ERF_TI_slow_headers.H;Source/TimeIntegration/ERF_TI_slow_rhs_fun.H;Source/TimeIntegration/ERF_TI_fast_headers.H;Source/TimeIntegration/ERF_TI_utils.H;Source/TimeIntegration/ERF_MRI.H;Source/TimeIntegration/ERF_TI_no_substep_fun.H;Source/LandSurfaceModel/Null/ERF_NullSurf.H;Source/LandSurfaceModel/ERF_LandSurface.H;Source/LandSurfaceModel/MM5/ERF_MM5.H;Source/LandSurfaceModel/SLM/ERF_SLM.H;Source/ERF_IndexDefines.H;Source/Advection/ERF_AdvectionSrcForMom_N.H;Source/Advection/ERF_AdvectionSrcForScalars.H;Source/Advection/ERF_AdvectionSrcForMom_T.H;Source/Advection/ERF_Advection.H;Source/MultiBlock/ERF_MultiBlockContainer.H;Source/Initialization/ERF_Metgrid_utils.H;Source/Diffusion/ERF_EddyViscosity.H;Source/Diffusion/ERF_Diffusion.H;Source/Microphysics/Null/ERF_NullMoistLagrangian.H;Source/Microphysics/Null/ERF_NullMoist.H;Source/Microphysics/ERF_Microphysics.H;Source/Microphysics/ERF_LagrangianMicrophysics.H;Source/Microphysics/ERF_EulerianMicrophysics.H;Source/Microphysics/Kessler/ERF_Kessler.H;Source/Microphysics/SAM/ERF_SAM.H;Source/DataStructs/ERF_InputSpongeData.H;Source/DataStructs/ERF_TurbPertStruct.H;Source/DataStructs/ERF_SpongeStruct.H;Source/DataStructs/ERF_AdvStruct.H;Source/DataStructs/ERF_DataStruct.H;Source/DataStructs/ERF_InputSoundingData.H;Source/DataStructs/ERF_DiffStruct.H;Source/DataStructs/ERF_TurbStruct.H ERF_TerrainPoisson.H")
set_target_properties(
erf_srclib PROPERTIES PUBLIC_HEADER "${ERF_INCLUDES}")

Expand Down
6 changes: 6 additions & 0 deletions Exec/Make.ERF
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ include $(ERF_INIT_DIR)/Make.package
ERF_DATA_DIR = $(ERF_SOURCE_DIR)/DataStructs
include $(ERF_DATA_DIR)/Make.package

ERF_SOLVERS_DIR = $(ERF_SOURCE_DIR)/LinearSolvers
include $(ERF_SOLVERS_DIR)/Make.package

ERF_UTIL_DIR = $(ERF_SOURCE_DIR)/Utils
include $(ERF_UTIL_DIR)/Make.package

Expand All @@ -57,6 +60,9 @@ INCLUDE_LOCATIONS += $(ERF_DATA_DIR)
VPATH_LOCATIONS += $(ERF_UTIL_DIR)
INCLUDE_LOCATIONS += $(ERF_UTIL_DIR)

VPATH_LOCATIONS += $(ERF_SOLVERS_DIR)
INCLUDE_LOCATIONS += $(ERF_SOLVERS_DIR)

ifeq ($(USE_MULTIBLOCK),TRUE)
VPATH_LOCATIONS += $(ERF_MULTIBLOCK_DIR)
INCLUDE_LOCATIONS += $(ERF_MULTIBLOCK_DIR)
Expand Down
File renamed without changes.
File renamed without changes.
51 changes: 51 additions & 0 deletions Source/LinearSolvers/ERF_TerrainPoisson.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#ifndef ERF_TERRAIN_POISSON_H_
#define ERF_TERRAIN_POISSON_H_

#include "ERF_TerrainPoisson_3D_K.H"

#include <AMReX_Geometry.H>
#include <AMReX_MultiFab.H>

class TerrainPoisson
{
public:

using RT = amrex::Real;

TerrainPoisson (amrex::Geometry const& geom, amrex::BoxArray const& ba,
amrex::DistributionMapping const& dm,
amrex::MultiFab const* z_phys_nd);

void apply(amrex::MultiFab& lhs, amrex::MultiFab const& rhs);

void getFluxes(amrex::MultiFab const& phi, amrex::Array<amrex::MultiFab,AMREX_SPACEDIM>& fluxes);

void assign(amrex::MultiFab& lhs, amrex::MultiFab const& rhs);

void scale(amrex::MultiFab& lhs, amrex::Real fac);

amrex::Real dotProduct(amrex::MultiFab const& v1, amrex::MultiFab const& v2);

void increment(amrex::MultiFab& lhs, amrex::MultiFab const& rhs, amrex::Real a);

void linComb(amrex::MultiFab& lhs, amrex::Real a, amrex::MultiFab const& rhs_a,
amrex::Real b, amrex::MultiFab const& rhs_b);

amrex::MultiFab makeVecRHS();

amrex::MultiFab makeVecLHS();

amrex::Real norm2(amrex::MultiFab const& v);

void precond(amrex::MultiFab& lhs, amrex::MultiFab const& rhs);

void setToZero(amrex::MultiFab& v);

private:
amrex::Geometry m_geom;
amrex::BoxArray m_grids;
amrex::DistributionMapping m_dmap;
const amrex::MultiFab* m_zphys;
};

#endif
137 changes: 137 additions & 0 deletions Source/LinearSolvers/ERF_TerrainPoisson.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#include "ERF_TerrainPoisson.H"

using namespace amrex;


TerrainPoisson::TerrainPoisson (amrex::Geometry const& geom, amrex::BoxArray const& ba,
amrex::DistributionMapping const& dm,
amrex::MultiFab const* z_phys_nd)
: m_geom(geom),
m_grids(ba),
m_dmap(dm),
m_zphys(z_phys_nd)
{
}

void TerrainPoisson::apply(amrex::MultiFab& lhs, amrex::MultiFab const& rhs)
{
AMREX_ASSERT(rhs.nGrowVect().allGT(0));

auto domlo = lbound(m_geom.Domain());
auto domhi = ubound(m_geom.Domain());

MultiFab& xx = const_cast<MultiFab&>(rhs);

auto const& dxinv = m_geom.InvCellSizeArray();

auto const& y = lhs.arrays();
auto const& x = xx.arrays();
auto const& zpa = m_zphys->const_arrays();

// Impose periodic and internal boundary conditions
xx.FillBoundary(m_geom.periodicity());

// Impose top and bottom Neumann bcs
amrex::ParallelFor(xx, [=] AMREX_GPU_DEVICE (int b, int i, int j, int k)
{
if (k == domlo.z) {
x[b](i,j,k-1) = x[b](i,j,k);
} else if (k == domhi.z) {
x[b](i,j,k+1) = x[b](i,j,k);
}
});

auto const& xc = xx.const_arrays();
amrex::ParallelFor(rhs, [=] AMREX_GPU_DEVICE (int b, int i, int j, int k)
{
terrpoisson_adotx(i,j,k,y[b], xc[b], zpa[b], dxinv[0], dxinv[1], dxinv[2]);
});
}

void TerrainPoisson::getFluxes(amrex::MultiFab const& phi,
amrex::Array<amrex::MultiFab,AMREX_SPACEDIM>& fluxes)
{
auto const& dxinv = m_geom.InvCellSizeArray();

auto domlo = lbound(m_geom.Domain());
auto domhi = ubound(m_geom.Domain());

auto const& x = phi.const_arrays();
auto const& zpa = m_zphys->const_arrays();

auto const& fx = fluxes[0].arrays();
amrex::ParallelFor(fluxes[0], [=] AMREX_GPU_DEVICE (int b, int i, int j, int k)
{
terrpoisson_flux_x(i,j,k,fx[b],x[b],zpa[b],dxinv[0],dxinv[2]);
});

auto const& fy = fluxes[1].arrays();
amrex::ParallelFor(fluxes[1], [=] AMREX_GPU_DEVICE (int b, int i, int j, int k)
{
terrpoisson_flux_y(i,j,k,fy[b],x[b],zpa[b],dxinv[1],dxinv[2]);
});

auto const& fz = fluxes[2].arrays();
amrex::ParallelFor(fluxes[2], [=] AMREX_GPU_DEVICE (int b, int i, int j, int k)
{
terrpoisson_flux_z(i,j,k,fz[b],x[b],zpa[b],dxinv[0],dxinv[1],dxinv[2]);

if (k == domlo.z) {
fz[b](i,j,k) = 0.0;
} else if (k == domhi.z+1) {
fz[b](i,j,k) = 0.0;
}
});
}

void TerrainPoisson::assign(amrex::MultiFab& lhs, amrex::MultiFab const& rhs)
{
MultiFab::Copy(lhs, rhs, 0, 0, 1, 0);
}

void TerrainPoisson::scale(amrex::MultiFab& lhs, amrex::Real fac)
{
lhs.mult(fac);
}

Real TerrainPoisson::dotProduct(amrex::MultiFab const& v1, amrex::MultiFab const& v2)
{
return MultiFab::Dot(v1, 0, v2, 0, 1, 0);
}

void TerrainPoisson::increment(amrex::MultiFab& lhs, amrex::MultiFab const& rhs, Real a)
{
MultiFab::Saxpy(lhs, a, rhs, 0, 0, 1, 0);
}

void TerrainPoisson::linComb(amrex::MultiFab& lhs, Real a, amrex::MultiFab const& rhs_a,
Real b, amrex::MultiFab const& rhs_b)
{
MultiFab::LinComb(lhs, a, rhs_a, 0, b, rhs_b, 0, 0, 1, 0);
}


MultiFab TerrainPoisson::makeVecRHS()
{
return MultiFab(m_grids, m_dmap, 1, 0);
}

MultiFab TerrainPoisson::makeVecLHS()
{
return MultiFab(m_grids, m_dmap, 1, 1);
}

Real TerrainPoisson::norm2(amrex::MultiFab const& v)
{
return v.norm2();
}

void TerrainPoisson::precond(amrex::MultiFab& lhs, amrex::MultiFab const& rhs)
{
MultiFab::Copy(lhs, rhs, 0, 0, 1, 0);
}

void TerrainPoisson::setToZero(amrex::MultiFab& v)
{
v.setVal(0);
}
Loading

0 comments on commit 89aa1d7

Please sign in to comment.