From a5a9e9eebccce5502ec91a80b6672ad3dd05f692 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 5 Sep 2024 22:52:39 -0400 Subject: [PATCH 01/32] Update use of numeric_limits::max --- .../mesh/utilities/ComputationalGeometry.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/mesh/utilities/ComputationalGeometry.hpp b/src/coreComponents/mesh/utilities/ComputationalGeometry.hpp index 1670a351364..7dc14fca186 100644 --- a/src/coreComponents/mesh/utilities/ComputationalGeometry.hpp +++ b/src/coreComponents/mesh/utilities/ComputationalGeometry.hpp @@ -551,7 +551,7 @@ int findVertexRefElement( arraySlice1d< localIndex const > const & nodeElements, arrayView1d< globalIndex const > const & elementGlobalIndex ) { localIndex minElement = -1; - globalIndex minElementGID = std::numeric_limits< globalIndex >::max(); + globalIndex minElementGID = LvArray::NumericLimits< globalIndex >::max; for( int i = 0; i < nodeElements.size(); i++ ) { localIndex e = nodeElements( i ); @@ -579,7 +579,7 @@ int findEdgeRefElement( arraySlice1d< localIndex const > const & nodeElements1, arrayView1d< globalIndex const > const & elementGlobalIndex ) { localIndex minElement = -1; - globalIndex minElementGID = std::numeric_limits< globalIndex >::max(); + globalIndex minElementGID = LvArray::NumericLimits< globalIndex >::max; for( int i = 0; i < nodeElements1.size(); i++ ) { localIndex e1 = nodeElements1( i ); @@ -616,7 +616,7 @@ int findTriangleRefElement( arraySlice1d< localIndex const > const & nodeElement arrayView1d< globalIndex const > const & elementGlobalIndex ) { localIndex minElement = -1; - globalIndex minElementGID = std::numeric_limits< globalIndex >::max(); + globalIndex minElementGID = LvArray::NumericLimits< globalIndex >::max; for( int i = 0; i < nodeElements1.size(); i++ ) { localIndex e1 = nodeElements1( i ); @@ -674,7 +674,7 @@ bool computeWindingNumber( localIndex element, // triangulate the face. The triangulation must be done in a consistent way across ranks. // This can be achieved by always picking the vertex with the lowest global index as root. localIndex const faceIndex = faceIndices[kf]; - globalIndex minGlobalId = std::numeric_limits< globalIndex >::max(); + globalIndex minGlobalId = LvArray::NumericLimits< globalIndex >::max; localIndex minVertex = -1; localIndex numFaceVertices = facesToNodes[faceIndex].size(); for( localIndex v = 0; v < numFaceVertices; v++ ) From 7a67fe26d5fe2cdebbaa6cefe6c3845b722d3cce Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 8 Sep 2024 22:41:47 -0400 Subject: [PATCH 02/32] Replace logic for performing SDC in GEOS with HYPRE_BoomerAMGSetFilterFunctions --- .../interfaces/hypre/HypreInterface.cpp | 11 ++++++- .../interfaces/hypre/HypreMGR.hpp | 2 ++ .../interfaces/hypre/HyprePreconditioner.cpp | 26 ----------------- .../interfaces/hypre/HyprePreconditioner.hpp | 29 ------------------- .../interfaces/hypre/HypreSolver.cpp | 20 +++++-------- .../interfaces/hypre/HypreSolver.hpp | 5 ---- .../mgrStrategies/MultiphasePoromechanics.hpp | 24 +++++++-------- 7 files changed, 32 insertions(+), 85 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index f13477eecc5..a3bd5c4d19e 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -45,7 +45,7 @@ void HypreInterface::initialize() "OMP_NUM_THREADS > 1 may not be optimal for certain hypre preconditioning options. " ); #endif - HYPRE_Init(); + HYPRE_Initialize(); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP HYPRE_SetExecutionPolicy( HYPRE_EXEC_DEVICE ); HYPRE_SetSpGemmUseVendor( 0 ); @@ -53,6 +53,15 @@ void HypreInterface::initialize() #endif HYPRE_SetMemoryLocation( hypre::memoryLocation ); HYPRE_SetPrintErrorMode( 1 ); + +#if defined(HYPRE_USING_UMPIRE) + HYPRE_SetUmpireUMPoolName( "HYPRE_UM" ); + HYPRE_SetUmpireHostPoolName( "HYPRE_HOST" ); + HYPRE_SetUmpireDevicePoolName( "HYPRE_DEVICE" ); + HYPRE_SetUmpirePinnedPoolName( "HYPRE_PINNED" ); +#endif + + HYPRE_SetLogLevel( getenv( "HYPRE_LOG_LEVEL" ) ? atoi( getenv( "HYPRE_LOG_LEVEL" ) ) : 0 ); } void HypreInterface::finalize() diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index eef5b70b45b..72d1e21d5b3 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -167,6 +167,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelSmoothIters( precond.ptr, m_levelGlobalSmootherIters ) ); GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetTruncateCoarseGridThreshold( precond.ptr, m_coarseGridThreshold ) ); GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetNonCpointsToFpoints( precond.ptr, 1 )); + GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetNonGalerkinMaxElmts( precond.ptr, 1 )); } /** @@ -191,6 +192,7 @@ class MGRStrategyBase #endif GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( solver.ptr, 3 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( solver.ptr, 1 ) ); solver.setup = HYPRE_BoomerAMGSetup; solver.solve = HYPRE_BoomerAMGSolve; diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp index cd4bc6321e9..dbd4fb6a56e 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp @@ -340,32 +340,6 @@ void HyprePreconditioner::create( DofManager const * const dofManager ) HypreMatrix const & HyprePreconditioner::setupPreconditioningMatrix( HypreMatrix const & mat ) { - GEOS_MARK_FUNCTION; - - if( m_params.preconditionerType == LinearSolverParameters::PreconditionerType::mgr && m_params.mgr.separateComponents ) - { - GEOS_LAI_ASSERT_MSG( mat.dofManager() != nullptr, "MGR preconditioner requires a DofManager instance" ); - HypreMatrix Pu; - HypreMatrix Auu; - { - Stopwatch timer( m_makeRestrictorTime ); - mat.dofManager()->makeRestrictor( { { m_params.mgr.displacementFieldName, { 3, true } } }, mat.comm(), true, Pu ); - } - { - Stopwatch timer( m_computeAuuTime ); - mat.multiplyPtAP( Pu, Auu ); - } - { - Stopwatch timer( m_componentFilterTime ); - Auu.separateComponentFilter( m_precondMatrix, m_params.dofsPerNode ); - } - } - else if( m_params.preconditionerType == LinearSolverParameters::PreconditionerType::amg && m_params.amg.separateComponents ) - { - Stopwatch timer( m_componentFilterTime ); - mat.separateComponentFilter( m_precondMatrix, m_params.dofsPerNode ); - return m_precondMatrix; - } return mat; } diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp index 1f13028f4e1..ee7382f8ae3 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.hpp @@ -95,26 +95,6 @@ class HyprePreconditioner final : public PreconditionerBase< HypreInterface > */ HyprePrecWrapper const & unwrapped() const; - /** - * @brief @return time spent setting up separate component matrix. - */ - real64 componentFilterTime() const - { - return m_componentFilterTime; - } - - /// @return time to construct restrictor matrix. - real64 makeRestrictorTime() const - { - return m_makeRestrictorTime; - } - - /// @return time to apply restrictor matrix. - real64 computeAuuTime() const - { - return m_computeAuuTime; - } - private: /** @@ -144,15 +124,6 @@ class HyprePreconditioner final : public PreconditionerBase< HypreInterface > /// Null space vectors std::unique_ptr< HypreNullSpace > m_nullSpace; - - /// Timing of separate component matrix construction - real64 m_componentFilterTime = 0.0; - - /// Timing of the restrictor matrix construction - real64 m_makeRestrictorTime = 0.0; - - /// Timing of the cost of applying the restrictor matrix to the system - real64 m_computeAuuTime = 0.0; }; } diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index e882d55faba..2ba57b05056 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -201,11 +201,7 @@ void HypreSolver::setup( HypreMatrix const & mat ) clear(); Base::setup( mat ); Stopwatch timer( m_result.setupTime ); - m_precond.setup( mat ); - m_componentFilterTime = m_precond.componentFilterTime(); - m_makeRestrictorTime = m_precond.makeRestrictorTime(); - m_computeAuuTime = m_precond.computeAuuTime(); m_solver = std::make_unique< HypreSolverWrapper >(); createHypreKrylovSolver( m_params, mat.comm(), *m_solver ); @@ -276,14 +272,14 @@ void HypreSolver::solve( HypreVector const & rhs, if( m_params.logLevel >= 1 ) { - GEOS_LOG_RANK_0( " Linear Solver | " << m_result.status << - " | Iterations: " << m_result.numIterations << - " | Final Rel Res: " << m_result.residualReduction << - " | Make Restrictor Time: " << m_makeRestrictorTime << - " | Compute Auu Time: " << m_computeAuuTime << - " | SC Filter Time: " << m_componentFilterTime << - " | Setup Time: " << m_result.setupTime << " s" << - " | Solve Time: " << m_result.solveTime << " s" ); + + GEOS_LOG_RANK_0( " Linear Solver | " << m_result.status << + " | Iterations: " << m_result.numIterations << + " | Final Rel Res: " << std::scientific << std::setprecision(4) + << m_result.residualReduction + << std::fixed << std::setprecision(3) << + " | Setup Time: " << m_result.setupTime << " s" << + " | Solve Time: " << m_result.solveTime << " s" ); } } diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.hpp index 40364992a06..7cc8c5b5738 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.hpp @@ -93,11 +93,6 @@ class HypreSolver final : public LinearSolverBase< HypreInterface > /// Pointers to hypre functions for the krylov solver std::unique_ptr< HypreSolverWrapper > m_solver; - - /// Time of the most recent SC matrix construction - real64 m_componentFilterTime; - real64 m_makeRestrictorTime; - real64 m_computeAuuTime; }; } // end geos namespace diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp index dcbe732e90e..90367c6cc77 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp @@ -72,20 +72,20 @@ class MultiphasePoromechanics : public MGRStrategyBase< 3 > setupLabels(); // Level 0 - m_levelFRelaxType[0] = MGRFRelaxationType::amgVCycle; - m_levelFRelaxIters[0] = 1; - m_levelInterpType[0] = MGRInterpolationType::jacobi; - m_levelRestrictType[0] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; - m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; + m_levelFRelaxType[0] = MGRFRelaxationType::amgVCycle; + m_levelFRelaxIters[0] = 1; + m_levelInterpType[0] = MGRInterpolationType::jacobi; + m_levelRestrictType[0] = MGRRestrictionType::injection; + m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; + m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; // Level 1 - m_levelFRelaxType[1] = MGRFRelaxationType::jacobi; - m_levelFRelaxIters[1] = 1; - m_levelInterpType[1] = MGRInterpolationType::jacobi; - m_levelRestrictType[1] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[1] = MGRCoarseGridMethod::galerkin; - m_levelGlobalSmootherType[1] = MGRGlobalSmootherType::none; + m_levelFRelaxType[1] = MGRFRelaxationType::jacobi; + m_levelFRelaxIters[1] = 1; + m_levelInterpType[1] = MGRInterpolationType::jacobi; + m_levelRestrictType[1] = MGRRestrictionType::injection; + m_levelCoarseGridMethod[1] = MGRCoarseGridMethod::galerkin; + m_levelGlobalSmootherType[1] = MGRGlobalSmootherType::none; // Level 2 m_levelFRelaxType[2] = MGRFRelaxationType::none; From 9974de5ca1736db011e46829652dd67c6c529999 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 1 Oct 2024 14:17:44 -0400 Subject: [PATCH 03/32] Add Unknowns and Nonzeros info to Linear Solver message --- .../interfaces/hypre/HypreSolver.cpp | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 2ba57b05056..991da11b868 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -272,14 +272,22 @@ void HypreSolver::solve( HypreVector const & rhs, if( m_params.logLevel >= 1 ) { - - GEOS_LOG_RANK_0( " Linear Solver | " << m_result.status << - " | Iterations: " << m_result.numIterations << - " | Final Rel Res: " << std::scientific << std::setprecision(4) - << m_result.residualReduction - << std::fixed << std::setprecision(3) << - " | Setup Time: " << m_result.setupTime << " s" << - " | Solve Time: " << m_result.solveTime << " s" ); + HYPRE_BigInt global_num_rows, global_num_nonzeros; + + GEOS_LAI_CHECK_ERROR( HYPRE_IJMatrixGetGlobalInfo( matrix().unwrappedIJ(), + &global_num_rows, + &global_num_rows, + &global_num_nonzeros) ); + + GEOS_LOG_RANK_0( " Linear Solver | " << m_result.status << + " | Unknowns: " << global_num_rows << + " | Nonzeros: " << global_num_nonzeros << + " | Iterations: " << m_result.numIterations << + " | Final Rel Res: " << std::scientific << std::setprecision(4) + << m_result.residualReduction + << std::fixed << std::setprecision(3) << + " | Setup Time: " << m_result.setupTime << " s" << + " | Solve Time: " << m_result.solveTime << " s" ); } } From b56dd05ea4884f1f96ad6b142e1795d72809f8a3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 13:15:57 -0400 Subject: [PATCH 04/32] Use GEOS_FMT --- .../interfaces/hypre/HypreSolver.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 991da11b868..19ad75d9baf 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -274,20 +274,15 @@ void HypreSolver::solve( HypreVector const & rhs, { HYPRE_BigInt global_num_rows, global_num_nonzeros; + // This involves an MPI collective call, and therefore we call it only when necessary GEOS_LAI_CHECK_ERROR( HYPRE_IJMatrixGetGlobalInfo( matrix().unwrappedIJ(), &global_num_rows, - &global_num_rows, + &global_num_rows, // This is intentional and assuming the matrix is square &global_num_nonzeros) ); - GEOS_LOG_RANK_0( " Linear Solver | " << m_result.status << - " | Unknowns: " << global_num_rows << - " | Nonzeros: " << global_num_nonzeros << - " | Iterations: " << m_result.numIterations << - " | Final Rel Res: " << std::scientific << std::setprecision(4) - << m_result.residualReduction - << std::fixed << std::setprecision(3) << - " | Setup Time: " << m_result.setupTime << " s" << - " | Solve Time: " << m_result.solveTime << " s" ); + GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {} | Nonzeros: {} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", + m_result.status, global_num_rows, global_num_nonzeros, m_result.numIterations, + m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); } } From 1af9315bd60f09bac13ece99e56ad38fa81f5104 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 14:39:43 -0400 Subject: [PATCH 05/32] Add missing coarse solver options --- .../linearAlgebra/interfaces/hypre/HypreUtils.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreUtils.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreUtils.hpp index a205fe074c7..a309557dd81 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreUtils.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreUtils.hpp @@ -399,6 +399,8 @@ inline HYPRE_Int getAMGCoarseType( LinearSolverParameters::AMG::CoarseType const { LinearSolverParameters::AMG::CoarseType::direct, 9 }, { LinearSolverParameters::AMG::CoarseType::chebyshev, 16 }, { LinearSolverParameters::AMG::CoarseType::l1jacobi, 18 }, + { LinearSolverParameters::AMG::CoarseType::gsElimWPivoting, 99 }, + { LinearSolverParameters::AMG::CoarseType::gsElimWInverse, 199 }, }; return findOption( typeMap, type, "multigrid coarse solver", "HyprePreconditioner" ); } From 1529f6fbc7693ff7410a6df345422480f0b973c9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 14:40:04 -0400 Subject: [PATCH 06/32] Remove unused code in GEOS regarding SDC approx. --- .../linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp index dbd4fb6a56e..a9d7dd84014 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp @@ -359,12 +359,6 @@ void HyprePreconditioner::setup( Matrix const & mat ) { LvArray::system::FloatingPointExceptionGuard guard( FE_ALL_EXCEPT ); - // Perform setup of the MGR mechanics F-solver with SDC matrix, if used - if( m_mgrData && m_mgrData->mechSolver.ptr && m_mgrData->mechSolver.setup ) - { -// GEOS_LAI_CHECK_ERROR( m_mgrData->mechSolver.setup( m_mgrData->mechSolver.ptr, m_precondMatrix.unwrapped(), nullptr, nullptr ) ); - } - // Perform setup of the main solver, if needed if( m_precond->setup ) { From 0ecfdf3a5692dd13a6ae5e681b8e19b9dbd6ea15 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 14:43:31 -0400 Subject: [PATCH 07/32] Remove displacementFieldName since it became unused after SDC changes --- .../utilities/LinearSolverParameters.hpp | 13 ++++++++----- .../contact/SolidMechanicsLagrangeContact.cpp | 1 - .../multiphysics/HydrofractureSolver.cpp | 1 - .../multiphysics/MultiphasePoromechanics.cpp | 1 - .../multiphysics/SinglePhasePoromechanics.cpp | 1 - .../SinglePhasePoromechanicsConformingFractures.cpp | 1 - .../SinglePhasePoromechanicsEmbeddedFractures.cpp | 1 - 7 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp b/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp index 90efd13f7ef..0953c97bdec 100644 --- a/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp +++ b/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp @@ -184,7 +184,9 @@ struct LinearSolverParameters l1sgs, ///< l1-Symmetric Gauss-Seidel chebyshev, ///< Chebyshev polynomial (GPU support in hypre) direct, ///< Direct solver as preconditioner - bgs ///< Gauss-Seidel smoothing (backward sweep) + bgs, ///< Gauss-Seidel smoothing (backward sweep) + gsElimWPivoting, ///< Gaussian Elimination with pivoting direct solver + gsElimWInverse ///< Direct inverse with Gaussian Elimination }; /// AMG coarsening types (HYPRE only) @@ -297,9 +299,8 @@ struct LinearSolverParameters StrategyType strategy = StrategyType::invalid; ///< Predefined MGR solution strategy (solver specific) integer separateComponents = false; ///< Apply a separate displacement component (SDC) filter before AMG construction - string displacementFieldName; ///< Displacement field name need for SDC filter - integer areWellsShut = false; ///< Flag to let MGR know that wells are shut, and that jacobi can be applied to the - ///< well block + integer areWellsShut = false; ///< Flag to let MGR know that wells are shut, and that jacobi can be applied to the + ///< well block } mgr; ///< Multigrid reduction (MGR) parameters @@ -422,7 +423,9 @@ ENUM_STRINGS( LinearSolverParameters::AMG::CoarseType, "l1sgs", "chebyshev", "direct", - "bgs" ); + "bgs", + "gsElimWPivoting", + "gsElimWInverse" ); /// Declare strings associated with enumeration values. ENUM_STRINGS( LinearSolverParameters::AMG::CoarseningType, diff --git a/src/coreComponents/physicsSolvers/contact/SolidMechanicsLagrangeContact.cpp b/src/coreComponents/physicsSolvers/contact/SolidMechanicsLagrangeContact.cpp index 2b66ae59da0..43accb9c05f 100644 --- a/src/coreComponents/physicsSolvers/contact/SolidMechanicsLagrangeContact.cpp +++ b/src/coreComponents/physicsSolvers/contact/SolidMechanicsLagrangeContact.cpp @@ -74,7 +74,6 @@ SolidMechanicsLagrangeContact::SolidMechanicsLagrangeContact( const string & nam LinearSolverParameters & linSolParams = m_linearSolverParameters.get(); linSolParams.mgr.strategy = LinearSolverParameters::MGR::StrategyType::lagrangianContactMechanics; linSolParams.mgr.separateComponents = true; - linSolParams.mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); linSolParams.dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp index 301eb93de28..c50b0d6fd46 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp @@ -80,7 +80,6 @@ HydrofractureSolver< POROMECHANICS_SOLVER >::HydrofractureSolver( const string & // This may need to be different depending on whether poroelasticity is on or not. m_linearSolverParameters.get().mgr.strategy = LinearSolverParameters::MGR::StrategyType::hydrofracture; m_linearSolverParameters.get().mgr.separateComponents = false; - m_linearSolverParameters.get().mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); m_linearSolverParameters.get().dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/MultiphasePoromechanics.cpp b/src/coreComponents/physicsSolvers/multiphysics/MultiphasePoromechanics.cpp index 476ebc35a94..42ca4884c4f 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/MultiphasePoromechanics.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/MultiphasePoromechanics.cpp @@ -49,7 +49,6 @@ MultiphasePoromechanics< FLOW_SOLVER, MECHANICS_SOLVER >::MultiphasePoromechanic LinearSolverParameters & linearSolverParameters = this->m_linearSolverParameters.get(); linearSolverParameters.mgr.strategy = LinearSolverParameters::MGR::StrategyType::multiphasePoromechanics; linearSolverParameters.mgr.separateComponents = true; - linearSolverParameters.mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); linearSolverParameters.dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanics.cpp b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanics.cpp index a8899c78569..079a96ea92d 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanics.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanics.cpp @@ -49,7 +49,6 @@ SinglePhasePoromechanics< FLOW_SOLVER, MECHANICS_SOLVER >::SinglePhasePoromechan LinearSolverParameters & linearSolverParameters = this->m_linearSolverParameters.get(); linearSolverParameters.mgr.strategy = LinearSolverParameters::MGR::StrategyType::singlePhasePoromechanics; linearSolverParameters.mgr.separateComponents = true; - linearSolverParameters.mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); linearSolverParameters.dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp index 32f5980986e..b387dea8804 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp @@ -46,7 +46,6 @@ SinglePhasePoromechanicsConformingFractures< FLOW_SOLVER >::SinglePhasePoromecha LinearSolverParameters & params = this->m_linearSolverParameters.get(); params.mgr.strategy = LinearSolverParameters::MGR::StrategyType::singlePhasePoromechanicsConformingFractures; params.mgr.separateComponents = false; - params.mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); params.dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp index d2c1c2ddecb..610211370f5 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp @@ -44,7 +44,6 @@ SinglePhasePoromechanicsEmbeddedFractures::SinglePhasePoromechanicsEmbeddedFract LinearSolverParameters & params = m_linearSolverParameters.get(); params.mgr.strategy = LinearSolverParameters::MGR::StrategyType::singlePhasePoromechanicsEmbeddedFractures; params.mgr.separateComponents = false; - params.mgr.displacementFieldName = solidMechanics::totalDisplacement::key(); params.dofsPerNode = 3; } From 08dd971736cdc96623cd05cc65fb45c9c565df75 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 15:50:31 -0400 Subject: [PATCH 08/32] Update a few MGR parameters + add separateComponents as input parameter in several functions --- .../interfaces/hypre/HypreMGR.hpp | 26 ++++++++-------- .../HybridSinglePhasePoromechanics.hpp | 5 ++-- .../hypre/mgrStrategies/Hydrofracture.hpp | 5 ++-- .../LagrangianContactMechanics.hpp | 4 +-- .../mgrStrategies/MultiphasePoromechanics.hpp | 5 ++-- .../MultiphasePoromechanicsReservoirFVM.hpp | 4 +-- .../SinglePhasePoromechanics.hpp | 5 ++-- .../SinglePhasePoromechanicsReservoirFVM.hpp | 5 ++-- .../SolidMechanicsEmbeddedFractures.hpp | 4 +-- .../ThermalMultiphasePoromechanics.hpp | 30 +++++++++---------- .../ThermalSinglePhasePoromechanics.hpp | 6 ++-- 11 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 72d1e21d5b3..afb21f1256d 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -91,11 +91,7 @@ class MGRStrategyBase MGRCoarseGridMethod m_levelCoarseGridMethod[numLevels]; ///< Coarse grid method for each level MGRGlobalSmootherType m_levelGlobalSmootherType[numLevels]; ///< Global smoother type for each level HYPRE_Int m_levelGlobalSmootherIters[numLevels]{ -1 }; ///< Number of global smoother iterations for each level -#if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CPU HYPRE_Real m_coarseGridThreshold{ 1.0e-20 }; ///< Coarse grid truncation threshold -#else - HYPRE_Real m_coarseGridThreshold{ 0.0 }; ///< Coarse grid truncation threshold -#endif // TODO: the following options are currently commented out in MGR's code. // Let's consider their use when re-enable in hypre @@ -174,15 +170,18 @@ class MGRStrategyBase * @brief Set up BoomerAMG to perform the solve for the displacement system * @param solver the solver wrapper */ - void setDisplacementAMG( HyprePrecWrapper & solver ) + void setDisplacementAMG( HyprePrecWrapper & solver, + integer const & separateComponents ) { GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGCreate( &solver.ptr ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.8 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 512 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, hypre::getAMGCoarseType( LinearSolverParameters::AMG::CoarseType::gsElimWPivoting ), 3 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType(solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); @@ -192,7 +191,7 @@ class MGRStrategyBase #endif GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( solver.ptr, 3 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( solver.ptr, 1 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( solver.ptr, separateComponents ) ); solver.setup = HYPRE_BoomerAMGSetup; solver.solve = HYPRE_BoomerAMGSolve; @@ -211,9 +210,11 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 16 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 1024 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, 99, 3 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, toUnderlying( AMGCoarseningType::PMIS ) ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); @@ -263,9 +264,10 @@ class MGRStrategyBase * any other solver) for F-relaxation at any level */ void setMechanicsFSolver( HyprePrecWrapper & precond, - HypreMGRData & mgrData ) + HypreMGRData & mgrData, + integer const & separateComponents ) { - setDisplacementAMG( mgrData.mechSolver ); + setDisplacementAMG( mgrData.mechSolver, separateComponents ); HYPRE_MGRSetFSolver( precond.ptr, mgrData.mechSolver.solve, mgrData.mechSolver.setup, mgrData.mechSolver.ptr ); } diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/HybridSinglePhasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/HybridSinglePhasePoromechanics.hpp index f79ad85fb27..687be08785a 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/HybridSinglePhasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/HybridSinglePhasePoromechanics.hpp @@ -84,10 +84,11 @@ class HybridSinglePhasePoromechanics : public MGRStrategyBase< 2 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { @@ -96,7 +97,7 @@ class HybridSinglePhasePoromechanics : public MGRStrategyBase< 2 > GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetPMaxElmts( precond.ptr, 0 )); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp index faf44e8adaf..6983d364a89 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/Hydrofracture.hpp @@ -70,17 +70,18 @@ class Hydrofracture : public MGRStrategyBase< 1 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp index 2ad38a87132..95726ed5df9 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp @@ -79,7 +79,7 @@ class LagrangianContactMechanics : public MGRStrategyBase< 1 > * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { @@ -87,7 +87,7 @@ class LagrangianContactMechanics : public MGRStrategyBase< 1 > // Configure the BoomerAMG solver used as mgr coarse solver for the displacement reduced system // (note that no separate displacement component approach is used here) - setDisplacementAMG( mgrData.coarseSolver ); + setDisplacementAMG( mgrData.coarseSolver, mgrParams.separateComponents ); } }; diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp index 90367c6cc77..cfeca18d3f1 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanics.hpp @@ -98,17 +98,18 @@ class MultiphasePoromechanics : public MGRStrategyBase< 3 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanicsReservoirFVM.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanicsReservoirFVM.hpp index 69038faa599..18293db1f12 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanicsReservoirFVM.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/MultiphasePoromechanicsReservoirFVM.hpp @@ -118,7 +118,7 @@ class MultiphasePoromechanicsReservoirFVM : public MGRStrategyBase< 4 > /** * @brief Setup the MGR strategy. - * @param mgrParams parameters for the configuration of the MGR recipe + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ @@ -136,7 +136,7 @@ class MultiphasePoromechanicsReservoirFVM : public MGRStrategyBase< 4 > setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanics.hpp index 6a4994b90a7..da9b9b8e2d8 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanics.hpp @@ -70,17 +70,18 @@ class SinglePhasePoromechanics : public MGRStrategyBase< 1 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanicsReservoirFVM.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanicsReservoirFVM.hpp index 26b054f9a2d..88be6a13d9b 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanicsReservoirFVM.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SinglePhasePoromechanicsReservoirFVM.hpp @@ -85,17 +85,18 @@ class SinglePhasePoromechanicsReservoirFVM : public MGRStrategyBase< 2 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure reduced system setPressureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp index 7da6fb6229f..45f6eee752c 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp @@ -78,14 +78,14 @@ class SolidMechanicsEmbeddedFractures : public MGRStrategyBase< 1 > * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); // Configure the BoomerAMG solver used as mgr coarse solver for the displacement reduced system - setDisplacementAMG( mgrData.coarseSolver ); + setDisplacementAMG( mgrData.coarseSolver, mgrParams.separateComponents ); } }; diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalMultiphasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalMultiphasePoromechanics.hpp index 129e2b4abd0..1abf2f77d94 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalMultiphasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalMultiphasePoromechanics.hpp @@ -75,20 +75,20 @@ class ThermalMultiphasePoromechanics : public MGRStrategyBase< 3 > setupLabels(); // Level 0 - m_levelFRelaxType[0] = MGRFRelaxationType::amgVCycle; - m_levelFRelaxIters[0] = 1; - m_levelInterpType[0] = MGRInterpolationType::jacobi; - m_levelRestrictType[0] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; - m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; + m_levelFRelaxType[0] = MGRFRelaxationType::amgVCycle; + m_levelFRelaxIters[0] = 1; + m_levelInterpType[0] = MGRInterpolationType::jacobi; + m_levelRestrictType[0] = MGRRestrictionType::injection; + m_levelCoarseGridMethod[0] = MGRCoarseGridMethod::nonGalerkin; + m_levelGlobalSmootherType[0] = MGRGlobalSmootherType::none; // Level 1 - m_levelFRelaxType[1] = MGRFRelaxationType::jacobi; - m_levelFRelaxIters[1] = 1; - m_levelInterpType[1] = MGRInterpolationType::jacobi; - m_levelRestrictType[1] = MGRRestrictionType::injection; - m_levelCoarseGridMethod[1] = MGRCoarseGridMethod::galerkin; - m_levelGlobalSmootherType[1] = MGRGlobalSmootherType::none; + m_levelFRelaxType[1] = MGRFRelaxationType::jacobi; + m_levelFRelaxIters[1] = 1; + m_levelInterpType[1] = MGRInterpolationType::jacobi; + m_levelRestrictType[1] = MGRRestrictionType::injection; + m_levelCoarseGridMethod[1] = MGRCoarseGridMethod::galerkin; + m_levelGlobalSmootherType[1] = MGRGlobalSmootherType::none; // Level 2 m_levelFRelaxType[2] = MGRFRelaxationType::none; @@ -101,18 +101,18 @@ class ThermalMultiphasePoromechanics : public MGRStrategyBase< 3 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); - // CHECK: the mechanics solver setup was missing: was there a reason? // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure/temperature reduced system setPressureTemperatureAMG( mgrData.coarseSolver ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalSinglePhasePoromechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalSinglePhasePoromechanics.hpp index af5889fc7cd..d14919e57ee 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalSinglePhasePoromechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/ThermalSinglePhasePoromechanics.hpp @@ -72,18 +72,18 @@ class ThermalSinglePhasePoromechanics : public MGRStrategyBase< 1 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ - void setup( LinearSolverParameters::MGR const &, + void setup( LinearSolverParameters::MGR const & mgrParams, HyprePrecWrapper & precond, HypreMGRData & mgrData ) { setReduction( precond, mgrData ); - // CHECK: the mechanics solver setup was missing: was there a reason? // Configure the BoomerAMG solver used as F-relaxation for the first level - setMechanicsFSolver( precond, mgrData ); + setMechanicsFSolver( precond, mgrData, mgrParams.separateComponents ); // Configure the BoomerAMG solver used as mgr coarse solver for the pressure/temperature reduced system setPressureTemperatureAMG( mgrData.coarseSolver ); From 0cae3ff2936ac1d47da07acdd52714c2dee8db00 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:54:57 -0400 Subject: [PATCH 09/32] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5757d8230fa..210783b5542 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "284-535" + "GEOS_TPL_TAG": "286-540" } }, "runArgs": [ From ecfe53c5bdf4d1a39b4006633b4b9426473ad6a8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 19:38:01 -0400 Subject: [PATCH 10/32] Turn on SDC approx. for fracture models --- .../SinglePhasePoromechanicsConformingFractures.cpp | 2 +- .../multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp index b387dea8804..5620f3d2efc 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsConformingFractures.cpp @@ -45,7 +45,7 @@ SinglePhasePoromechanicsConformingFractures< FLOW_SOLVER >::SinglePhasePoromecha { LinearSolverParameters & params = this->m_linearSolverParameters.get(); params.mgr.strategy = LinearSolverParameters::MGR::StrategyType::singlePhasePoromechanicsConformingFractures; - params.mgr.separateComponents = false; + params.mgr.separateComponents = true; params.dofsPerNode = 3; } diff --git a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp index 610211370f5..a85bca7fabd 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/SinglePhasePoromechanicsEmbeddedFractures.cpp @@ -43,7 +43,7 @@ SinglePhasePoromechanicsEmbeddedFractures::SinglePhasePoromechanicsEmbeddedFract { LinearSolverParameters & params = m_linearSolverParameters.get(); params.mgr.strategy = LinearSolverParameters::MGR::StrategyType::singlePhasePoromechanicsEmbeddedFractures; - params.mgr.separateComponents = false; + params.mgr.separateComponents = true; params.dofsPerNode = 3; } From 5c1b4d8625a4bae1bffc21c018d4f0444710fbe9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 19:40:49 -0400 Subject: [PATCH 11/32] Uncrustify --- .../linearAlgebra/interfaces/hypre/HypreMGR.hpp | 6 +++--- .../linearAlgebra/interfaces/hypre/HypreSolver.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index afb21f1256d..7530ddb2072 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -181,7 +181,7 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 512 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, hypre::getAMGCoarseType( LinearSolverParameters::AMG::CoarseType::gsElimWPivoting ), 3 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType(solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, hypre::getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::chebyshev ) ) ); @@ -210,8 +210,8 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 16 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 1024 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, 99, 3 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 512 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, hypre::getAMGCoarseType( LinearSolverParameters::AMG::CoarseType::gsElimWPivoting ), 3 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 19ad75d9baf..bf336d4deba 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -278,11 +278,11 @@ void HypreSolver::solve( HypreVector const & rhs, GEOS_LAI_CHECK_ERROR( HYPRE_IJMatrixGetGlobalInfo( matrix().unwrappedIJ(), &global_num_rows, &global_num_rows, // This is intentional and assuming the matrix is square - &global_num_nonzeros) ); + &global_num_nonzeros ) ); GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {} | Nonzeros: {} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", - m_result.status, global_num_rows, global_num_nonzeros, m_result.numIterations, - m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); + m_result.status, global_num_rows, global_num_nonzeros, m_result.numIterations, + m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); } } From eeeac10c3af07d145688c23ff9aef3cbafd21437 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 19:58:04 -0400 Subject: [PATCH 12/32] Turn on SDC for contact mechanics models --- .../SolidMechanicsAugmentedLagrangianContact.cpp | 12 ++++++------ .../contact/SolidMechanicsEmbeddedFractures.cpp | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/coreComponents/physicsSolvers/contact/SolidMechanicsAugmentedLagrangianContact.cpp b/src/coreComponents/physicsSolvers/contact/SolidMechanicsAugmentedLagrangianContact.cpp index eb9fc81049e..02102cbb6cb 100644 --- a/src/coreComponents/physicsSolvers/contact/SolidMechanicsAugmentedLagrangianContact.cpp +++ b/src/coreComponents/physicsSolvers/contact/SolidMechanicsAugmentedLagrangianContact.cpp @@ -43,13 +43,13 @@ SolidMechanicsAugmentedLagrangianContact::SolidMechanicsAugmentedLagrangianConta m_faceTypeToFiniteElements["Quadrilateral"] = std::make_unique< finiteElement::H1_QuadrilateralFace_Lagrange1_GaussLegendre2 >(); m_faceTypeToFiniteElements["Triangle"] = std::make_unique< finiteElement::H1_TriangleFace_Lagrange1_Gauss1 >(); - // TODO Implement the MGR strategy + LinearSolverParameters & linParams = m_linearSolverParameters.get(); - // Set the default linear solver parameters - //LinearSolverParameters & linParams = m_linearSolverParameters.get(); - //linParams.dofsPerNode = 3; - //linParams.isSymmetric = true; - //linParams.amg.separateComponents = true; + linParams.isSymmetric = true; + linParams.dofsPerNode = 3; + linParams.mgr.separateComponents = true; + // TODO Implement the MGR strategy + //linParams.mgr.strategy = LinearSolverParameters::MGR::StrategyType::solidMechanicsAugumentedLagrangianContact; } SolidMechanicsAugmentedLagrangianContact::~SolidMechanicsAugmentedLagrangianContact() diff --git a/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp b/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp index 90f33a650e0..d2f68b61f72 100644 --- a/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp +++ b/src/coreComponents/physicsSolvers/contact/SolidMechanicsEmbeddedFractures.cpp @@ -68,15 +68,16 @@ void SolidMechanicsEmbeddedFractures::postInputInitialization() LinearSolverParameters & linParams = m_linearSolverParameters.get(); + linParams.dofsPerNode = 3; if( m_useStaticCondensation ) { - linParams.dofsPerNode = 3; linParams.isSymmetric = true; linParams.amg.separateComponents = true; } else { linParams.mgr.strategy = LinearSolverParameters::MGR::StrategyType::solidMechanicsEmbeddedFractures; + linParams.mgr.separateComponents = true; } } From 237610e9219601da7b7a387cfcc06da225ee7d07 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 4 Oct 2024 20:03:41 -0400 Subject: [PATCH 13/32] Documentation fix --- src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp | 2 ++ .../hypre/mgrStrategies/LagrangianContactMechanics.hpp | 1 + .../hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp | 1 + 3 files changed, 4 insertions(+) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 7530ddb2072..1f332d66b6e 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -169,6 +169,7 @@ class MGRStrategyBase /** * @brief Set up BoomerAMG to perform the solve for the displacement system * @param solver the solver wrapper + * @param separateComponents flag controlling the use of the separate displacement component (SDC) approximation */ void setDisplacementAMG( HyprePrecWrapper & solver, integer const & separateComponents ) @@ -259,6 +260,7 @@ class MGRStrategyBase * @brief Set up BoomerAMG to perform the mechanics F-solve for the first F-relaxation * @param precond the preconditioner wrapper * @param mgrData auxiliary MGR data + * @param separateComponents flag controlling the use of the separate displacement component (SDC) approximation * * @note This function should be rethought once MGR allows for customizing boomerAMG (or * any other solver) for F-relaxation at any level diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp index 95726ed5df9..68dd3e5c837 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/LagrangianContactMechanics.hpp @@ -76,6 +76,7 @@ class LagrangianContactMechanics : public MGRStrategyBase< 1 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp index 45f6eee752c..52324cc6ba1 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/mgrStrategies/SolidMechanicsEmbeddedFractures.hpp @@ -75,6 +75,7 @@ class SolidMechanicsEmbeddedFractures : public MGRStrategyBase< 1 > /** * @brief Setup the MGR strategy. + * @param mgrParams MGR configuration parameters * @param precond preconditioner wrapper * @param mgrData auxiliary MGR data */ From 8837015e70e99444bc9de511fa748a35412c8d90 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 09:50:35 -0400 Subject: [PATCH 14/32] Add set_global_locale to allow for comma as thousands separator --- src/coreComponents/common/format/Format.hpp | 9 +++++++++ src/coreComponents/common/initializeEnvironment.cpp | 1 + .../linearAlgebra/interfaces/hypre/HypreSolver.cpp | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/common/format/Format.hpp b/src/coreComponents/common/format/Format.hpp index de5954dcc37..e3e8f8da7a9 100644 --- a/src/coreComponents/common/format/Format.hpp +++ b/src/coreComponents/common/format/Format.hpp @@ -37,6 +37,15 @@ #include #define GEOS_FMT_NS std #endif +#include + +/** + * @brief Inline function to set global locale only once + */ +inline void set_global_locale() +{ + static std::locale locale_set = std::locale::global(std::locale("en_US.UTF-8")); +} #ifdef GEOS_USE_FMT /** diff --git a/src/coreComponents/common/initializeEnvironment.cpp b/src/coreComponents/common/initializeEnvironment.cpp index ee3b826aec6..ccca52a25a5 100644 --- a/src/coreComponents/common/initializeEnvironment.cpp +++ b/src/coreComponents/common/initializeEnvironment.cpp @@ -62,6 +62,7 @@ namespace geos /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void setupLogger() { + set_global_locale(); #ifdef GEOS_USE_MPI logger::InitializeLogger( MPI_COMM_GEOS ); #else diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index bf336d4deba..735fa63ecfc 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -280,7 +280,7 @@ void HypreSolver::solve( HypreVector const & rhs, &global_num_rows, // This is intentional and assuming the matrix is square &global_num_nonzeros ) ); - GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {} | Nonzeros: {} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", + GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {:L} | Nonzeros: {:L} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", m_result.status, global_num_rows, global_num_nonzeros, m_result.numIterations, m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); } From fa9de9015f807461b394ea7af79357351d99f59f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 09:56:46 -0400 Subject: [PATCH 15/32] Uncrustify --- src/coreComponents/common/format/Format.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/common/format/Format.hpp b/src/coreComponents/common/format/Format.hpp index e3e8f8da7a9..ad2c9a67b12 100644 --- a/src/coreComponents/common/format/Format.hpp +++ b/src/coreComponents/common/format/Format.hpp @@ -44,7 +44,7 @@ */ inline void set_global_locale() { - static std::locale locale_set = std::locale::global(std::locale("en_US.UTF-8")); + static std::locale locale_set = std::locale::global( std::locale( "en_US.UTF-8" )); } #ifdef GEOS_USE_FMT From 2aca888722c2b663ae3a91ac71eb06a655077d8a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 10:01:15 -0400 Subject: [PATCH 16/32] Update style --- src/coreComponents/common/format/Format.hpp | 4 ++-- src/coreComponents/common/initializeEnvironment.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/common/format/Format.hpp b/src/coreComponents/common/format/Format.hpp index ad2c9a67b12..1d1aed99d28 100644 --- a/src/coreComponents/common/format/Format.hpp +++ b/src/coreComponents/common/format/Format.hpp @@ -40,9 +40,9 @@ #include /** - * @brief Inline function to set global locale only once + * @brief Sets the global locale to "en_US.UTF-8" once. */ -inline void set_global_locale() +inline void setGlobalLocale() { static std::locale locale_set = std::locale::global( std::locale( "en_US.UTF-8" )); } diff --git a/src/coreComponents/common/initializeEnvironment.cpp b/src/coreComponents/common/initializeEnvironment.cpp index ccca52a25a5..bc9e8bcae02 100644 --- a/src/coreComponents/common/initializeEnvironment.cpp +++ b/src/coreComponents/common/initializeEnvironment.cpp @@ -62,7 +62,7 @@ namespace geos /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void setupLogger() { - set_global_locale(); + setGlobalLocale(); #ifdef GEOS_USE_MPI logger::InitializeLogger( MPI_COMM_GEOS ); #else From ceb524c7c15022721d907e1e336a613bd8e33b3a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 15:23:05 -0400 Subject: [PATCH 17/32] Remove locale changes --- src/coreComponents/common/format/Format.hpp | 9 --------- src/coreComponents/common/initializeEnvironment.cpp | 1 - 2 files changed, 10 deletions(-) diff --git a/src/coreComponents/common/format/Format.hpp b/src/coreComponents/common/format/Format.hpp index 1d1aed99d28..de5954dcc37 100644 --- a/src/coreComponents/common/format/Format.hpp +++ b/src/coreComponents/common/format/Format.hpp @@ -37,15 +37,6 @@ #include #define GEOS_FMT_NS std #endif -#include - -/** - * @brief Sets the global locale to "en_US.UTF-8" once. - */ -inline void setGlobalLocale() -{ - static std::locale locale_set = std::locale::global( std::locale( "en_US.UTF-8" )); -} #ifdef GEOS_USE_FMT /** diff --git a/src/coreComponents/common/initializeEnvironment.cpp b/src/coreComponents/common/initializeEnvironment.cpp index bc9e8bcae02..ee3b826aec6 100644 --- a/src/coreComponents/common/initializeEnvironment.cpp +++ b/src/coreComponents/common/initializeEnvironment.cpp @@ -62,7 +62,6 @@ namespace geos /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void setupLogger() { - setGlobalLocale(); #ifdef GEOS_USE_MPI logger::InitializeLogger( MPI_COMM_GEOS ); #else From 032672a82d56dbca11938d9eb8026d0e7fc63a38 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 15:59:42 -0400 Subject: [PATCH 18/32] Move addCommaSeparators to StringUtilities and use it in HYPRESolver.cpp --- .../common/format/StringUtilities.cpp | 23 +++++++ .../common/format/StringUtilities.hpp | 9 +++ .../interfaces/hypre/HypreSolver.cpp | 5 +- src/coreComponents/mesh/DomainPartition.cpp | 64 +++++++------------ 4 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/coreComponents/common/format/StringUtilities.cpp b/src/coreComponents/common/format/StringUtilities.cpp index e9bffe8c0f8..195b98a360a 100644 --- a/src/coreComponents/common/format/StringUtilities.cpp +++ b/src/coreComponents/common/format/StringUtilities.cpp @@ -73,6 +73,29 @@ string removeStringAndFollowingContent( string const & str, return newStr; } +// Add comma separators for thousands +template +string addCommaSeparators(T const & num) +{ + static_assert(std::is_integral::value, "addCommaSeparators only supports integral types"); + + string const numStr = std::to_string(num); + string result; + + for (std::size_t i = 0; i < numStr.size(); ++i) + { + result += numStr[i]; + if ((numStr.size() - i - 1) % 3 == 0 && i != numStr.size() - 1) + { + result += ","; + } + } + return result; +} + +template string addCommaSeparators(localIndex const & num); +template string addCommaSeparators(globalIndex const & num); + // put definition here so we can control the allowable values of T and // modication of this function triggers a whole code recompile...which // should be avoided. diff --git a/src/coreComponents/common/format/StringUtilities.hpp b/src/coreComponents/common/format/StringUtilities.hpp index d8a2ace618a..40ef6ca4f35 100644 --- a/src/coreComponents/common/format/StringUtilities.hpp +++ b/src/coreComponents/common/format/StringUtilities.hpp @@ -184,6 +184,15 @@ string_view trimSpaces( string_view str ); string removeStringAndFollowingContent( string const & str, string const & strToRemove ); +/** + * @brief Add comma separators to an integral number for readability. + * @tparam T the integral type of the number to format. + * @param[in] num the integral number to format. + * @return a string representation of the number with comma separators. + */ +template +string addCommaSeparators(T const & num); + /** * @brief Take a string, and return a array1d with the cast values * @tparam T the type to which the string will be cast diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 735fa63ecfc..79a165ec4ef 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -280,8 +280,9 @@ void HypreSolver::solve( HypreVector const & rhs, &global_num_rows, // This is intentional and assuming the matrix is square &global_num_nonzeros ) ); - GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {:L} | Nonzeros: {:L} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", - m_result.status, global_num_rows, global_num_nonzeros, m_result.numIterations, + GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {} | Nonzeros: {} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", + m_result.status, stringutilities::addCommaSeparators(global_num_rows), + stringutilities::addCommaSeparators(global_num_nonzeros), m_result.numIterations, m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); } } diff --git a/src/coreComponents/mesh/DomainPartition.cpp b/src/coreComponents/mesh/DomainPartition.cpp index 0dd8a8a3623..9604c30932c 100644 --- a/src/coreComponents/mesh/DomainPartition.cpp +++ b/src/coreComponents/mesh/DomainPartition.cpp @@ -328,24 +328,8 @@ void DomainPartition::outputPartitionInformation() const return std::make_pair( objectManager.getNumberOfLocalIndices(), objectManager.getNumberOfGhosts() ); }; - auto addCommaSeparators = []( localIndex const num ) - { - std::string const numStr = std::to_string( num ); - std::string result; - for( std::size_t i = 0; i < numStr.size(); ++i ) - { - result += numStr[i]; - if( ( numStr.size() - i - 1 ) % 3 == 0 && i != numStr.size() - 1 ) - { - result += ","; - } - } - return result; - }; - GEOS_LOG_RANK_0( "MPI Partition information:" ); - forMeshBodies( [&]( MeshBody const & meshBody ) { meshBody.getMeshLevels().forSubGroupsIndex< MeshLevel >( [&]( int const level, MeshLevel const & meshLevel ) @@ -427,24 +411,24 @@ void DomainPartition::outputPartitionInformation() const GEOS_LOG_RANK_0( GEOS_FMT( " | min | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |", - addCommaSeparators( minNumLocalNodes ), - addCommaSeparators( minNumGhostNodes ), - addCommaSeparators( minNumLocalEdges ), - addCommaSeparators( minNumGhostEdges ), - addCommaSeparators( minNumLocalFaces ), - addCommaSeparators( minNumGhostFaces ), - addCommaSeparators( minNumLocalElems ), - addCommaSeparators( minNumGhostElems ) ) ); + stringutilities::addCommaSeparators( minNumLocalNodes ), + stringutilities::addCommaSeparators( minNumGhostNodes ), + stringutilities::addCommaSeparators( minNumLocalEdges ), + stringutilities::addCommaSeparators( minNumGhostEdges ), + stringutilities::addCommaSeparators( minNumLocalFaces ), + stringutilities::addCommaSeparators( minNumGhostFaces ), + stringutilities::addCommaSeparators( minNumLocalElems ), + stringutilities::addCommaSeparators( minNumGhostElems ) ) ); GEOS_LOG_RANK_0( GEOS_FMT( " | max | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} |", - addCommaSeparators( maxNumLocalNodes ), - addCommaSeparators( maxNumGhostNodes ), - addCommaSeparators( maxNumLocalEdges ), - addCommaSeparators( maxNumGhostEdges ), - addCommaSeparators( maxNumLocalFaces ), - addCommaSeparators( maxNumGhostFaces ), - addCommaSeparators( maxNumLocalElems ), - addCommaSeparators( maxNumGhostElems ) ) ); + stringutilities::addCommaSeparators( maxNumLocalNodes ), + stringutilities::addCommaSeparators( maxNumGhostNodes ), + stringutilities::addCommaSeparators( maxNumLocalEdges ), + stringutilities::addCommaSeparators( maxNumGhostEdges ), + stringutilities::addCommaSeparators( maxNumLocalFaces ), + stringutilities::addCommaSeparators( maxNumGhostFaces ), + stringutilities::addCommaSeparators( maxNumLocalElems ), + stringutilities::addCommaSeparators( maxNumGhostElems ) ) ); GEOS_LOG_RANK_0( " |------------------------------------------------------------------------------------------------------------------------------------------------|" ); @@ -456,14 +440,14 @@ void DomainPartition::outputPartitionInformation() const { GEOS_LOG( GEOS_FMT( " | {:14} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | {:>13} | ", rank, - addCommaSeparators( numLocalNodes ), - addCommaSeparators( numGhostNodes ), - addCommaSeparators( numLocalEdges ), - addCommaSeparators( numGhostEdges ), - addCommaSeparators( numLocalFaces ), - addCommaSeparators( numGhostFaces ), - addCommaSeparators( numLocalElems ), - addCommaSeparators( numGhostElems ) ) ); + stringutilities::addCommaSeparators( numLocalNodes ), + stringutilities::addCommaSeparators( numGhostNodes ), + stringutilities::addCommaSeparators( numLocalEdges ), + stringutilities::addCommaSeparators( numGhostEdges ), + stringutilities::addCommaSeparators( numLocalFaces ), + stringutilities::addCommaSeparators( numGhostFaces ), + stringutilities::addCommaSeparators( numLocalElems ), + stringutilities::addCommaSeparators( numGhostElems ) ) ); } MpiWrapper::barrier(); } From 8c93309a46040815b50513f0e55a1998b9e48529 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 16:00:35 -0400 Subject: [PATCH 19/32] Update schema files --- src/coreComponents/schema/schema.xsd | 8 +- src/coreComponents/schema/schema.xsd.other | 130 ++++++++++----------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/coreComponents/schema/schema.xsd b/src/coreComponents/schema/schema.xsd index a8430b98131..3c84156acad 100644 --- a/src/coreComponents/schema/schema.xsd +++ b/src/coreComponents/schema/schema.xsd @@ -1780,7 +1780,7 @@ stress - traction is applied to the faces as specified by the inner product of i - + @@ -1851,7 +1851,7 @@ the relative residual norm satisfies: - + @@ -2946,7 +2946,7 @@ Equal to 1 for surface conditions, and to 0 for reservoir conditions--> - + @@ -2969,7 +2969,7 @@ Local- Add jump stabilization on interior of macro elements--> - + diff --git a/src/coreComponents/schema/schema.xsd.other b/src/coreComponents/schema/schema.xsd.other index d59f6b947e5..215191d89f5 100644 --- a/src/coreComponents/schema/schema.xsd.other +++ b/src/coreComponents/schema/schema.xsd.other @@ -471,7 +471,7 @@ - + @@ -553,7 +553,7 @@ - + @@ -590,7 +590,7 @@ - + @@ -641,7 +641,7 @@ - + @@ -682,7 +682,7 @@ - + @@ -715,7 +715,7 @@ - + @@ -726,7 +726,7 @@ - + @@ -739,7 +739,7 @@ - + @@ -752,7 +752,7 @@ - + @@ -768,7 +768,7 @@ - + @@ -802,7 +802,7 @@ - + @@ -865,7 +865,7 @@ - + @@ -896,7 +896,7 @@ - + @@ -909,7 +909,7 @@ - + @@ -922,7 +922,7 @@ - + @@ -935,7 +935,7 @@ - + @@ -948,7 +948,7 @@ - + @@ -963,7 +963,7 @@ - + @@ -974,7 +974,7 @@ - + @@ -987,7 +987,7 @@ - + @@ -998,7 +998,7 @@ - + @@ -1009,7 +1009,7 @@ - + @@ -1022,7 +1022,7 @@ - + @@ -1033,7 +1033,7 @@ - + @@ -1044,7 +1044,7 @@ - + @@ -1057,7 +1057,7 @@ - + @@ -1072,7 +1072,7 @@ - + @@ -1087,7 +1087,7 @@ - + @@ -1100,7 +1100,7 @@ - + @@ -1115,7 +1115,7 @@ - + @@ -1126,7 +1126,7 @@ - + @@ -1139,7 +1139,7 @@ - + @@ -1152,7 +1152,7 @@ - + @@ -1168,7 +1168,7 @@ - + @@ -1183,7 +1183,7 @@ - + @@ -1200,7 +1200,7 @@ - + @@ -1217,7 +1217,7 @@ - + @@ -1232,7 +1232,7 @@ - + @@ -1245,7 +1245,7 @@ - + @@ -1284,7 +1284,7 @@ - + @@ -1313,7 +1313,7 @@ - + @@ -1404,7 +1404,7 @@ - + @@ -2958,7 +2958,7 @@ - + @@ -2986,7 +2986,7 @@ - + @@ -3005,11 +3005,11 @@ - + - + @@ -3019,7 +3019,7 @@ - + @@ -3029,11 +3029,11 @@ - + - + @@ -3043,7 +3043,7 @@ - + @@ -3053,7 +3053,7 @@ - + @@ -3063,7 +3063,7 @@ - + @@ -3087,7 +3087,7 @@ - + @@ -3105,7 +3105,7 @@ - + @@ -3117,7 +3117,7 @@ - + @@ -3129,7 +3129,7 @@ - + @@ -3137,11 +3137,11 @@ - + - + @@ -3164,7 +3164,7 @@ - + @@ -3190,7 +3190,7 @@ - + @@ -3211,7 +3211,7 @@ - + @@ -3241,7 +3241,7 @@ - + @@ -3255,7 +3255,7 @@ - + @@ -3282,7 +3282,7 @@ - + @@ -3319,7 +3319,7 @@ - + From a7be464334e9e01599b1973a6f04f5e97ca0e2b9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 5 Oct 2024 16:03:31 -0400 Subject: [PATCH 20/32] Uncrustify --- .../common/format/StringUtilities.cpp | 16 ++++++++-------- .../common/format/StringUtilities.hpp | 4 ++-- .../interfaces/hypre/HypreSolver.cpp | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/coreComponents/common/format/StringUtilities.cpp b/src/coreComponents/common/format/StringUtilities.cpp index 195b98a360a..09e65221964 100644 --- a/src/coreComponents/common/format/StringUtilities.cpp +++ b/src/coreComponents/common/format/StringUtilities.cpp @@ -74,18 +74,18 @@ string removeStringAndFollowingContent( string const & str, } // Add comma separators for thousands -template -string addCommaSeparators(T const & num) +template< typename T > +string addCommaSeparators( T const & num ) { - static_assert(std::is_integral::value, "addCommaSeparators only supports integral types"); + static_assert( std::is_integral< T >::value, "addCommaSeparators only supports integral types" ); - string const numStr = std::to_string(num); + string const numStr = std::to_string( num ); string result; - for (std::size_t i = 0; i < numStr.size(); ++i) + for( std::size_t i = 0; i < numStr.size(); ++i ) { result += numStr[i]; - if ((numStr.size() - i - 1) % 3 == 0 && i != numStr.size() - 1) + if((numStr.size() - i - 1) % 3 == 0 && i != numStr.size() - 1 ) { result += ","; } @@ -93,8 +93,8 @@ string addCommaSeparators(T const & num) return result; } -template string addCommaSeparators(localIndex const & num); -template string addCommaSeparators(globalIndex const & num); +template string addCommaSeparators( localIndex const & num ); +template string addCommaSeparators( globalIndex const & num ); // put definition here so we can control the allowable values of T and // modication of this function triggers a whole code recompile...which diff --git a/src/coreComponents/common/format/StringUtilities.hpp b/src/coreComponents/common/format/StringUtilities.hpp index 40ef6ca4f35..4fe7300b400 100644 --- a/src/coreComponents/common/format/StringUtilities.hpp +++ b/src/coreComponents/common/format/StringUtilities.hpp @@ -190,8 +190,8 @@ string removeStringAndFollowingContent( string const & str, * @param[in] num the integral number to format. * @return a string representation of the number with comma separators. */ -template -string addCommaSeparators(T const & num); +template< typename T > +string addCommaSeparators( T const & num ); /** * @brief Take a string, and return a array1d with the cast values diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp index 79a165ec4ef..a3f87354ed6 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreSolver.cpp @@ -281,8 +281,8 @@ void HypreSolver::solve( HypreVector const & rhs, &global_num_nonzeros ) ); GEOS_LOG_RANK_0( GEOS_FMT( " Linear Solver | {} | Unknowns: {} | Nonzeros: {} | Iterations: {} | Final Rel Res: {:.4e} | Setup Time: {:.3f} s | Solve Time: {:.3f} s", - m_result.status, stringutilities::addCommaSeparators(global_num_rows), - stringutilities::addCommaSeparators(global_num_nonzeros), m_result.numIterations, + m_result.status, stringutilities::addCommaSeparators( global_num_rows ), + stringutilities::addCommaSeparators( global_num_nonzeros ), m_result.numIterations, m_result.residualReduction, m_result.setupTime, m_result.solveTime ) ); } } From df915435cd53a1f644acc4c0fcb327ba505050d3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 22 Oct 2024 09:10:41 -0700 Subject: [PATCH 21/32] Fix issues with SDC setup in BoomerAMG --- .../linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp | 3 ++- .../solidMechanics/SolidMechanicsLagrangianFEM.cpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp index a9d7dd84014..0d76b175eab 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HyprePreconditioner.cpp @@ -88,7 +88,6 @@ void createAMG( LinearSolverParameters const & params, GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( precond.ptr, 0.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( precond.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( precond.ptr, logLevel ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( precond.ptr, params.dofsPerNode ) ); // Set maximum number of multigrid levels (default 25) GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxLevels( precond.ptr, LvArray::integerConversion< HYPRE_Int >( params.amg.maxLevels ) ) ); @@ -174,7 +173,9 @@ void createAMG( LinearSolverParameters const & params, GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPMaxElmts( precond.ptr, params.amg.interpolationMaxNonZeros ) ); } + // Unknown-based AMG parameters GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( precond.ptr, params.amg.numFunctions ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( precond.ptr, params.amg.separateComponents ) ); if( params.amg.aggressiveNumLevels ) { diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp index 1c1c5cf9785..67b8b9b1fd5 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp @@ -133,6 +133,7 @@ void SolidMechanicsLagrangianFEM::postInputInitialization() LinearSolverParameters & linParams = m_linearSolverParameters.get(); linParams.isSymmetric = true; linParams.dofsPerNode = 3; + linParams.amg.numFunctions = linParams.dofsPerNode; linParams.amg.separateComponents = true; m_surfaceGenerator = this->getParent().getGroupPointer< SolverBase >( m_surfaceGeneratorName ); From bcc463955767fdea79a22a26b711ad02d40d6b57 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:21:23 -0400 Subject: [PATCH 22/32] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 210783b5542..fddd08109e4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "286-540" + "GEOS_TPL_TAG": "286-546" } }, "runArgs": [ From e0bb7a8de5d8a68c8e53b3c1165ed00b5c885fd4 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 22 Oct 2024 15:32:23 -0700 Subject: [PATCH 23/32] Added missing SDC flag for poromechanics with wells --- .../CompositionalMultiphaseReservoirAndWells.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/physicsSolvers/multiphysics/CompositionalMultiphaseReservoirAndWells.cpp b/src/coreComponents/physicsSolvers/multiphysics/CompositionalMultiphaseReservoirAndWells.cpp index 5853f513231..4f6bb67e34a 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/CompositionalMultiphaseReservoirAndWells.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/CompositionalMultiphaseReservoirAndWells.cpp @@ -75,15 +75,20 @@ void CompositionalMultiphaseReservoirAndWells<>:: setMGRStrategy() { + LinearSolverParameters & linearSolverParameters = this->m_linearSolverParameters.get(); + + linearSolverParameters.mgr.separateComponents = true; + linearSolverParameters.dofsPerNode = 3; + if( flowSolver()->getLinearSolverParameters().mgr.strategy == LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseHybridFVM ) { // add Reservoir - m_linearSolverParameters.get().mgr.strategy = LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseReservoirHybridFVM; + linearSolverParameters.mgr.strategy = LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseReservoirHybridFVM; } else { // add Reservoir - m_linearSolverParameters.get().mgr.strategy = LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseReservoirFVM; + linearSolverParameters.mgr.strategy = LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseReservoirFVM; } } @@ -92,6 +97,11 @@ void CompositionalMultiphaseReservoirAndWells< MultiphasePoromechanics<> >:: setMGRStrategy() { + LinearSolverParameters & linearSolverParameters = this->m_linearSolverParameters.get(); + + linearSolverParameters.mgr.separateComponents = true; + linearSolverParameters.dofsPerNode = 3; + // flow solver here is indeed flow solver, not poromechanics solver if( flowSolver()->getLinearSolverParameters().mgr.strategy == LinearSolverParameters::MGR::StrategyType::compositionalMultiphaseHybridFVM ) { @@ -100,7 +110,7 @@ setMGRStrategy() else { // add Reservoir - m_linearSolverParameters.get().mgr.strategy = LinearSolverParameters::MGR::StrategyType::multiphasePoromechanicsReservoirFVM; + linearSolverParameters.mgr.strategy = LinearSolverParameters::MGR::StrategyType::multiphasePoromechanicsReservoirFVM; } } From 54c455c9147d2da2fcc3e108573419f8436356a9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 30 Oct 2024 20:16:27 -0400 Subject: [PATCH 24/32] Reset coarsest AMG level parameters --- .../linearAlgebra/interfaces/hypre/HypreMGR.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 1f332d66b6e..1a5bd375a35 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -180,8 +180,6 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 512 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, hypre::getAMGCoarseType( LinearSolverParameters::AMG::CoarseType::gsElimWPivoting ), 3 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetInterpType( solver.ptr, hypre::getAMGInterpolationType( LinearSolverParameters::AMG::InterpType::modifiedExtendedE )) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); @@ -210,11 +208,10 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 16 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::multipass ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMinCoarseSize( solver.ptr, 512 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, hypre::getAMGCoarseType( LinearSolverParameters::AMG::CoarseType::gsElimWPivoting ), 3 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); + //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); From ac1fb3cce7982ee1b6893cc507f155fa213d7ad7 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:32:27 -0400 Subject: [PATCH 25/32] Update TPL tag --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fddd08109e4..216993fba02 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "GEOS_TPL_TAG": "286-546" + "GEOS_TPL_TAG": "286-547" } }, "runArgs": [ From cde7e90ed4907e136414a9ccff704bbfb14bcdde Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 31 Oct 2024 00:36:41 -0400 Subject: [PATCH 26/32] Uncrustify --- .../interfaces/hypre/HypreInterface.cpp | 13 ++++++++++--- .../linearAlgebra/interfaces/hypre/HypreMGR.hpp | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp index a0cfb4d9d3b..70591e59973 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp @@ -40,9 +40,16 @@ namespace geos void HypreInterface::initialize() { -#ifdef GEOS_USE_OPENMP - GEOS_LOG_RANK_0_IF( omp_get_max_threads()>1, - "OMP_NUM_THREADS > 1 may not be optimal for certain hypre preconditioning options. " ); +#if defined(GEOS_USE_OPENMP) && defined(HYPRE_USING_OPENMP) + GEOS_LOG_RANK_0_IF( omp_get_max_threads() > 1, + "\n" + "********************************************************************\n" + "* *\n" + "* WARNING: OMP_NUM_THREADS > 1 MAY NOT BE OPTIMAL FOR CERTAIN *\n" + "* HYPRE PRECONDITIONING OPTIONS! *\n" + "* *\n" + "********************************************************************\n" + ); #endif HYPRE_Initialize(); diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 1a5bd375a35..3eabfe5d23d 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -211,7 +211,6 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::multipass ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - //GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); From 4a353e65677a999d3f8a836d9733375cb7ecf109 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 1 Nov 2024 13:08:22 -0700 Subject: [PATCH 27/32] Use SDC for hydrofrac MGR strategy --- .../physicsSolvers/multiphysics/HydrofractureSolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp index 699b4ad20bf..5bb57081f29 100644 --- a/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp +++ b/src/coreComponents/physicsSolvers/multiphysics/HydrofractureSolver.cpp @@ -83,7 +83,7 @@ HydrofractureSolver< POROMECHANICS_SOLVER >::HydrofractureSolver( const string & // This may need to be different depending on whether poroelasticity is on or not. m_linearSolverParameters.get().mgr.strategy = LinearSolverParameters::MGR::StrategyType::hydrofracture; - m_linearSolverParameters.get().mgr.separateComponents = false; + m_linearSolverParameters.get().mgr.separateComponents = true; m_linearSolverParameters.get().dofsPerNode = 3; } From 44eb9b5cbcace6198496519567e218689cc77fb2 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 1 Nov 2024 13:09:11 -0700 Subject: [PATCH 28/32] Better robustness and performance for MGR on GPUs --- src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp | 3 ++- .../linearAlgebra/utilities/LinearSolverParameters.hpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp index 3eabfe5d23d..feca89dd6d0 100644 --- a/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp +++ b/src/coreComponents/linearAlgebra/interfaces/hypre/HypreMGR.hpp @@ -207,10 +207,11 @@ class MGRStrategyBase GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) ); - GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 16 ) ); + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 20 ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::multipass ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) ); #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP + GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggInterpType( solver.ptr, hypre::getAMGAggressiveInterpolationType( LinearSolverParameters::AMG::AggInterpType::modifiedExtendedE ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, hypre::getAMGCoarseningType( LinearSolverParameters::AMG::CoarseningType::PMIS ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr, getAMGRelaxationType( LinearSolverParameters::AMG::SmootherType::l1jacobi ) ) ); GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) ); diff --git a/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp b/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp index fd72cb0d5b2..b1942a61d4c 100644 --- a/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp +++ b/src/coreComponents/linearAlgebra/utilities/LinearSolverParameters.hpp @@ -118,7 +118,7 @@ struct LinearSolverParameters real64 relTolerance = 1e-6; ///< Relative convergence tolerance for iterative solvers integer maxIterations = 200; ///< Max iterations before declaring convergence failure #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP - integer maxRestart = 50; ///< Max number of vectors in Krylov basis before restarting (GPUs) + integer maxRestart = 100; ///< Max number of vectors in Krylov basis before restarting (GPUs) #else integer maxRestart = 200; ///< Max number of vectors in Krylov basis before restarting (CPUs) #endif From 6f853d7b4231ca30dc9a7886f6d97c642805815e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:23:22 -0500 Subject: [PATCH 29/32] Rebaseline --- .integrated_tests.yaml | 2 +- BASELINE_NOTES.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.integrated_tests.yaml b/.integrated_tests.yaml index 85dd1523a2b..ca17b554315 100644 --- a/.integrated_tests.yaml +++ b/.integrated_tests.yaml @@ -1,6 +1,6 @@ baselines: bucket: geosx - baseline: integratedTests/baseline_integratedTests-pr3337-8529-a3e5cb6 + baseline: integratedTests/baseline_integratedTests-pr3339-8552-1237e99 allow_fail: all: '' diff --git a/BASELINE_NOTES.md b/BASELINE_NOTES.md index 89f53d9f0c0..4bbcd7609b5 100644 --- a/BASELINE_NOTES.md +++ b/BASELINE_NOTES.md @@ -6,6 +6,10 @@ This file is designed to track changes to the integrated test baselines. Any developer who updates the baseline ID in the .integrated_tests.yaml file is expected to create an entry in this file with the pull request number, date, and their justification for rebaselining. These notes should be in reverse-chronological order, and use the following time format: (YYYY-MM-DD). +PR #3339 (2024-11-08) +==================== +Fix default parameter (amgNumFunctions = 3) for models involving mechanics. + PR #3337 (2024-11-06) ==================== Change density treatment for gravity in multiphase flow solver. From 7f6b6efafa0ebb8d7bc328c9543c08775cfae77f Mon Sep 17 00:00:00 2001 From: Pavel Tomin Date: Thu, 14 Nov 2024 09:36:45 -0600 Subject: [PATCH 30/32] Update .integrated_tests.yaml --- .integrated_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.integrated_tests.yaml b/.integrated_tests.yaml index 94c95c593c7..00502a919e4 100644 --- a/.integrated_tests.yaml +++ b/.integrated_tests.yaml @@ -1,6 +1,6 @@ baselines: bucket: geosx - baseline: integratedTests/baseline_integratedTests-pr3434-8663-1be8ad2 + baseline: integratedTests/baseline_integratedTests-pr3339-8707-7c55c70 allow_fail: all: '' streak: '' From baf55fc0735c7f8904bf315ee7f7252c1136240f Mon Sep 17 00:00:00 2001 From: Pavel Tomin Date: Thu, 14 Nov 2024 09:37:58 -0600 Subject: [PATCH 31/32] Update BASELINE_NOTES.md --- BASELINE_NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/BASELINE_NOTES.md b/BASELINE_NOTES.md index e4c2a6b4606..f4dcac6a76e 100644 --- a/BASELINE_NOTES.md +++ b/BASELINE_NOTES.md @@ -7,6 +7,10 @@ Any developer who updates the baseline ID in the .integrated_tests.yaml file is These notes should be in reverse-chronological order, and use the following time format: (YYYY-MM-DD). +PR #3339 (2024-11-14) +===================== +Hypre improvements, rebaseline is due to field value change (amgNumFunctions). + PR #3434 (2024-11-09) ===================== Bugfix: Fixed output of ArrayOfArray objects to restart files. From 362be8593f793f650f970b7afa91dbbf655c4786 Mon Sep 17 00:00:00 2001 From: Matteo Cusini Date: Thu, 14 Nov 2024 10:22:36 -0800 Subject: [PATCH 32/32] LvArray update. --- src/coreComponents/LvArray | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/LvArray b/src/coreComponents/LvArray index c9d97b4676d..9b1c0049497 160000 --- a/src/coreComponents/LvArray +++ b/src/coreComponents/LvArray @@ -1 +1 @@ -Subproject commit c9d97b4676d47d1da7d2a63a1a5cbf31f7b54965 +Subproject commit 9b1c00494974c73ff38f8590f010f624efe9964c