Linear algebra cleanup (#1363)

* Linear algebra cleanup
* Update integratedTests
* Update TPL tag

.travis.yml

@@ -2,7 +2,7 @@ language: minimal
 
 env:
   global:
-  - GEOSX_TPL_TAG=152-620
+  - GEOSX_TPL_TAG=154-638
   - secure: CGs2uH6efq1Me6xJWRr0BnwtwxoujzlowC4FHXHdWbNOkPsXf7nCgdaW5vthfD3bhnOeEUQSrfxdhTRtyU/NfcKLmKgGBnZOdUG4/JJK4gDSJ2Wp8LZ/mB0QEoODKVxbh+YtoAiHe3y4M9PGCs+wkNDw/3eEU00cK12DZ6gad0RbLjI3xkhEr/ZEZDZkcYg9yHAhl5bmpqoh/6QGnIg8mxIqdAtGDw+6tT0EgUqjeqc5bG5WwsamKzJItHSXD5zx8IJAlgDk4EzEGjZe0m56YnNfb9iwqqUsmL3Cuwgs7ByVDYw78JC5Kv42YqoxA5BxMT2mFsEe37TpYNXlzofU7ma2Duw9DGXWQd4IkTCcBxlyR0I0bfo0TmgO+y7PYG9lIyHPUkENemdozsZcWamqqkqegiEdRhDVYlSRo3mu7iCwTS6ZTALliVyEYjYxYb7oAnR3cNywXjblTCI8oKfgLSY+8WijM9SRl57JruIHLkLMCjmRI+cZBfv5tS2tYQTBPkygGrigrrN77ZiC7/TGyfggSN0+y0oYtOAgqEnBcKcreiibMW7tKcV2Z1RFD9ZvIkSc1EXLUPDP8FX1oyhmqBMqVo8LksrYLDJHQ05+F3YNgl2taSt7uMjQ4e8iZ3/IjFeMnbylDw+cj/RbS520HXsFPbWFm2Pb9pceA9n6GnY=
 
 # The integrated test repository contains large data (using git lfs) and we do not use them here.

src/cmake/GeosxConfig.cmake

@@ -60,7 +60,13 @@ function( make_full_config_file
         set( GEOSX_USE_${DEP} TRUE )
         set( ${DEP} TRUE )
     endforeach()
+
+    # Fix some options to avoid changes in commited config for doxygen
     set( GEOSX_CMAKE_BUILD_TYPE "\"Release\"" )
+    set( GEOSX_LOCALINDEX_TYPE "std::ptrdiff_t" )
+    set( GEOSX_LOCALINDEX_TYPE_FLAG "3" )
+    set( GEOSX_GLOBALINDEX_TYPE "long long int" )
+    set( GEOSX_GLOBALINDEX_TYPE_FLAG "2" )
 
     configure_file( ${CMAKE_SOURCE_DIR}/coreComponents/common/GeosxConfig.hpp.in
                     ${CMAKE_SOURCE_DIR}/docs/doxygen/GeosxConfig.hpp )

src/cmake/GeosxOptions.cmake

@@ -15,7 +15,7 @@ option( BUILD_LOCAL_CHAI "Use the local mirrored CHAI" OFF )
 
 option( ENABLE_RAJA "Enables RAJA" ON )
 option( BUILD_LOCAL_RAJA "Use the local mirrored RAJA" OFF )
-option( RAJA_ENABLE_TBB "" OFF)
+option( RAJA_ENABLE_TBB "" OFF )
 option( RAJA_ENABLE_OPENMP "" OFF )
 option( RAJA_ENABLE_CUDA "" OFF )
 option( RAJA_ENABLE_TESTS "" OFF )
@@ -28,7 +28,7 @@ option( ENABLE_UNCRUSTIFY "" ON )
 
 option( ENABLE_XML_UPDATES "" ON )
 
-option( ENABLE_FORTRAN "Enables Fortran support" OFF)
+option( ENABLE_FORTRAN "Enables Fortran support" OFF )
 
 option( ENABLE_METIS "Enables METIS" ON )
 option( ENABLE_PARMETIS "Enables PARMETIS" ON )

src/cmake/thirdparty/SetupGeosxThirdParty.cmake

@@ -422,7 +422,7 @@ endif()
 ################################
 # HYPRE
 ################################
-if(DEFINED HYPRE_DIR)
+if(DEFINED HYPRE_DIR AND ENABLE_HYPRE)
     message(STATUS "HYPRE_DIR = ${HYPRE_DIR}")
 
     if( ENABLE_HYPRE_CUDA )
@@ -460,7 +460,7 @@ endif()
 ################################
 # TRILINOS
 ################################
-if(DEFINED TRILINOS_DIR)
+if(DEFINED TRILINOS_DIR AND ENABLE_TRILINOS)
     message(STATUS "TRILINOS_DIR = ${TRILINOS_DIR}")
 
     include(${TRILINOS_DIR}/lib/cmake/Trilinos/TrilinosConfig.cmake)
@@ -492,7 +492,7 @@ endif()
 ###############################
 # PETSC
 ###############################
-if(DEFINED PETSC_DIR)
+if(DEFINED PETSC_DIR AND ENABLE_PETSC)
     message(STATUS "PETSC_DIR = ${PETSC_DIR}")
 
     find_and_register(NAME petsc

src/coreComponents/CMakeLists.txt

@@ -95,6 +95,11 @@ else()
                               )
 endif()
 
+# Seems to be required on some CMake versions (e.g. 3.16) to get enforce device linking
+if( ENABLE_HYPRE_CUDA )
+  set_target_properties( geosx_core PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS TRUE )
+endif()
+
 # To install the library with the runtime path used during the building
 set_target_properties( geosx_core PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE )
 

src/coreComponents/codingUtilities/UnitTestUtilities.hpp

@@ -16,8 +16,8 @@
  * @file UnitTestUtilities.hpp
  */
 
-#ifndef GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP
-#define GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP
+#ifndef GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP_
+#define GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP_
 
 #include "gtest/gtest.h"
 
@@ -199,4 +199,4 @@ void compareLocalMatrices( CRSMatrixView< T const, COL_INDEX const > const & mat
 
 } // namespace geosx
 
-#endif //GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP
+#endif //GEOSX_CODINGUTILITIES_UNITTESTUTILITIES_HPP_

src/coreComponents/common/GeosxConfig.hpp.in

@@ -90,7 +90,7 @@
 /// CMake option CMAKE_BUILD_TYPE
 #cmakedefine GEOSX_CMAKE_BUILD_TYPE @GEOSX_CMAKE_BUILD_TYPE@
 
-/// The type that localIndex will be alaised to.
+/// The type that localIndex will be aliased to.
 #define GEOSX_LOCALINDEX_TYPE @GEOSX_LOCALINDEX_TYPE@
 
 /// An integer flag representing the type that localIndex will be aliased to.

src/coreComponents/common/GeosxMacros.hpp

@@ -100,4 +100,10 @@ void i_g_n_o_r_e( ARGS const & ... ) {}
 /// Doxygen can't parse a `decltype( auto )` return type, using this gets around that.
 #define GEOSX_DECLTYPE_AUTO_RETURN decltype( auto )
 
+/// Macro to concatenate two tokens (low level)
+#define GEOSX_CONCAT_IMPL( A, B ) A ## B
+
+/// Macro to concatenate two tokens (user level)
+#define GEOSX_CONCAT( A, B ) GEOSX_CONCAT_IMPL( A, B )
+
 #endif // GEOSX_COMMON_GEOSXMACROS_HPP_

src/coreComponents/common/MpiWrapper.hpp

@@ -178,6 +178,17 @@ struct MpiWrapper
     return size;
   }
 
+  inline static bool commCompare( MPI_Comm const & comm1, MPI_Comm const & comm2 )
+  {
+#ifdef GEOSX_USE_MPI
+    int result;
+    MPI_Comm_compare( comm1, comm2, &result );
+    return result == MPI_IDENT || result == MPI_CONGRUENT;
+#else
+    return comm1 == comm2;
+#endif
+  }
+
   static bool initialized();
 
   static int init( int * argc, char * * * argv );
@@ -524,7 +535,7 @@ struct MpiWrapper
    * @return a pair where first is the prefix sum, second is the full sum
    */
   template< typename U, typename T >
-  static U prefixSum( T const value );
+  static U prefixSum( T const value, MPI_Comm comm = MPI_COMM_GEOSX );
 
   /**
    * @brief Convenience function for the MPI_Reduce function.
@@ -892,13 +903,13 @@ int MpiWrapper::iSend( T const * const buf,
 }
 
 template< typename U, typename T >
-U MpiWrapper::prefixSum( T const value )
+U MpiWrapper::prefixSum( T const value, MPI_Comm comm )
 {
   U localResult;
 
 #ifdef GEOSX_USE_MPI
   U const convertedValue = value;
-  int const error = MPI_Exscan( &convertedValue, &localResult, 1, getMpiType< U >(), MPI_SUM, MPI_COMM_GEOSX );
+  int const error = MPI_Exscan( &convertedValue, &localResult, 1, getMpiType< U >(), MPI_SUM, comm );
   MPI_CHECK_ERROR( error );
 #endif
   if( commRank() == 0 )

src/coreComponents/common/Stopwatch.hpp

@@ -40,6 +40,27 @@ class Stopwatch
     zero();
   }
 
+  /**
+   * @brief Constructor.
+   * @param resultVar reference to the variable that will store the result upon destruction (a common use case).
+   */
+  Stopwatch( real64 & resultVar )
+    : m_result( &resultVar )
+  {
+    zero();
+  }
+
+  /**
+   * @brief Destructor.
+   */
+  ~Stopwatch()
+  {
+    if( m_result )
+    {
+      *m_result = elapsedTime();
+    }
+  }
+
   /**
    * @brief Zero out the timer.
    */
@@ -63,6 +84,9 @@ class Stopwatch
 
   /// Time point of the last timer restart
   std::chrono::steady_clock::time_point m_start;
+
+  /// The variable to store timer elapsed time result upon leaving scope
+  real64 * m_result{};
 };
 
 } // end geosx

src/coreComponents/linearAlgebra/CMakeLists.txt

@@ -1,28 +1,27 @@
-message( "Entering /src/coreComponents/linearAlgebra/CMakeLists.txt")
-
 #
 # Specify all headers
 #
 set( linearAlgebra_headers
-     common.hpp
-     interfaces/BlasLapackFunctions.h
-     interfaces/BlasLapackLA.hpp
+     common/common.hpp
+     interfaces/dense/BlasLapackFunctions.h
+     interfaces/dense/BlasLapackLA.hpp
      interfaces/InterfaceTypes.hpp
-     interfaces/LinearOperator.hpp
+     common/LinearOperator.hpp
      interfaces/MatrixBase.hpp
      interfaces/VectorBase.hpp
-     interfaces/direct/Arnoldi.hpp
-     solvers/BiCGSTABsolver.hpp
+     solvers/BicgstabSolver.hpp
      solvers/BlockPreconditioner.hpp
-     solvers/CGsolver.hpp
-     solvers/GMRESsolver.hpp
+     solvers/CgSolver.hpp
+     solvers/GmresSolver.hpp
      solvers/KrylovSolver.hpp
      solvers/KrylovUtils.hpp
-     solvers/PreconditionerBase.hpp
+     common/LinearSolverBase.hpp
+     common/PreconditionerBase.hpp
      solvers/PreconditionerIdentity.hpp
      solvers/PreconditionerJacobi.hpp
      solvers/PreconditionerBlockJacobi.hpp
      solvers/SeparateComponentPreconditioner.hpp
+     utilities/Arnoldi.hpp
      utilities/BlockOperatorView.hpp
      utilities/BlockOperatorWrapper.hpp
      utilities/BlockOperator.hpp
@@ -32,56 +31,57 @@ set( linearAlgebra_headers
      utilities/LAIHelperFunctions.hpp
      utilities/LinearSolverParameters.hpp
      utilities/LinearSolverResult.hpp
+     utilities/InverseNormalOperator.hpp
+     utilities/NormalOperator.hpp
+     utilities/TransposeOperator.hpp
+     common/traits.hpp
      DofManager.hpp
      DofManagerHelpers.hpp )
 
 #
 # Specify all sources
 #
 set( linearAlgebra_sources
-     interfaces/BlasLapackLA.cpp
-     solvers/BiCGSTABsolver.cpp
+     interfaces/dense/BlasLapackLA.cpp
+     solvers/BicgstabSolver.cpp
      solvers/BlockPreconditioner.cpp
-     solvers/CGsolver.cpp
-     solvers/GMRESsolver.cpp
+     solvers/CgSolver.cpp
+     solvers/GmresSolver.cpp
      solvers/KrylovSolver.cpp
      solvers/SeparateComponentPreconditioner.cpp
-     utilities/LAIHelperFunctions.cpp
      DofManager.cpp )
 
 set( dependencyList mesh blas lapack RAJA)
 
 
 list( APPEND linearAlgebra_headers
       interfaces/direct/SuiteSparse.hpp
-      interfaces/direct/SuperLU_Dist.hpp )
+      interfaces/direct/SuperLUDist.hpp )
 
 list( APPEND linearAlgebra_sources
       interfaces/direct/SuiteSparse.cpp
-      interfaces/direct/SuperLU_Dist.cpp )
+      interfaces/direct/SuperLUDist.cpp )
 
 list( APPEND dependencyList suitesparse superlu_dist )
 
 if( ENABLE_TRILINOS )
 
     list( APPEND linearAlgebra_headers
+          interfaces/trilinos/EpetraExport.hpp
           interfaces/trilinos/EpetraMatrix.hpp
           interfaces/trilinos/EpetraVector.hpp
           interfaces/trilinos/EpetraUtils.hpp
           interfaces/trilinos/TrilinosPreconditioner.hpp
           interfaces/trilinos/TrilinosSolver.hpp
-          interfaces/trilinos/TrilinosInterface.hpp
-          interfaces/trilinos/EpetraSuiteSparse.hpp
-          interfaces/trilinos/EpetraSuperLU_Dist.hpp )
+          interfaces/trilinos/TrilinosInterface.hpp )
 
     list( APPEND linearAlgebra_sources
+          interfaces/trilinos/EpetraExport.cpp
           interfaces/trilinos/EpetraMatrix.cpp
           interfaces/trilinos/EpetraVector.cpp
           interfaces/trilinos/TrilinosPreconditioner.cpp
           interfaces/trilinos/TrilinosSolver.cpp
-          interfaces/trilinos/TrilinosInterface.cpp
-          interfaces/trilinos/EpetraSuiteSparse.cpp
-          interfaces/trilinos/EpetraSuperLU_Dist.cpp )
+          interfaces/trilinos/TrilinosInterface.cpp )
 
     list( APPEND dependencyList trilinos )
 
@@ -90,24 +90,22 @@ endif()
 if( ENABLE_HYPRE )
 
     list( APPEND linearAlgebra_headers
+          interfaces/hypre/HypreExport.hpp
           interfaces/hypre/HypreMGRStrategies.hpp
           interfaces/hypre/HypreMatrix.hpp
           interfaces/hypre/HypreVector.hpp
           interfaces/hypre/HyprePreconditioner.hpp
           interfaces/hypre/HypreSolver.hpp
           interfaces/hypre/HypreInterface.hpp
-          interfaces/hypre/HypreUtils.hpp
-          interfaces/hypre/HypreSuiteSparse.hpp
-          interfaces/hypre/HypreSuperLU_Dist.hpp )
+          interfaces/hypre/HypreUtils.hpp )
 
     list( APPEND linearAlgebra_sources
+          interfaces/hypre/HypreExport.cpp
           interfaces/hypre/HypreMatrix.cpp
           interfaces/hypre/HypreVector.cpp
           interfaces/hypre/HyprePreconditioner.cpp
           interfaces/hypre/HypreSolver.cpp
-          interfaces/hypre/HypreInterface.cpp
-          interfaces/hypre/HypreSuiteSparse.cpp
-          interfaces/hypre/HypreSuperLU_Dist.cpp )
+          interfaces/hypre/HypreInterface.cpp )
 
     list( APPEND dependencyList hypre )
 
@@ -116,23 +114,21 @@ endif()
 if( ENABLE_PETSC )
 
     list( APPEND linearAlgebra_headers
+          interfaces/petsc/PetscExport.hpp
           interfaces/petsc/PetscVector.hpp
           interfaces/petsc/PetscMatrix.hpp
           interfaces/petsc/PetscPreconditioner.hpp
           interfaces/petsc/PetscSolver.hpp
           interfaces/petsc/PetscInterface.hpp
-          interfaces/petsc/PetscUtils.hpp
-          interfaces/petsc/PetscSuiteSparse.hpp
-          interfaces/petsc/PetscSuperLU_Dist.hpp )
+          interfaces/petsc/PetscUtils.hpp )
 
     list( APPEND linearAlgebra_sources
+          interfaces/petsc/PetscExport.cpp
           interfaces/petsc/PetscVector.cpp
           interfaces/petsc/PetscMatrix.cpp
           interfaces/petsc/PetscPreconditioner.cpp
           interfaces/petsc/PetscSolver.cpp
-          interfaces/petsc/PetscInterface.cpp
-          interfaces/petsc/PetscSuiteSparse.cpp
-          interfaces/petsc/PetscSuperLU_Dist.cpp )
+          interfaces/petsc/PetscInterface.cpp )
 
     list( APPEND dependencyList petsc )
 
@@ -168,5 +164,3 @@ target_include_directories( linearAlgebra PUBLIC ${CMAKE_CURRENT_LIST_DIR} )
 add_subdirectory(unitTests)
 
 geosx_add_code_checks( PREFIX linearAlgebra )
-
-message( "Leaving /src/coreComponents/linearAlgebra/CMakeLists.txt")