Merge 'trilinos/Trilinos:develop' (d454829) into 'tcad-charon/Trilino…

…s:develop' (1dd3da3). * trilinos-develop: (22 commits) Fix logic error when parsing --kokkos-develop flag Correct spec inheritance for CXX20 build Tpetra: Use Kokkos::Experimental::SYCLDeviceUSMSpace for KokkosSYCLWrapperNode Panzer q2 mesh support (trilinos#11364) Skip PR merge when using --kokkos-develop flag. Add flag to set Kokkos and KokkosKernels to develop Tpetra_Kokkos_Compat_KokkosSYCLWrapperNode->Tpetra_KokkosCompat_KokkosSYCLWrapperNode typo Cherry-pick kokkos/kokkos-kernels#1914 Update GDSW_Proxy_def.hpp Tpetra: Oops Tpetra: Adding docs Tpetra: Adding unit labels Tpetra: Adding GDSW memory diagnostic XML support Kokkos: Disable default oneDPL support Tpetra: Adding GDSWStyle perf tests Tpetra: GDSW Fixes Tpetra: GDSW test mods Tpetra: Modifying test Tpetra: Modifying test Tpetra: More mods to the test ...
tcad-charon · Aug 16, 2023 · 1da0f3a · 1da0f3a
2 parents 1dd3da3 + d454829
commit 1da0f3a
Show file tree

Hide file tree

Showing 77 changed files with 4,319 additions and 601 deletions.
diff --git a/packages/framework/ini-files/config-specs.ini b/packages/framework/ini-files/config-specs.ini
@@ -2667,8 +2667,7 @@ use rhel7_sems-gnu-8.3.0-openmpi-1.10.7-serial_debug_shared_no-kokkos-arch_no-as
 use PACKAGE-ENABLES|ALL
 
 [rhel7_cxx-20-sems-gnu-8.3.0-openmpi-1.10.1-serial_debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_all]
-# uses sems-v2 modules
-use rhel7_sems-gnu-8.3.0-openmpi-1.10.7-serial_debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables
+use rhel7_sems-gnu-8.3.0-openmpi-1.10.1-serial_debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables
 use PACKAGE-ENABLES|ALL
 opt-set-cmake-var CMAKE_CXX_STANDARD             STRING FORCE : 20
 

diff --git a/packages/framework/pr_tools/LaunchDriver.py b/packages/framework/pr_tools/LaunchDriver.py
@@ -83,6 +83,8 @@ def main(argv):
                       help='The INI file containing supported systems')
   parser.add_argument('--in-container', default=False, action="store_true",
                       help="Build is happening in a container")
+  parser.add_argument("--kokkos-develop", default=False, action="store_true",
+                       help="Build is requiring to pull the current develop of kokkos and kokkos-kernels packages")
   args = parser.parse_args(argv)
 
   if os.getenv("TRILINOS_DIR") == None:
@@ -108,6 +110,9 @@ def main(argv):
   if args.in_container:
      cmd += " --no-bootstrap"
 
+  if args.kokkos_develop:
+     cmd += " --kokkos-develop"
+
   print("LaunchDriver> EXEC: " + cmd, flush=True)
 
   cmd_output = subprocess.run(cmd, shell=True)

diff --git a/packages/framework/pr_tools/PullRequestLinuxDriver.sh b/packages/framework/pr_tools/PullRequestLinuxDriver.sh
@@ -7,6 +7,8 @@ source ${SCRIPTPATH:?}/common.bash
 # Fetch arguments
 on_weaver=$(echo "$@" | grep '\-\-on_weaver' &> /dev/null && echo "1")
 on_ats2=$(echo "$@" | grep '\-\-on_ats2' &> /dev/null && echo "1")
+on_kokkos_develop=$(echo "$@" | grep '\-\-kokkos\-develop' &> /dev/null && echo "1")
+
 bootstrap=$(echo "$@" | grep '\-\-\no\-bootstrap' &> /dev/null && echo "0" || echo "1")
 
 # Configure ccache via environment variables
@@ -89,67 +91,71 @@ sig_script_old=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullReque
 # Get the md5 checksum of the Merge script
 sig_merge_old=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py)
 
-
-print_banner "Merge Source into Target"
-message_std "PRDriver> " "TRILINOS_SOURCE_SHA: ${TRILINOS_SOURCE_SHA:?}"
-
-# Prepare the command for the MERGE operation
-merge_cmd_options=(
-    ${TRILINOS_SOURCE_REPO:?}
-    ${TRILINOS_TARGET_REPO:?}
-    ${TRILINOS_TARGET_BRANCH:?}
-    ${TRILINOS_SOURCE_SHA:?}
-    ${WORKSPACE:?}
-    )
-merge_cmd="${PYTHON_EXE:?} ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py ${merge_cmd_options[@]}"
-
-
-# Call the script to handle merging the incoming branch into
-# the current trilinos/develop branch for testing.
-message_std "PRDriver> " ""
-message_std "PRDriver> " "Execute Merge Command: ${merge_cmd:?}"
-message_std "PRDriver> " ""
-execute_command_checked "${merge_cmd:?}"
-#err=$?
-#if [ $err != 0 ]; then
-#    print_banner "An error occurred during merge"
-#    exit $err
-#fi
-print_banner "Merge completed"
+if [[ ${on_kokkos_develop} == "1" ]]; then
+    message_std "PRDriver> --kokkos-develop is set - setting kokkos and kokkos-kernels packages to current develop"
+    ./$SCRIPTPATH/SetKokkosDevelop.sh
+else
+    print_banner "Merge Source into Target"
+    message_std "PRDriver> " "TRILINOS_SOURCE_SHA: ${TRILINOS_SOURCE_SHA:?}"
+
+    # Prepare the command for the MERGE operation
+    merge_cmd_options=(
+        ${TRILINOS_SOURCE_REPO:?}
+        ${TRILINOS_TARGET_REPO:?}
+        ${TRILINOS_TARGET_BRANCH:?}
+        ${TRILINOS_SOURCE_SHA:?}
+        ${WORKSPACE:?}
+        )
+    merge_cmd="${PYTHON_EXE:?} ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py ${merge_cmd_options[@]}"
+
+
+    # Call the script to handle merging the incoming branch into
+    # the current trilinos/develop branch for testing.
+    message_std "PRDriver> " ""
+    message_std "PRDriver> " "Execute Merge Command: ${merge_cmd:?}"
+    message_std "PRDriver> " ""
+    execute_command_checked "${merge_cmd:?}"
+    #err=$?
+    #if [ $err != 0 ]; then
+    #    print_banner "An error occurred during merge"
+    #    exit $err
+    #fi
+    print_banner "Merge completed"
 
 
-print_banner "Check for PR Driver Script Modifications"
+    print_banner "Check for PR Driver Script Modifications"
 
-# Get the md5 checksum of this script:
-#sig_script_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh)
-sig_script_new=$(get_md5sum ${SCRIPTFILE:?})
-message_std "PRDriver> " ""
-message_std "PRDriver> " "Script File: ${SCRIPTFILE:?}"
-message_std "PRDriver> " "Old md5sum : ${sig_script_old:?}"
-message_std "PRDriver> " "New md5sum : ${sig_script_new:?}"
+    # Get the md5 checksum of this script:
+    #sig_script_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh)
+    sig_script_new=$(get_md5sum ${SCRIPTFILE:?})
+    message_std "PRDriver> " ""
+    message_std "PRDriver> " "Script File: ${SCRIPTFILE:?}"
+    message_std "PRDriver> " "Old md5sum : ${sig_script_old:?}"
+    message_std "PRDriver> " "New md5sum : ${sig_script_new:?}"
+
+    # Get the md5 checksum of the Merge script
+    #sig_merge_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py)
+    export MERGE_SCRIPT=${SCRIPTPATH:?}/PullRequestLinuxDriverMerge.py
+    sig_merge_new=$(get_md5sum ${MERGE_SCRIPT:?})
+    message_std "PRDriver> " ""
+    message_std "PRDriver> " "Script File: ${MERGE_SCRIPT:?}"
+    message_std "PRDriver> " "Old md5sum : ${sig_merge_old:?}"
+    message_std "PRDriver> " "New md5sum : ${sig_merge_new:?}"
+
+    if [ "${sig_script_old:?}" != "${sig_script_new:?}" ] || [ "${sig_merge_old:?}" != "${sig_merge_new:?}"  ]
+    then
+        message_std "PRDriver> " ""
+        message_std "PRDriver> " "Driver or Merge script change detected. Re-launching PR Driver"
+        message_std "PRDriver> " ""
+        ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh
+        exit $?
+    fi
 
-# Get the md5 checksum of the Merge script
-#sig_merge_new=$(get_md5sum ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverMerge.py)
-export MERGE_SCRIPT=${SCRIPTPATH:?}/PullRequestLinuxDriverMerge.py
-sig_merge_new=$(get_md5sum ${MERGE_SCRIPT:?})
-message_std "PRDriver> " ""
-message_std "PRDriver> " "Script File: ${MERGE_SCRIPT:?}"
-message_std "PRDriver> " "Old md5sum : ${sig_merge_old:?}"
-message_std "PRDriver> " "New md5sum : ${sig_merge_new:?}"
-
-if [ "${sig_script_old:?}" != "${sig_script_new:?}" ] || [ "${sig_merge_old:?}" != "${sig_merge_new:?}"  ]
-then
     message_std "PRDriver> " ""
-    message_std "PRDriver> " "Driver or Merge script change detected. Re-launching PR Driver"
+    message_std "PRDriver> " "Driver and Merge scripts unchanged, proceeding to TEST phase"
     message_std "PRDriver> " ""
-    ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriver.sh
-    exit $?
 fi
 
-message_std "PRDriver> " ""
-message_std "PRDriver> " "Driver and Merge scripts unchanged, proceeding to TEST phase"
-message_std "PRDriver> " ""
-
 # determine what MODE we are using
 mode="standard"
 if [[ "${JOB_BASE_NAME:?}" == "Trilinos_pullrequest_gcc_8.3.0_installation_testing" ]]; then

diff --git a/packages/framework/pr_tools/SetKokkosDevelop.sh b/packages/framework/pr_tools/SetKokkosDevelop.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+SCRIPTFILE=$(realpath ${WORKSPACE:?}/Trilinos/packages/framework/pr_tools/SetKokkosDevelop.sh)
+SCRIPTPATH=$(dirname $SCRIPTFILE)
+source ${SCRIPTPATH:?}/common.bash
+PACKAGESPATH=$(realpath ${WORKSPACE:?}/Trilinos/packages)
+
+# Ensures git is loaded properly
+if ! command -v git &> /dev/null; then
+    message_std "SetKokkosDevelop> ERROR: git is not available"
+    exit 1
+fi
+
+cd $PACKAGESPATH
+rm -rf kokkos kokkos-kernels
+git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos.git
+git clone --depth=1 --single-branch --branch=develop --shallow-submodules https://github.com/kokkos/kokkos-kernels.git
+message_std "SetKokkosDevelop> INFO: updated kokkos and kokkos-kernels packages with current develop"
+
+# Returns to previous path from before running this script
+cd -
diff --git a/packages/ifpack2/src/Ifpack2_ETI_LO_GO_SYCL.tmpl b/packages/ifpack2/src/Ifpack2_ETI_LO_GO_SYCL.tmpl
@@ -56,7 +56,7 @@
 TPETRA_ETI_MANGLING_TYPEDEFS()
 
 #define LCLINST(LO, GO) \
-  IFPACK2_@UPPER_CASE_CLASS@_INSTANT(LO, GO, Tpetra_Kokkos_Compat_KokkosSYCLWrapperNode)
+  IFPACK2_@UPPER_CASE_CLASS@_INSTANT(LO, GO, Tpetra_KokkosCompat_KokkosSYCLWrapperNode)
 
 IFPACK2_INSTANTIATE_LG(LCLINST)
 

diff --git a/packages/ifpack2/src/Ifpack2_ETI_SC_LO_GO_SYCL.tmpl b/packages/ifpack2/src/Ifpack2_ETI_SC_LO_GO_SYCL.tmpl
@@ -56,7 +56,7 @@
 IFPACK2_ETI_MANGLING_TYPEDEFS()
 
 #define LCLINST(SC, LO, GO) \
-  IFPACK2_@UPPER_CASE_CLASS@_INSTANT(SC, LO, GO, Tpetra_Kokkos_Compat_KokkosSYCLWrapperNode)
+  IFPACK2_@UPPER_CASE_CLASS@_INSTANT(SC, LO, GO, Tpetra_KokkosCompat_KokkosSYCLWrapperNode)
 
 IFPACK2_INSTANTIATE_SLG(LCLINST)
 

diff --git a/packages/kokkos-kernels/blas/src/KokkosBlas2_gemv.hpp b/packages/kokkos-kernels/blas/src/KokkosBlas2_gemv.hpp
@@ -82,14 +82,6 @@ void gemv(const execution_space& space, const char trans[],
       Kokkos::SpaceAccessibility<execution_space,
                                  typename YViewType::memory_space>::accessible,
       "KokkosBlas::gemv: YViewType must be accessible from execution_space");
-  static_assert(
-      Kokkos::SpaceAccessibility<typename YViewType::memory_space,
-                                 typename AViewType::memory_space>::assignable,
-      "KokkosBlas::gemv: AViewType must be assignable to YViewType");
-  static_assert(
-      Kokkos::SpaceAccessibility<typename YViewType::memory_space,
-                                 typename XViewType::memory_space>::assignable,
-      "KokkosBlas::gemv: XViewType must be assignable to YViewType");
 
   // Check compatibility of dimensions at run time.
   if (trans[0] == 'N' || trans[0] == 'n') {

diff --git a/packages/kokkos-kernels/blas/src/KokkosBlas3_gemm.hpp b/packages/kokkos-kernels/blas/src/KokkosBlas3_gemm.hpp
@@ -142,14 +142,6 @@ void gemm(const execution_space& space, const char transA[],
       Kokkos::SpaceAccessibility<execution_space,
                                  typename CViewType::memory_space>::accessible,
       "KokkosBlas::gemm: CViewType must be accessible from execution_space");
-  static_assert(
-      Kokkos::SpaceAccessibility<typename CViewType::memory_space,
-                                 typename AViewType::memory_space>::assignable,
-      "KokkosBlas::gemm: CViewType must be assignable by AViewType");
-  static_assert(
-      Kokkos::SpaceAccessibility<typename CViewType::memory_space,
-                                 typename BViewType::memory_space>::assignable,
-      "KokkosBlas::gemm: CViewType must be assignable by BViewType");
 
   // Check validity of transpose argument
   bool valid_transA = (transA[0] == 'N') || (transA[0] == 'n') ||

diff --git a/packages/kokkos-kernels/sparse/src/KokkosSparse_spmv.hpp b/packages/kokkos-kernels/sparse/src/KokkosSparse_spmv.hpp
@@ -161,6 +161,12 @@ void spmv(KokkosKernels::Experimental::Controls controls, const char mode[],
     useFallback = useFallback || (mode[0] == Conjugate[0]);
 #endif
   }
+  // cuSPARSE 12 requires that the output (y) vector is 16-byte aligned for all
+  // scalar types
+#if defined(CUSPARSE_VER_MAJOR) && (CUSPARSE_VER_MAJOR == 12)
+  uintptr_t yptr = uintptr_t((void*)y.data());
+  if (yptr % 16 != 0) useFallback = true;
+#endif
 #endif
 
 #ifdef KOKKOSKERNELS_ENABLE_TPL_ROCSPARSE

diff --git a/packages/kokkos/cmake/kokkos_tpls.cmake b/packages/kokkos/cmake/kokkos_tpls.cmake
@@ -45,7 +45,12 @@ ELSE()
   SET(ROCM_DEFAULT OFF)
 ENDIF()
 KOKKOS_TPL_OPTION(ROCM    ${ROCM_DEFAULT})
-KOKKOS_TPL_OPTION(ONEDPL  ${Kokkos_ENABLE_SYCL})
+IF(KOKKOS_ENABLE_SYCL AND NOT KOKKOS_HAS_TRILINOS)
+  SET(ONEDPL_DEFAULT ON)
+ELSE()
+  SET(ONEDPL_DEFAULT OFF)
+ENDIF()
+KOKKOS_TPL_OPTION(ONEDPL  ${ONEDPL_DEFAULT})
 
 IF (WIN32)
   SET(LIBDL_DEFAULT Off)

diff --git a/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in b/packages/panzer/adapters-stk/cmake/PanzerAdaptersSTK_config.hpp.in
@@ -16,5 +16,6 @@
 #cmakedefine PANZER_HAVE_IFPACK2
 #cmakedefine PANZER_HAVE_TEMPUS
 #cmakedefine PANZER_HAVE_STKSEARCH
+#define PANZER_TEMPORARY_TRANSITION_TO_SECOND_ORDER_MESH
 
 #endif
diff --git a/packages/panzer/adapters-stk/example/PoissonExample/CMakeLists.txt b/packages/panzer/adapters-stk/example/PoissonExample/CMakeLists.txt
@@ -104,3 +104,34 @@ FOREACH( ORDER 1 2 3 4 )
     COMM mpi
   )
 ENDFOREACH()
+
+TRIBITS_ADD_ADVANCED_TEST(
+  PoissonExampleCurvilinear-Q2MeshTest
+  TEST_0 EXEC PoissonExample
+    ARGS --mesh-filename=annulus_rad_intervals_3_Q2.exo --basis-order=2 --problem=annulus
+    PASS_REGULAR_EXPRESSION "ALL PASSED"
+    NUM_MPI_PROCS 2
+    OUTPUT_FILE PoissonExampleCurvilinear_3_Q2.out
+  COMM mpi
+  TEST_1 EXEC PoissonExample
+    ARGS --mesh-filename=annulus_rad_intervals_3_Q1.exo --basis-order=2 --problem=annulus
+    PASS_REGULAR_EXPRESSION "ALL PASSED"
+    NUM_MPI_PROCS 2
+    OUTPUT_FILE PoissonExampleCurvilinear_3_Q1.out
+  COMM mpi
+  TEST_2 CMND python
+  ARGS   ${CMAKE_CURRENT_SOURCE_DIR}/compareWithGold_Curvilinear.py
+         ${CMAKE_CURRENT_SOURCE_DIR}/PoissonExampleCurvilinear_3_Q2.gold
+         ${CMAKE_CURRENT_SOURCE_DIR}/PoissonExampleCurvilinear_3_Q1.gold
+         PoissonExampleCurvilinear_3_Q2.out
+         PoissonExampleCurvilinear_3_Q1.out
+    PASS_REGULAR_EXPRESSION "Test Passed"
+  COMM mpi
+  )
+
+TRIBITS_COPY_FILES_TO_BINARY_DIR(Curvilinear_Poisson_mesh_copy
+SOURCE_FILES annulus_rad_intervals_3_Q1.exo.2.0 annulus_rad_intervals_3_Q1.exo.2.1 annulus_rad_intervals_3_Q2.exo.2.0 annulus_rad_intervals_3_Q2.exo.2.1
+SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}
+DEST_DIR ${CMAKE_CURRENT_BINARY_DIR}
+EXEDEPS PoissonExample
+)
diff --git a/packages/panzer/adapters-stk/example/PoissonExample/Example_ClosureModel_Factory_impl.hpp b/packages/panzer/adapters-stk/example/PoissonExample/Example_ClosureModel_Factory_impl.hpp
@@ -84,10 +84,12 @@ buildClosureModels(const std::string& model_id,
   RCP< vector< RCP<Evaluator<panzer::Traits> > > > evaluators = 
     rcp(new vector< RCP<Evaluator<panzer::Traits> > > );
 
+  const bool curvilinear = models.get<bool>("Curvilinear");
+
   if (!models.isSublist(model_id)) {
     models.print(std::cout);
     std::stringstream msg;
-    msg << "Falied to find requested model, \"" << model_id 
+    msg << "Failed to find requested model, \"" << model_id 
 	<< "\", for equation set:\n" << std::endl;
     TEUCHOS_TEST_FOR_EXCEPTION(!models.isSublist(model_id), std::logic_error, msg.str());
   }
@@ -133,15 +135,15 @@ buildClosureModels(const std::string& model_id,
     if (plist.isType<std::string>("Type")) {
       std::string type = plist.get<std::string>("Type");
       if(type=="SIMPLE SOURCE") {
-	RCP< Evaluator<panzer::Traits> > e = 
-	  rcp(new Example::SimpleSource<EvalT,panzer::Traits>(key,*ir));
-	evaluators->push_back(e);
+        RCP< Evaluator<panzer::Traits> > e = 
+          rcp(new Example::SimpleSource<EvalT,panzer::Traits>(key,*ir,curvilinear));
+        evaluators->push_back(e);
 
         found = true;
       }
       else if(type=="TEMPERATURE_EXACT") {
         RCP< Evaluator<panzer::Traits> > e = 
-          rcp(new Example::SimpleSolution<EvalT,panzer::Traits>(key,*ir));
+          rcp(new Example::SimpleSolution<EvalT,panzer::Traits>(key,*ir,curvilinear));
         evaluators->push_back(e);
 
         found = true;

diff --git a/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution.hpp b/packages/panzer/adapters-stk/example/PoissonExample/Example_SimpleSolution.hpp
@@ -70,7 +70,8 @@ class SimpleSolution : public panzer::EvaluatorWithBaseImpl<Traits>,
 
 public:
     SimpleSolution(const std::string & name,
-                       const panzer::IntegrationRule & ir);
+                   const panzer::IntegrationRule & ir,
+                   const bool curvilinear);
 
     void postRegistrationSetup(typename Traits::SetupData d,           
                                PHX::FieldManager<Traits>& fm);        
@@ -85,6 +86,8 @@ class SimpleSolution : public panzer::EvaluatorWithBaseImpl<Traits>,
   PHX::MDField<ScalarT,Cell,Point> solution;
   PHX::MDField<ScalarT,Cell,Point,Dim> solution_grad;
   int ir_degree, ir_index;
+
+  const bool curvilinear;
 };
 
 }