Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tpetra: Potential fix for #12574 #12620

Merged
merged 6 commits into from
Dec 21, 2023
Merged

Tpetra: Potential fix for #12574 #12620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3c276e5
Tpetra: Adding unit test
csiefer2 Nov 30, 2023
b098617
Tpetra: Adding H2D counters to test
csiefer2 Nov 30, 2023
1f0417d
Tpetra: Removing unused code
csiefer2 Nov 30, 2023
9c6c38c
Tpetra: Fixing for CPUs
csiefer2 Nov 30, 2023
9d4f438
Tpetra: Map mods for UVM
csiefer2 Dec 5, 2023
9d5c982
Merge branch 'develop' into csiefer-map-uvm
csiefer2 Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 4 additions & 13 deletions packages/tpetra/core/src/Tpetra_Details_LocalMap.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,20 +85,11 @@ class LocalMap {
//! The Kokkos memory space.
using memory_space = typename device_type::memory_space;

//! The hash will be CudaSpace, not CudaUVMSpace
#ifdef KOKKOS_ENABLE_CUDA
using no_uvm_memory_space = typename std::conditional<std::is_same<memory_space, Kokkos::CudaUVMSpace>::value,
Kokkos::CudaSpace, memory_space>::type;
using no_uvm_device_type = Kokkos::Device<execution_space, no_uvm_memory_space>;
#else
using no_uvm_device_type = device_type;
#endif

//! Default constructor.
KOKKOS_DEFAULTED_FUNCTION LocalMap() = default;

LocalMap (const ::Tpetra::Details::FixedHashTable<GlobalOrdinal, LocalOrdinal, no_uvm_device_type>& glMap,
const ::Kokkos::View<const GlobalOrdinal*, ::Kokkos::LayoutLeft, no_uvm_device_type>& lgMap,
LocalMap (const ::Tpetra::Details::FixedHashTable<GlobalOrdinal, LocalOrdinal, device_type>& glMap,
const ::Kokkos::View<const GlobalOrdinal*, ::Kokkos::LayoutLeft, device_type>& lgMap,
const GlobalOrdinal indexBase,
const GlobalOrdinal myMinGid,
const GlobalOrdinal myMaxGid,
Expand Down Expand Up @@ -199,7 +190,7 @@ class LocalMap {

private:
//! Table that maps from global index to local index.
::Tpetra::Details::FixedHashTable<GlobalOrdinal, LocalOrdinal, no_uvm_device_type> glMap_;
::Tpetra::Details::FixedHashTable<GlobalOrdinal, LocalOrdinal, device_type> glMap_;
/// \brief Mapping from local indices to global indices.
///
/// If this is empty, then it could be either that the Map is
Expand All @@ -214,7 +205,7 @@ class LocalMap {
/// LayoutRight because LayoutRight is the default on non-CUDA
/// Devices, and we want to make sure we catch assignment or
/// copying from the default to the nondefault layout.
::Kokkos::View<const GlobalOrdinal*, ::Kokkos::LayoutLeft, no_uvm_device_type> lgMap_;
::Kokkos::View<const GlobalOrdinal*, ::Kokkos::LayoutLeft, device_type> lgMap_;

GlobalOrdinal indexBase_ = 0;
GlobalOrdinal myMinGid_ = Tpetra::Details::OrdinalTraits<GlobalOrdinal>::invalid();
Expand Down
13 changes: 2 additions & 11 deletions packages/tpetra/core/src/Tpetra_Map_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,15 +252,6 @@ namespace Tpetra {
//! Legacy typedef that will go away at some point.
using node_type = Node;

//! The hash will be CudaSpace, not CudaUVMSpace
#ifdef KOKKOS_ENABLE_CUDA
using no_uvm_memory_space = typename std::conditional<std::is_same<memory_space, Kokkos::CudaUVMSpace>::value,
Kokkos::CudaSpace, memory_space>::type;
using no_uvm_device_type = Kokkos::Device<execution_space, no_uvm_memory_space>;
#else
using no_uvm_device_type = device_type;
#endif

/// \brief Type of the "local" Map.
///
/// \warning This object's interface is not yet fixed. We provide
Expand Down Expand Up @@ -1239,7 +1230,7 @@ namespace Tpetra {
/// the nondefault layout.
mutable Kokkos::View<const global_ordinal_type*,
Kokkos::LayoutLeft,
no_uvm_device_type> lgMap_;
device_type> lgMap_;

/// \brief Host View of lgMap_.
///
Expand All @@ -1256,7 +1247,7 @@ namespace Tpetra {

//! Type of a mapping from global IDs to local IDs.
typedef ::Tpetra::Details::FixedHashTable<global_ordinal_type,
local_ordinal_type, no_uvm_device_type> global_to_local_table_type;
local_ordinal_type, device_type> global_to_local_table_type;

/// \brief A mapping from global IDs to local IDs.
///
Expand Down
4 changes: 2 additions & 2 deletions packages/tpetra/core/src/Tpetra_Map_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1692,7 +1692,7 @@ namespace Tpetra {
os << *prefix << "Fill lgMap" << endl;
std::cerr << os.str();
}
FillLgMap<LO, GO, no_uvm_device_type> fillIt (lgMap, minMyGID_);
FillLgMap<LO, GO, device_type> fillIt (lgMap, minMyGID_);

if (verbose) {
std::ostringstream os;
Expand Down Expand Up @@ -1779,7 +1779,7 @@ namespace Tpetra {
os << *prefix << "Fill lgMap" << endl;
std::cerr << os.str();
}
FillLgMap<LO, GO, no_uvm_device_type> fillIt (lgMap, minMyGID_);
FillLgMap<LO, GO, device_type> fillIt (lgMap, minMyGID_);

// "Commit" the local-to-global lookup table we filled in above.
lgMap_ = lgMap;
Expand Down
3 changes: 2 additions & 1 deletion packages/tpetra/core/test/CrsMatrix/CrsMatrix_MatvecH2D.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ namespace {

const size_t nsize=3;

/* Create the identity matrix, three rows per proc */
RCP<MAT> A1 = Tpetra::Utils::MatrixGenerator<MAT>::generate_miniFE_matrix(nsize, comm);
if(!A1->isFillComplete()) A1->fillComplete();
auto map = A1->getRowMap();
Expand Down Expand Up @@ -169,7 +170,7 @@ namespace {


if (numRanks ==1 || Node::is_cpu || Tpetra::Details::Behavior::assumeMpiIsGPUAware()) {
// No H2D/D2H if CPU, MPI is GPU aware or if there's no MPI
// No H2D/D2H if MPI is GPU aware, not a CPU, or if there's no MPI
TEST_EQUALITY(DeepCopyCounter::get_count_different_space(),0);
}
else {
Expand Down