Skip to content

Commit

Permalink
Merge 'trilinos/Trilinos:develop' (065e3d8) into 'tcad-charon/Trilino…
Browse files Browse the repository at this point in the history 
…s:develop' (2fd7918).

* trilinos-develop:
  TrilinosCouplings: Fix scaling of CurlCurl in Maxwell example
  Add short reason for the disablement and note the issue for more details
  Geminga CUDA nightly: Enable complex
  Turn off 3 Zoltan tests that fail  due to a bug in spectrummpi
  muelu:  changes needed to handle Kokkos::complex in Cuda builds
  • Loading branch information
Charonops Jenkins Pipeline committed Apr 21, 2021
2 parents 2fd7918 + 065e3d8 commit 9d975ec
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 27 deletions.
4 changes: 3 additions & 1 deletion .../drivers/geminga/ctest_linux_nightly_mpi_release_muelu_kokkos_refactor_cuda_geminga.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ SET(Trilinos_PACKAGES MueLu Xpetra Amesos2)
SET(EXTRA_CONFIGURE_OPTIONS
### ETI ###
"-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON"
"-DTrilinos_ENABLE_COMPLEX:BOOL=OFF"
"-DTrilinos_ENABLE_COMPLEX:BOOL=ON"
"-DTeuchos_ENABLE_COMPLEX:BOOL=ON"
"-DTpetra_INST_COMPLEX_DOUBLE:BOOL=ON"

### MISC ###
"-DTrilinos_ENABLE_DEPENDENCY_UNIT_TESTS:BOOL=OFF"
Expand Down
10 changes: 5 additions & 5 deletions cmake/std/PullRequestLinuxCuda10.1.243TestingSettings.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,11 @@ set (KokkosCore_UnitTest_Cuda_MPI_1_EXTRA_ARGS
"--gtest_filter=-cuda.debug_pin_um_to_host:cuda.debug_serial_execution"
CACHE STRING "Temporary disable for CUDA PR testing")

# Disable a few failing tests for initial release of Cuda 10.1.105 PR build
# set (EpetraExt_inout_test_LL_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
# set (EpetraExt_inout_test_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
# set (Teko_testdriver_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
# set (Zoltan2_fix4785_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
# Disable a few failing tests for initial release of Cuda 10.1.243 PR build
# these fail due to a bug in spectrum-mpi : see issue #8798 for details
set (Zoltan_ch_simple_zoltan_parallel_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (Zoltan_hg_simple_zoltan_parallel_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (Zoltan_mpiMinLoc_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")

# Disable SEACAS tests that grep results out of stderr...
set (SEACASIoss_create_path_fpp_DISABLE ON CACHE BOOL "Temporary disable due to jsrun polluting stderr")
Expand Down
40 changes: 21 additions & 19 deletions packages/muelu/src/Transfers/Smoothed-Aggregation/MueLu_TentativePFactory_kokkos_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,12 @@ namespace MueLu {
typedef SCType SC;

typedef typename DeviceType::execution_space execution_space;
typedef Kokkos::ArithTraits<SC> ATS;
typedef typename ATS::magnitudeType Magnitude;
typedef typename Kokkos::ArithTraits<SC>::val_type impl_SC;
typedef Kokkos::ArithTraits<impl_SC> impl_ATS;
typedef typename impl_ATS::magnitudeType Magnitude;

typedef Kokkos::View<SC**,typename execution_space::scratch_memory_space, Kokkos::MemoryUnmanaged> shared_matrix;
typedef Kokkos::View<SC* ,typename execution_space::scratch_memory_space, Kokkos::MemoryUnmanaged> shared_vector;
typedef Kokkos::View<impl_SC**,typename execution_space::scratch_memory_space, Kokkos::MemoryUnmanaged> shared_matrix;
typedef Kokkos::View<impl_SC* ,typename execution_space::scratch_memory_space, Kokkos::MemoryUnmanaged> shared_vector;

private:

Expand Down Expand Up @@ -142,10 +143,10 @@ namespace MueLu {
// size of aggregate: number of DOFs in aggregate
auto aggSize = aggRows(agg+1) - aggRows(agg);

const SC one = ATS::one();
const SC two = one + one;
const SC zero = ATS::zero();
const auto zeroM = ATS::magnitude(zero);
const impl_SC one = impl_ATS::one();
const impl_SC two = one + one;
const impl_SC zero = impl_ATS::zero();
const auto zeroM = impl_ATS::magnitude(zero);

int m = aggSize;
int n = fineNS.extent(1);
Expand Down Expand Up @@ -186,8 +187,8 @@ namespace MueLu {
// FIXME_KOKKOS: use team
Magnitude s = zeroM, norm, norm_x;
for (int i = k+1; i < m; i++)
s += pow(ATS::magnitude(r(i,k)), 2);
norm = sqrt(pow(ATS::magnitude(r(k,k)), 2) + s);
s += pow(impl_ATS::magnitude(r(i,k)), 2);
norm = sqrt(pow(impl_ATS::magnitude(r(k,k)), 2) + s);

if (norm == zero) {
isSingular = true;
Expand All @@ -196,7 +197,7 @@ namespace MueLu {

r(k,k) -= norm*one;

norm_x = sqrt(pow(ATS::magnitude(r(k,k)), 2) + s);
norm_x = sqrt(pow(impl_ATS::magnitude(r(k,k)), 2) + s);
if (norm_x == zeroM) {
// We have a single diagonal element in the column.
// No reflections required. Just need to restor r(k,k).
Expand All @@ -211,7 +212,7 @@ namespace MueLu {
// Update R(k:m,k+1:n)
for (int j = k+1; j < n; j++) {
// FIXME_KOKKOS: use team in the loops
SC si = zero;
impl_SC si = zero;
for (int i = k; i < m; i++)
si += r(i,k) * r(i,j);
for (int i = k; i < m; i++)
Expand All @@ -221,7 +222,7 @@ namespace MueLu {
// Update Q^T (k:m,k:m)
for (int j = k; j < m; j++) {
// FIXME_KOKKOS: use team in the loops
SC si = zero;
impl_SC si = zero;
for (int i = k; i < m; i++)
si += r(i,k) * qt(i,j);
for (int i = k; i < m; i++)
Expand All @@ -238,7 +239,7 @@ namespace MueLu {
// Q = (Q^T)^T
for (int i = 0; i < m; i++)
for (int j = 0; j < i; j++) {
SC tmp = qt(i,j);
impl_SC tmp = qt(i,j);
qt(i,j) = qt(j,i);
qt(j,i) = tmp;
}
Expand Down Expand Up @@ -345,7 +346,7 @@ namespace MueLu {
size_t rowStart = rowsAux(localRow);
size_t lnnz = 0;
for (int k = 0; k < n; k++) {
const SC qr_jk = fineNS(localRow,k);
const impl_SC qr_jk = fineNS(localRow,k);
// skip zeros
if (qr_jk != zero) {
colsAux(rowStart+lnnz) = offset + k;
Expand Down Expand Up @@ -574,8 +575,9 @@ namespace MueLu {
const size_t NSDim = fineNullspace->getNumVectors();

typedef Kokkos::ArithTraits<SC> ATS;
using impl_ATS = Kokkos::ArithTraits<typename ATS::val_type>;
const SC zero = ATS::zero(), one = ATS::one();
using impl_SC = typename ATS::val_type;
using impl_ATS = Kokkos::ArithTraits<impl_SC>;
const impl_SC zero = impl_ATS::zero(), one = impl_ATS::one();

const LO INVALID = Teuchos::OrdinalTraits<LO>::invalid();

Expand Down Expand Up @@ -802,7 +804,7 @@ namespace MueLu {
// Q = localQR(:,0)/norm
for (decltype(aggSize) k = 0; k < aggSize; k++) {
LO localRow = agg2RowMapLO(aggRows(agg)+k);
SC localVal = fineNSRandom(agg2RowMapLO(aggRows(agg)+k),0) / norm;
impl_SC localVal = fineNSRandom(agg2RowMapLO(aggRows(agg)+k),0) / norm;

rows(localRow+1) = 1;
colsAux(localRow) = agg;
Expand Down Expand Up @@ -838,7 +840,7 @@ namespace MueLu {
// Q = localQR(:,0)/norm
for (decltype(aggSize) k = 0; k < aggSize; k++) {
LO localRow = agg2RowMapLO(aggRows(agg)+k);
SC localVal = fineNSRandom(agg2RowMapLO(aggRows(agg)+k),0);
impl_SC localVal = fineNSRandom(agg2RowMapLO(aggRows(agg)+k),0);

rows(localRow+1) = 1;
colsAux(localRow) = agg;
Expand Down
3 changes: 2 additions & 1 deletion packages/muelu/src/Utils/MueLu_Utilities_kokkos_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,8 @@ namespace MueLu {
DetectDirichletRows(const Xpetra::Matrix<SC,LO,GO,NO>& A,
const typename Teuchos::ScalarTraits<SC>::magnitudeType& tol,
const bool count_twos_as_dirichlet) {
using ATS = Kokkos::ArithTraits<SC>;
using impl_scalar_type = typename Kokkos::ArithTraits<SC>::val_type;
using ATS = Kokkos::ArithTraits<impl_scalar_type>;
using range_type = Kokkos::RangePolicy<LO, typename NO::execution_space>;

auto localMatrix = A.getLocalMatrix();
Expand Down
2 changes: 1 addition & 1 deletion packages/trilinoscouplings/examples/scaling/example_Maxwell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1713,7 +1713,7 @@ int main(int argc, char *argv[]) {

// multiply by weighted measure
IntrepidFSTools::multiplyMeasure<double>(HCurlsTransformedWeighted,
weightedMeasure, HCurlsTransformed);
weightedMeasureMuInv, HCurlsTransformed);

// integrate to compute element stiffness matrix
IntrepidFSTools::integrate<double>(stiffMatrixHCurl,
Expand Down

0 comments on commit 9d975ec

Please sign in to comment.