diff --git a/src/batched/dense/KokkosBatched_Scale_Decl.hpp b/src/batched/dense/KokkosBatched_Scale_Decl.hpp index baf301466d..7b07bc06a3 100644 --- a/src/batched/dense/KokkosBatched_Scale_Decl.hpp +++ b/src/batched/dense/KokkosBatched_Scale_Decl.hpp @@ -3,9 +3,6 @@ /// \author Kyungjoo Kim (kyukim@sandia.gov) -#include "KokkosBatched_Util.hpp" -#include "KokkosBatched_Vector.hpp" - namespace KokkosBatched { /// @@ -30,7 +27,10 @@ struct TeamScale { template KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const ScalarType alpha, - const AViewType &A); + const AViewType &A) { + assert(false && "Deprecated: use KokkosBlas::TeamScale"); + return 0; + } }; /// @@ -42,11 +42,13 @@ struct TeamVectorScale { template KOKKOS_INLINE_FUNCTION static int invoke(const MemberType &member, const ScalarType alpha, - const AViewType &A); + const AViewType &A) { + // static_assert(false); + assert(false && "Deprecated: use KokkosBlas::TeamVectorScale"); + return 0; + } }; } // namespace KokkosBatched -#include "KokkosBatched_Scale_Impl.hpp" - #endif diff --git a/src/batched/dense/impl/KokkosBatched_Gemm_TeamVector_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Gemm_TeamVector_Internal.hpp index 8c8e913f01..630fcf6c02 100644 --- a/src/batched/dense/impl/KokkosBatched_Gemm_TeamVector_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Gemm_TeamVector_Internal.hpp @@ -40,7 +40,8 @@ TeamVectorGemmInternal::invoke( if (beta == zero) TeamVectorSetInternal ::invoke(member, m, n, zero, C, cs0, cs1); else if (beta != one) - TeamVectorScaleInternal::invoke(member, m, n, beta, C, cs0, cs1); + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, beta, C, + cs0, cs1); if (alpha != ScalarType(0.0)) { if (m <= 0 || n <= 0 || k <= 0) return 0; @@ -80,7 +81,8 @@ TeamVectorGemmInternal::invoke( if (beta == zero) TeamVectorSetInternal ::invoke(member, m, n, zero, C, cs0, cs1); else if (beta != one) - TeamVectorScaleInternal::invoke(member, m, n, beta, C, cs0, cs1); + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, beta, C, + cs0, cs1); if (alpha != ScalarType(0.0)) { if (m <= 0 || n <= 0 || k <= 0) return 0; diff --git a/src/batched/dense/impl/KokkosBatched_Gemm_Team_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Gemm_Team_Internal.hpp index 73d831586b..5825d0cb60 100644 --- a/src/batched/dense/impl/KokkosBatched_Gemm_Team_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Gemm_Team_Internal.hpp @@ -7,7 +7,7 @@ #include "KokkosKernels_ExecSpaceUtils.hpp" #include "KokkosBatched_Set_Internal.hpp" -#include "KokkosBatched_Scale_Internal.hpp" +#include "KokkosBlas1_team_scal.hpp" #include "KokkosBatched_InnerGemmFixC_Serial_Impl.hpp" @@ -43,7 +43,8 @@ KOKKOS_INLINE_FUNCTION int TeamGemmInternal::invoke( if (beta == zero) TeamSetInternal ::invoke(member, m, n, zero, C, cs0, cs1); else if (beta != one) - TeamScaleInternal::invoke(member, m, n, beta, C, cs0, cs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, beta, C, cs0, + cs1); if (alpha != ScalarType(0.0)) { if (m <= 0 || n <= 0 || k <= 0) return 0; @@ -84,7 +85,8 @@ KOKKOS_INLINE_FUNCTION int TeamGemmInternal::invoke( if (beta == zero) TeamSetInternal ::invoke(member, m, n, zero, C, cs0, cs1); else if (beta != one) - TeamScaleInternal::invoke(member, m, n, beta, C, cs0, cs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, beta, C, cs0, + cs1); if (alpha != ScalarType(0.0)) { if (m <= 0 || n <= 0 || k <= 0) return 0; diff --git a/src/batched/dense/impl/KokkosBatched_Gemv_TeamVector_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Gemv_TeamVector_Internal.hpp index 419698a24e..6536a00eb7 100644 --- a/src/batched/dense/impl/KokkosBatched_Gemv_TeamVector_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Gemv_TeamVector_Internal.hpp @@ -6,7 +6,7 @@ #include "KokkosBatched_Util.hpp" #include "KokkosBatched_Set_Internal.hpp" -#include "KokkosBatched_Scale_Internal.hpp" +#include "KokkosBlas1_team_scal_impl.hpp" #include "KokkosBatched_InnerMultipleDotProduct_Serial_Impl.hpp" @@ -60,7 +60,7 @@ TeamVectorGemvInternal::invoke( if (beta == zero) TeamVectorSetInternal ::invoke(member, m, zero, y, ys0); else if (beta != one) - TeamVectorScaleInternal::invoke(member, m, beta, y, ys0); + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, beta, y, ys0); if (alpha != zero) { if (m <= 0 || n <= 0) return 0; diff --git a/src/batched/dense/impl/KokkosBatched_Gemv_Team_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Gemv_Team_Internal.hpp index efc08144d2..f8746e98b9 100644 --- a/src/batched/dense/impl/KokkosBatched_Gemv_Team_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Gemv_Team_Internal.hpp @@ -49,7 +49,7 @@ KOKKOS_INLINE_FUNCTION int TeamGemvInternal::invoke( if (beta == zero) TeamSetInternal ::invoke(member, m, zero, y, ys0); else if (beta != one) - TeamScaleInternal::invoke(member, m, beta, y, ys0); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, beta, y, ys0); if (alpha != zero) { if (m <= 0 || n <= 0) return 0; @@ -88,7 +88,7 @@ KOKKOS_INLINE_FUNCTION int TeamGemvInternal::invoke( if (beta == zero) TeamSetInternal ::invoke(member, m, zero, y, ys0); else if (beta != one) - TeamScaleInternal::invoke(member, m, beta, y, ys0); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, beta, y, ys0); if (alpha != zero) { if (m <= 0 || n <= 0) return 0; diff --git a/src/batched/dense/impl/KokkosBatched_Scale_Impl.hpp b/src/batched/dense/impl/KokkosBatched_Scale_Impl.hpp deleted file mode 100644 index 4b0ed29bb9..0000000000 --- a/src/batched/dense/impl/KokkosBatched_Scale_Impl.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef __KOKKOSBATCHED_SCALE_IMPL_HPP__ -#define __KOKKOSBATCHED_SCALE_IMPL_HPP__ - -/// \author Kyungjoo Kim (kyukim@sandia.gov) - -#include "KokkosBatched_Util.hpp" -#include "KokkosBatched_Scale_Internal.hpp" - -namespace KokkosBatched { - -/// -/// Team Impl -/// ========= - -template -template -KOKKOS_INLINE_FUNCTION int TeamScale::invoke( - const MemberType &member, const ScalarType alpha, const AViewType &A) { - return TeamScaleInternal::invoke(member, A.extent(0), A.extent(1), alpha, - A.data(), A.stride_0(), A.stride_1()); -} - -/// -/// TeamVector Impl -/// =============== - -template -template -KOKKOS_INLINE_FUNCTION int TeamVectorScale::invoke( - const MemberType &member, const ScalarType alpha, const AViewType &A) { - return TeamVectorScaleInternal::invoke(member, A.extent(0), A.extent(1), - alpha, A.data(), A.stride_0(), - A.stride_1()); -} - -} // namespace KokkosBatched - -#endif diff --git a/src/batched/dense/impl/KokkosBatched_Trsm_TeamVector_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Trsm_TeamVector_Internal.hpp index 8308200f12..87d9a88122 100644 --- a/src/batched/dense/impl/KokkosBatched_Trsm_TeamVector_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Trsm_TeamVector_Internal.hpp @@ -37,7 +37,8 @@ TeamVectorTrsmInternalLeftLower::invoke( TeamVectorSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamVectorScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, alpha, B, + bs0, bs1); if (m <= 0 || n <= 0) return 0; for (int p = 0; p < m; ++p) { @@ -98,7 +99,8 @@ TeamVectorTrsmInternalLeftUpper::invoke( TeamVectorSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamVectorScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, n, alpha, B, + bs0, bs1); if (m <= 0 || n <= 0) return 0; ValueType *KOKKOS_RESTRICT B0 = B; diff --git a/src/batched/dense/impl/KokkosBatched_Trsm_Team_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Trsm_Team_Internal.hpp index 5baac85374..407ed281db 100644 --- a/src/batched/dense/impl/KokkosBatched_Trsm_Team_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Trsm_Team_Internal.hpp @@ -41,7 +41,8 @@ TeamTrsmInternalLeftLower::invoke( TeamSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, + bs1); if (m <= 0 || n <= 0) return 0; for (int p = 0; p < m; ++p) { @@ -92,7 +93,8 @@ TeamTrsmInternalLeftLower::invoke( TeamSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, + bs1); if (m <= 0 || n <= 0) return 0; /// @@ -175,7 +177,8 @@ TeamTrsmInternalLeftUpper::invoke( TeamSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, + bs1); if (m <= 0 || n <= 0) return 0; ValueType *KOKKOS_RESTRICT B0 = B; @@ -231,7 +234,8 @@ TeamTrsmInternalLeftUpper::invoke( TeamSetInternal ::invoke(member, m, n, zero, B, bs0, bs1); else { if (alpha != one) - TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, bs1); + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, n, alpha, B, bs0, + bs1); if (m <= 0 || n <= 0) return 0; InnerTrsmLeftUpperUnitDiag trsm_u(as0, as1, bs0, bs1); diff --git a/src/batched/dense/impl/KokkosBatched_Trsv_TeamVector_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Trsv_TeamVector_Internal.hpp index baca8bad13..5b673b91b9 100644 --- a/src/batched/dense/impl/KokkosBatched_Trsv_TeamVector_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Trsv_TeamVector_Internal.hpp @@ -44,7 +44,9 @@ TeamVectorTrsvInternalLower::invoke( if (alpha == zero) TeamVectorSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamVectorScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, alpha, b, + bs0); if (m <= 0) return 0; for (int p = 0; p < m; ++p) { @@ -106,7 +108,9 @@ TeamVectorTrsvInternalUpper::invoke( if (alpha == zero) TeamVectorSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamVectorScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamVectorScaleInternal::invoke(member, m, alpha, b, + bs0); if (m <= 0) return 0; ValueType *KOKKOS_RESTRICT b0 = b; diff --git a/src/batched/dense/impl/KokkosBatched_Trsv_Team_Internal.hpp b/src/batched/dense/impl/KokkosBatched_Trsv_Team_Internal.hpp index f1f6faed8c..a71f71dd71 100644 --- a/src/batched/dense/impl/KokkosBatched_Trsv_Team_Internal.hpp +++ b/src/batched/dense/impl/KokkosBatched_Trsv_Team_Internal.hpp @@ -46,7 +46,8 @@ KOKKOS_INLINE_FUNCTION int TeamTrsvInternalLower::invoke( if (alpha == zero) TeamSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0); if (m <= 0) return 0; for (int p = 0; p < m; ++p) { @@ -91,7 +92,8 @@ KOKKOS_INLINE_FUNCTION int TeamTrsvInternalLower::invoke( if (alpha == zero) TeamSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0); if (m <= 0) return 0; /// case GPU: team size is large and blocksize (mb,nb) is small @@ -155,7 +157,8 @@ KOKKOS_INLINE_FUNCTION int TeamTrsvInternalUpper::invoke( if (alpha == zero) TeamSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0); if (m <= 0) return 0; ValueType *KOKKOS_RESTRICT b0 = b; @@ -198,7 +201,8 @@ KOKKOS_INLINE_FUNCTION int TeamTrsvInternalUpper::invoke( if (alpha == zero) TeamSetInternal::invoke(member, m, zero, b, bs0); else { - if (alpha != one) TeamScaleInternal::invoke(member, m, alpha, b, bs0); + if (alpha != one) + KokkosBlas::Impl::TeamScaleInternal::invoke(member, m, alpha, b, bs0); if (m <= 0) return 0; InnerTrsmLeftUpperUnitDiag trsm_u(as0, as1, bs0, 0); diff --git a/src/blas/KokkosBlas1_team_scal.hpp b/src/blas/KokkosBlas1_team_scal.hpp index 5fbe9688d1..af6c61f609 100644 --- a/src/blas/KokkosBlas1_team_scal.hpp +++ b/src/blas/KokkosBlas1_team_scal.hpp @@ -45,9 +45,46 @@ #ifndef KOKKOSBLAS1_TEAM_SCAL_HPP_ #define KOKKOSBLAS1_TEAM_SCAL_HPP_ +#include + +// TODO: deprecate/remove ? #include namespace KokkosBlas { + +/// +/// Team Scale +/// + +template +struct TeamScale { + template + KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, + const ScalarType alpha, + const AViewType& A) { + return Impl::TeamScaleInternal::invoke(member, A.extent(0), A.extent(1), + alpha, A.data(), A.stride_0(), + A.stride_1()); + } +}; + +/// +/// TeamVector Scale +/// + +template +struct TeamVectorScale { + template + KOKKOS_INLINE_FUNCTION static int invoke(const MemberType& member, + const ScalarType alpha, + const AViewType& A) { + return Impl::TeamVectorScaleInternal::invoke(member, A.extent(0), + A.extent(1), alpha, A.data(), + A.stride_0(), A.stride_1()); + } +}; + +// TODO: deprecate/remove ? namespace Experimental { template diff --git a/src/batched/dense/impl/KokkosBatched_Scale_Internal.hpp b/src/blas/impl/KokkosBlas1_team_scal_impl.hpp similarity index 59% rename from src/batched/dense/impl/KokkosBatched_Scale_Internal.hpp rename to src/blas/impl/KokkosBlas1_team_scal_impl.hpp index f02d295267..6f4fdf40b0 100644 --- a/src/batched/dense/impl/KokkosBatched_Scale_Internal.hpp +++ b/src/blas/impl/KokkosBlas1_team_scal_impl.hpp @@ -1,12 +1,55 @@ -#ifndef __KOKKOSBATCHED_SCALE_INTERNAL_HPP__ -#define __KOKKOSBATCHED_SCALE_INTERNAL_HPP__ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Siva Rajamanickam (srajama@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ -/// \author Kyungjoo Kim (kyukim@sandia.gov) +#ifndef KOKKOSBLAS1_TEAM_SCAL_IMPL_HPP_ +#define KOKKOSBLAS1_TEAM_SCAL_IMPL_HPP_ -#include "KokkosBatched_Util.hpp" +#include #include "KokkosBlas1_serial_scal_impl.hpp" -namespace KokkosBatched { +namespace KokkosBlas { +namespace Impl { /// /// Team Internal Impl @@ -29,7 +72,6 @@ struct TeamScaleInternal { const ScalarType alpha, /* */ ValueType *KOKKOS_RESTRICT A, const int as0, const int as1) { - using KokkosBlas::Impl::SerialScaleInternal; if (m > n) { Kokkos::parallel_for( Kokkos::TeamThreadRange(member, m), [&](const int &i) { @@ -87,6 +129,7 @@ struct TeamVectorScaleInternal { } }; -} // namespace KokkosBatched +} // namespace Impl +} // namespace KokkosBlas #endif diff --git a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp index 888c168191..131924f418 100644 --- a/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp +++ b/src/sparse/impl/KokkosSparse_spmv_bsrmatrix_impl.hpp @@ -638,7 +638,7 @@ struct BSR_GEMV_Functor { const y_value_type val_one = Kokkos::ArithTraits::one(); ; if (beta != val_one) { - KokkosBatched::TeamVectorScaleInternal::invoke( + KokkosBlas::Impl::TeamVectorScaleInternal::invoke( dev, block_dim, beta, Y_cur.data(), static_cast(Y_cur.stride_0())); } @@ -1275,7 +1275,7 @@ struct BSR_GEMM_Functor { const y_value_type val_one = Kokkos::ArithTraits::one(); if (beta != val_one) { - KokkosBatched::TeamVectorScaleInternal::invoke( + KokkosBlas::Impl::TeamVectorScaleInternal::invoke( dev, block_dim, num_rhs, beta, Y_cur.data(), static_cast(Y_cur.stride_0()), static_cast(Y_cur.stride_1())); diff --git a/unit_test/batched/dense/Test_Batched_TeamMatUtil.hpp b/unit_test/batched/dense/Test_Batched_TeamMatUtil.hpp index 16879444f7..d098edf0fb 100644 --- a/unit_test/batched/dense/Test_Batched_TeamMatUtil.hpp +++ b/unit_test/batched/dense/Test_Batched_TeamMatUtil.hpp @@ -7,8 +7,7 @@ #include "KokkosBatched_Set_Decl.hpp" #include "KokkosBatched_Set_Impl.hpp" -#include "KokkosBatched_Scale_Decl.hpp" -#include "KokkosBatched_Scale_Impl.hpp" +// #include "KokkosBatched_Scale_Decl.hpp" #include "KokkosKernels_TestUtils.hpp" @@ -40,7 +39,7 @@ struct Functor_TestBatchedTeamMatUtil { switch (TestID) { case BatchedSet: TeamSet::invoke(member, _alpha, A); break; case BatchedScale: - TeamScale::invoke(member, _alpha, A); + KokkosBlas::TeamScale::invoke(member, _alpha, A); break; } }