Skip to content

Extend memcpy, memmove and memcmp optimizations #2158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
StephanTLavavej merged 17 commits into from
Sep 25, 2021
Merged

Extend memcpy, memmove and memcmp optimizations #2158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
63b5a60
Optimize algorithms
AdamBucior Aug 25, 2021
d847ba0
formatting
AdamBucior Aug 25, 2021
4f9e4c0
Fix conversions between int and float
AdamBucior Aug 25, 2021
5703798
Reduce GH_000431_lex_compare_memcmp_classify test so that x86 compile…
AdamBucior Aug 25, 2021
9ae8588
Merge remote-tracking branch 'upstream/main' into algorithm-optimizat…
AdamBucior Aug 27, 2021
27fc5c1
remove transition
AdamBucior Aug 27, 2021
01049d3
remove indentation
AdamBucior Aug 27, 2021
d82359b
Update comment
AdamBucior Sep 3, 2021
489afd3
Code review
AdamBucior Sep 10, 2021
64696a0
Merge remote-tracking branch 'upstream/main' into algorithm-optimizat…
AdamBucior Sep 11, 2021
1616b83
Remove _Bitcopy_constructible_and_assignable
AdamBucior Sep 11, 2021
dcf4639
clang-format
AdamBucior Sep 11, 2021
3510a30
Code review
AdamBucior Sep 21, 2021
aa46235
Clang was correctly forbidding narrowing.
StephanTLavavej Sep 21, 2021
85b7685
Call dual-range `equal()` with pred.
StephanTLavavej Sep 21, 2021
5378a50
Work around compiler bugs.
StephanTLavavej Sep 23, 2021
d5cd58c
Reduce compiler memory consumption in GH_000431_equal_memcmp_is_safe.
StephanTLavavej Sep 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions stl/inc/algorithm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1311,7 +1311,7 @@ namespace ranges {
requires indirectly_copyable<_It, _Out>
constexpr copy_n_result<_It, _Out> operator()(_It _First, iter_difference_t<_It> _Count, _Out _Result) const {
auto _UFirst = _Get_unwrapped_n(_STD move(_First), _Count);
if constexpr (_Ptr_copy_cat<decltype(_UFirst), _Out>::_Trivially_copyable) {
if constexpr (_Iter_copy_cat<decltype(_UFirst), _Out>::_Bitcopy_assignable) {
if (!_STD is_constant_evaluated()) {
auto _Final = _UFirst + _Count;
_Result = _Copy_memmove(_STD move(_UFirst), _Final, _STD move(_Result));
Expand Down Expand Up @@ -1458,7 +1458,7 @@ namespace ranges {
requires indirectly_movable<_It, _Out>
constexpr move_result<_It, _Out> _Move_unchecked(_It _First, _Se _Last, _Out _Result) {
// clang-format on
if constexpr (_Ptr_move_cat<_It, _Out>::_Trivially_copyable) {
if constexpr (_Iter_move_cat<_It, _Out>::_Bitcopy_assignable) {
if (!_STD is_constant_evaluated()) {
auto _Final = _RANGES next(_First, _STD move(_Last));
_Result = _Copy_memmove(_STD move(_First), _Final, _STD move(_Result));
Expand Down Expand Up @@ -1514,7 +1514,7 @@ namespace ranges {
template <bidirectional_iterator _It1, bidirectional_iterator _It2>
requires indirectly_movable<_It1, _It2>
constexpr _It2 _Move_backward_common(const _It1 _First, _It1 _Last, _It2 _Result) {
if constexpr (_Ptr_move_cat<_It1, _It2>::_Trivially_copyable) {
if constexpr (_Iter_move_cat<_It1, _It2>::_Bitcopy_assignable) {
if (!_STD is_constant_evaluated()) {
return _Copy_backward_memmove(_First, _Last, _Result);
}
Expand Down Expand Up @@ -9944,10 +9944,9 @@ namespace ranges {
_STL_INTERNAL_STATIC_ASSERT(sentinel_for<_Se2, _It2>);
_STL_INTERNAL_STATIC_ASSERT(indirect_strict_weak_order<_Pr, projected<_It1, _Pj1>, projected<_It2, _Pj2>>);

using _Memcmp_classification_pred =
typename decltype(_Lex_compare_memcmp_classify(_First1, _First2, _Pred))::_Pred;
constexpr bool _Is_sized1 = sized_sentinel_for<_Se1, _It1>;
constexpr bool _Is_sized2 = sized_sentinel_for<_Se2, _It2>;
using _Memcmp_classification_pred = _Lex_compare_memcmp_classify<_It1, _It2, _Pr>;
constexpr bool _Is_sized1 = sized_sentinel_for<_Se1, _It1>;
constexpr bool _Is_sized2 = sized_sentinel_for<_Se2, _It2>;
if constexpr (!is_void_v<_Memcmp_classification_pred> && _Sized_or_unreachable_sentinel_for<_Se1, _It1> //
&& _Sized_or_unreachable_sentinel_for<_Se2, _It2> //
&& same_as<_Pj1, identity> && same_as<_Pj2, identity>) {
Expand Down
32 changes: 16 additions & 16 deletions stl/inc/compare
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -659,12 +659,12 @@ concept _Can_fallback_eq_lt = requires(_Ty1& _Left, _Ty2& _Right) {
{ _Left < _Right } -> _Implicitly_convertible_to<bool>;
};

namespace _Compare_strong_order_fallback {
template <class _Ty1, class _Ty2>
concept _Can_strong_order = requires(_Ty1 & _Left, _Ty2 & _Right) {
_STD strong_order(_Left, _Right);
};
template <class _Ty1, class _Ty2>
concept _Can_strong_order = requires(_Ty1& _Left, _Ty2& _Right) {
_STD strong_order(_Left, _Right);
};

namespace _Compare_strong_order_fallback {
class _Cpo {
private:
enum class _St { _None, _Strong, _Fallback };
Expand Down Expand Up @@ -713,12 +713,12 @@ inline namespace _Cpos {
inline constexpr _Compare_strong_order_fallback::_Cpo compare_strong_order_fallback;
}

namespace _Compare_weak_order_fallback {
template <class _Ty1, class _Ty2>
concept _Can_weak_order = requires(_Ty1& _Left, _Ty2& _Right) {
_STD weak_order(_Left, _Right);
};
template <class _Ty1, class _Ty2>
concept _Can_weak_order = requires(_Ty1& _Left, _Ty2& _Right) {
_STD weak_order(_Left, _Right);
};

namespace _Compare_weak_order_fallback {
class _Cpo {
private:
enum class _St { _None, _Weak, _Fallback };
Expand Down Expand Up @@ -767,14 +767,14 @@ inline namespace _Cpos {
inline constexpr _Compare_weak_order_fallback::_Cpo compare_weak_order_fallback;
}

namespace _Compare_partial_order_fallback {
template <class _Ty1, class _Ty2>
concept _Can_partial_order = requires(_Ty1& _Left, _Ty2& _Right) {
_STD partial_order(_Left, _Right);
};
template <class _Ty1, class _Ty2>
concept _Can_partial_order = requires(_Ty1& _Left, _Ty2& _Right) {
_STD partial_order(_Left, _Right);
};

namespace _Compare_partial_order_fallback {
template <class _Ty1, class _Ty2>
concept _Can_fallback_eq_lt_twice = requires(_Ty1& _Left, _Ty2& _Right) {
concept _Can_fallback_eq_lt_twice = requires(_Ty1 & _Left, _Ty2 & _Right) {
{ _Left == _Right } -> _Implicitly_convertible_to<bool>;
{ _Left < _Right } -> _Implicitly_convertible_to<bool>;
{ _Right < _Left } -> _Implicitly_convertible_to<bool>;
Expand Down
13 changes: 8 additions & 5 deletions stl/inc/memory
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ namespace ranges {

constexpr bool _Is_sized1 = sized_sentinel_for<_Se, _It>;
constexpr bool _Is_sized2 = sized_sentinel_for<_OSe, _Out>;
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_Se, _It> //
if constexpr (_Iter_copy_cat<_It, _Out>::_Bitcopy_constructible
&& _Sized_or_unreachable_sentinel_for<_Se, _It> //
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Is_sized1 && _Is_sized2) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
Expand Down Expand Up @@ -116,7 +117,7 @@ _NoThrowFwdIt uninitialized_copy_n(const _InIt _First, const _Diff _Count_raw, _

auto _UFirst = _Get_unwrapped_n(_First, _Count);
auto _UDest = _Get_unwrapped_n(_Dest, _Count);
if constexpr (_Ptr_copy_cat<decltype(_UFirst), decltype(_UDest)>::_Really_trivial) {
if constexpr (_Iter_copy_cat<decltype(_UFirst), decltype(_UDest)>::_Bitcopy_constructible) {
_UDest = _Copy_memmove(_UFirst, _UFirst + _Count, _UDest);
} else {
_Uninitialized_backout<decltype(_UDest)> _Backout{_UDest};
Expand Down Expand Up @@ -155,7 +156,8 @@ namespace ranges {
auto _IFirst = _Get_unwrapped_n(_STD move(_First1), _Count);
auto _OFirst = _Get_unwrapped(_STD move(_First2));
auto _OLast = _Get_unwrapped(_STD move(_Last2));
if constexpr (_Ptr_copy_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Iter_copy_cat<_It, _Out>::_Bitcopy_constructible
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (sized_sentinel_for<_OSe, _Out>) {
auto _UResult = _Copy_memcpy_common(
_IFirst, _IFirst + _Count, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
Expand Down Expand Up @@ -250,7 +252,7 @@ pair<_InIt, _NoThrowFwdIt> uninitialized_move_n(_InIt _First, const _Diff _Count

auto _UFirst = _Get_unwrapped_n(_First, _Count);
auto _UDest = _Get_unwrapped_n(_Dest, _Count);
if constexpr (_Ptr_move_cat<decltype(_UFirst), decltype(_UDest)>::_Really_trivial) {
if constexpr (_Iter_move_cat<decltype(_UFirst), decltype(_UDest)>::_Bitcopy_constructible) {
_UDest = _Copy_memmove(_UFirst, _UFirst + _Count, _UDest);
_UFirst += _Count;
} else {
Expand Down Expand Up @@ -293,7 +295,8 @@ namespace ranges {
auto _IFirst = _Get_unwrapped_n(_STD move(_First1), _Count);
auto _OFirst = _Get_unwrapped(_STD move(_First2));
const auto _OLast = _Get_unwrapped(_STD move(_Last2));
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Iter_move_cat<_It, _Out>::_Bitcopy_constructible
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (sized_sentinel_for<_OSe, _Out>) {
auto _UResult = _Copy_memcpy_common(
_IFirst, _IFirst + _Count, _OFirst, _RANGES next(_OFirst, _STD move(_OLast)));
Expand Down
11 changes: 6 additions & 5 deletions stl/inc/xmemory
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1473,7 +1473,7 @@ struct _NODISCARD _Uninitialized_backout {
template <class _InIt, class _NoThrowFwdIt>
_CONSTEXPR20 _NoThrowFwdIt _Uninitialized_move_unchecked(_InIt _First, const _InIt _Last, _NoThrowFwdIt _Dest) {
// move [_First, _Last) to raw [_Dest, ...)
if constexpr (_Ptr_move_cat<_InIt, _NoThrowFwdIt>::_Really_trivial) {
if constexpr (_Iter_move_cat<_InIt, _NoThrowFwdIt>::_Bitcopy_constructible) {
#if _HAS_CXX20
if (!_STD is_constant_evaluated())
#endif // _HAS_CXX20
Expand Down Expand Up @@ -1602,7 +1602,8 @@ namespace ranges {
// clang-format on
constexpr bool _Is_sized1 = sized_sentinel_for<_Se, _It>;
constexpr bool _Is_sized2 = sized_sentinel_for<_OSe, _Out>;
if constexpr (_Ptr_move_cat<_It, _Out>::_Really_trivial && _Sized_or_unreachable_sentinel_for<_Se, _It> //
if constexpr (_Iter_move_cat<_It, _Out>::_Bitcopy_constructible
&& _Sized_or_unreachable_sentinel_for<_Se, _It> //
&& _Sized_or_unreachable_sentinel_for<_OSe, _Out>) {
if constexpr (_Is_sized1 && _Is_sized2) {
return _Copy_memcpy_common(_IFirst, _RANGES next(_IFirst, _STD move(_ILast)), _OFirst,
Expand Down Expand Up @@ -1669,7 +1670,7 @@ _CONSTEXPR20 _Alloc_ptr_t<_Alloc> _Uninitialized_copy(
auto _UFirst = _Get_unwrapped(_First);
const auto _ULast = _Get_unwrapped(_Last);

if constexpr (conjunction_v<bool_constant<_Ptr_copy_cat<decltype(_UFirst), _Ptrval>::_Really_trivial>,
if constexpr (conjunction_v<bool_constant<_Iter_copy_cat<decltype(_UFirst), _Ptrval>::_Bitcopy_constructible>,
_Uses_default_construct<_Alloc, _Ptrval, decltype(*_UFirst)>>) {
#if _HAS_CXX20
if (!_STD is_constant_evaluated())
Expand All @@ -1692,7 +1693,7 @@ _CONSTEXPR20 _Alloc_ptr_t<_Alloc> _Uninitialized_copy(
template <class _InIt, class _NoThrowFwdIt>
_CONSTEXPR20 _NoThrowFwdIt _Uninitialized_copy_unchecked(_InIt _First, const _InIt _Last, _NoThrowFwdIt _Dest) {
// copy [_First, _Last) to raw [_Dest, ...)
if constexpr (_Ptr_copy_cat<_InIt, _NoThrowFwdIt>::_Really_trivial) {
if constexpr (_Iter_copy_cat<_InIt, _NoThrowFwdIt>::_Bitcopy_constructible) {
#if _HAS_CXX20
if (!_STD is_constant_evaluated())
#endif // _HAS_CXX20
Expand Down Expand Up @@ -1728,7 +1729,7 @@ _CONSTEXPR20 _Alloc_ptr_t<_Alloc> _Uninitialized_move(
using _Ptrval = typename _Alloc::value_type*;
auto _UFirst = _Get_unwrapped(_First);
const auto _ULast = _Get_unwrapped(_Last);
if constexpr (conjunction_v<bool_constant<_Ptr_move_cat<decltype(_UFirst), _Ptrval>::_Really_trivial>,
if constexpr (conjunction_v<bool_constant<_Iter_move_cat<decltype(_UFirst), _Ptrval>::_Bitcopy_constructible>,
_Uses_default_construct<_Alloc, _Ptrval, decltype(_STD move(*_UFirst))>>) {
#if _HAS_CXX20
if (!_STD is_constant_evaluated())
Expand Down
11 changes: 10 additions & 1 deletion stl/inc/xstring
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,16 @@ struct _Char_traits_lt {
// library-provided char_traits::eq behaves like equal_to<_Elem>
// TRANSITION: This should not be activated for user-defined specializations of char_traits
template <class _Elem>
_INLINE_VAR constexpr bool _Pred_is_consistent_with_memcmp<_Elem, _Elem, _Char_traits_eq<char_traits<_Elem>>> = true;
_INLINE_VAR constexpr bool _Can_memcmp_elements_with_pred<_Elem, _Elem, _Char_traits_eq<char_traits<_Elem>>> =
_Can_memcmp_elements<_Elem, _Elem>;

// library-provided char_traits::lt behaves like less<make_unsigned_t<_Elem>>
// TRANSITION: This should not be activated for user-defined specializations of char_traits
template <class _Elem>
struct _Lex_compare_memcmp_classify_pred<_Elem, _Elem, _Char_traits_lt<char_traits<_Elem>>> {
using _UElem = make_unsigned_t<_Elem>;
using _Pred = conditional_t<_Lex_compare_memcmp_classify_elements<_UElem, _UElem>, less<int>, void>;
};

template <class _Traits>
using _Traits_ch_t = typename _Traits::char_type;
Expand Down
Loading