atomic_ref (#2, p0019r8)

stl/CMakeLists.txt

@@ -389,6 +389,7 @@ set(SOURCES_SATELLITE_2
 )
 
 set(SOURCES_SATELLITE_ATOMIC_WAIT
+ ${CMAKE_CURRENT_LIST_DIR}/src/atomic_ref.cpp
  ${CMAKE_CURRENT_LIST_DIR}/src/atomic_wait.cpp
  ${CMAKE_CURRENT_LIST_DIR}/src/parallel_algorithms.cpp
 )

stl/inc/yvals.h

@@ -208,6 +208,21 @@ _STL_DISABLE_CLANG_WARNINGS
 #define _STL_INTERNAL_STATIC_ASSERT(...)
 #endif // _ENABLE_STL_INTERNAL_CHECK
 
+#ifndef _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+#ifdef _DEBUG
+#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 1
+#else // ^^^ _DEBUG ^^^ // vvv !_DEBUG vvv
+#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 0
+#endif // _DEBUG
+#endif // _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+
+#if _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _STL_VERIFY(cond, mesg)
+#else
+#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _Analysis_assume_(cond)
+#endif
+
+
 #include <use_ansi.h>
 
 #define _WARNING_MESSAGE(NUMBER, MESSAGE) __FILE__ "(" _CRT_STRINGIZE(__LINE__) "): warning " NUMBER ": " MESSAGE

stl/inc/yvals_core.h

@@ -1137,6 +1137,7 @@
 #define __cpp_lib_atomic_flag_test 201907L
 #define __cpp_lib_atomic_float 201711L
 #define __cpp_lib_atomic_lock_free_type_aliases 201907L
+#define __cpp_lib_atomic_ref 201806L
 #define __cpp_lib_atomic_shared_ptr 201711L
 #define __cpp_lib_atomic_wait 201907L
 #define __cpp_lib_bind_front 201907L

stl/src/atomic_ref.cpp

@@ -0,0 +1,100 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// implement atomic_ref spin lock
+
+#include <atomic>
+#include <cstdint>
+#include <intrin.h>
+#include <new>
+#include <synchapi.h>
+#include <xthreads.h>
+
+_EXTERN_C
+_Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept {
+ constexpr size_t _Table_size_power = 8;
+ constexpr size_t _Table_size = 1 << _Table_size_power;
+ constexpr size_t _Table_index_mask = _Table_size - 1;
+
+ struct alignas(std::hardware_destructive_interference_size) _Table_entry {
+ _Smtx_t _Mutex = 0;
+ char _Pad[std::hardware_destructive_interference_size - sizeof(_Smtx_t)] = {};
+ };
+ static _Table_entry _Table[_Table_size];
+
+ auto _Index = reinterpret_cast<const std::uintptr_t>(_Key);
+ _Index ^= _Index >> (_Table_size_power * 2);
+ _Index ^= _Index >> _Table_size_power;
+ return &_Table[_Index & _Table_index_mask]._Mutex;
+}
+
+namespace {
+
+ _NODISCARD unsigned char __std_atomic_compare_exchange_128_fallback(_Inout_bytecount_(16) long long* _Destination,
+ _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow,
+ _Inout_bytecount_(16) long long* _ComparandResult) noexcept {
+ static SRWLOCK _Mtx = SRWLOCK_INIT;
+ AcquireSRWLockExclusive(&_Mtx);
+ if (_Destination[0] == _ComparandResult[0] && _Destination[1] == _ComparandResult[1]) {
+ _ComparandResult[0] = _Destination[0];
+ _ComparandResult[1] = _Destination[1];
+ _Destination[0] = _ExchangeLow;
+ _Destination[1] = _ExchangeHigh;
+ ReleaseSRWLockExclusive(&_Mtx);
+ return true;
+ } else {
+ _ComparandResult[0] = _Destination[0];
+ _ComparandResult[1] = _Destination[1];
+ ReleaseSRWLockExclusive(&_Mtx);
+ return false;
+ }
+ }
+
+} // unnamed namespace
+
+_NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination,
+ _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow,
+ _Inout_bytecount_(16) long long* _ComparandResult) noexcept {
+#if defined(_M_X64)
+ if (__std_atomic_has_cmpxchg16b()) {
+ return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+ } else {
+ return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+ }
+#elif defined(_M_ARM64)
+ return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+#else
+ return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+#endif
+}
+
+_NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept {
+#if defined(_M_X64)
+ enum class _Cmpxchg16_support : char {
+ _Absent = false,
+ _Present = true,
+ _Unknown,
+ };
+ static std::atomic<_Cmpxchg16_support> _Cached_value{_Cmpxchg16_support::_Unknown};
+
+ _Cmpxchg16_support _Value = _Cached_value.load(std::memory_order_relaxed);
+
+ if (_Value == _Cmpxchg16_support::_Unknown) {
+ int regs[4];
+ __cpuid(regs, 1); // assume leaf 1 exists
+ if (regs[2] & (1 << 13)) {
+ _Value = _Cmpxchg16_support::_Present;
+ } else {
+ _Value = _Cmpxchg16_support::_Absent;
+ }
+ _Cached_value.store(_Value, std::memory_order_relaxed);
+ }
+ return reinterpret_cast<bool&>(_Value);
+#elif defined(_M_ARM64)
+ return true;
+#else
+ return false;
+#endif
+}
+
+_END_EXTERN_C

stl/src/msvcp_atomic_wait.src

@@ -6,6 +6,9 @@
 LIBRARY LIBRARYNAME
 
 EXPORTS
+ __std_atomic_compare_exchange_128
+ __std_atomic_has_cmpxchg16b
+ __std_atomic_get_mutex
  __std_atomic_wait_get_deadline
  __std_atomic_wait_get_remaining_timeout
  __std_atomic_notify_all_direct

tests/libcxx/expected_results.txt

@@ -49,6 +49,7 @@ std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_
 std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp FAIL
 
 # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t`
+std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/filesystem.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/istream.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/limits.version.pass.cpp FAIL
@@ -252,9 +253,6 @@ std/utilities/memory/default.allocator/allocator.members/allocate.verify.cpp SKI
 
 
 # *** MISSING STL FEATURES ***
-# C++20 P0019R8 "atomic_ref"
-std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL
-
 # C++20 P0355R7 "<chrono> Calendars And Time Zones"
 std/utilities/time/days.pass.cpp FAIL
 std/utilities/time/months.pass.cpp FAIL

tests/libcxx/skipped_tests.txt

@@ -49,6 +49,7 @@ containers\unord\unord.map\unord.map.modifiers\insert_and_emplace_allocator_requ
 containers\unord\unord.set\insert_and_emplace_allocator_requirements.pass.cpp
 
 # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t`
+language.support\support.limits\support.limits.general\atomic.version.pass.cpp
 language.support\support.limits\support.limits.general\filesystem.version.pass.cpp
 language.support\support.limits\support.limits.general\istream.version.pass.cpp
 language.support\support.limits\support.limits.general\limits.version.pass.cpp
@@ -252,9 +253,6 @@ utilities\memory\default.allocator\allocator.members\allocate.verify.cpp
 
 
 # *** MISSING STL FEATURES ***
-# C++20 P0019R8 "atomic_ref"
-language.support\support.limits\support.limits.general\atomic.version.pass.cpp
-
 # C++20 P0355R7 "<chrono> Calendars And Time Zones"
 utilities\time\days.pass.cpp
 utilities\time\months.pass.cpp

tests/std/test.lst

@@ -166,6 +166,7 @@ tests\GH_001010_filesystem_error_encoding
 tests\GH_001017_discrete_distribution_out_of_range
 tests\LWG2597_complex_branch_cut
 tests\LWG3018_shared_ptr_function
+tests\P0019R8_atomic_ref
 tests\P0024R2_parallel_algorithms_adjacent_difference
 tests\P0024R2_parallel_algorithms_adjacent_find
 tests\P0024R2_parallel_algorithms_all_of

tests/std/tests/P0019R8_atomic_ref/env.lst

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+RUNALL_INCLUDE ..\usual_latest_matrix.lst

tests/std/tests/P0019R8_atomic_ref/test.cpp

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <atomic>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <execution>
+#include <vector>
+
+struct bigint {
+ int value;
+ int more_value[10];
+
+ bigint(int value = 0) : value(value) {}
+
+ operator int() const {
+ return value;
+ }
+};
+
+struct int128 {
+ long long value;
+ long long more_value;
+
+ int128(int value = 0) : value(value), more_value(0) {}
+
+ operator int() const {
+ return static_cast<int>(value);
+ }
+};
+
+
+// code reuse of ../P1135R6_atomic_flag_test/test.cpp
+
+template <bool AddViaCas, typename ValueType>
+void test_ops() {
+ constexpr std::size_t unique = 80; // small to avoid overflow even for char
+ constexpr std::size_t repetitions = 8000;
+ constexpr std::size_t total = unique * repetitions;
+ constexpr std::size_t range = 10;
+
+ struct alignas(std::atomic_ref<ValueType>::required_alignment) Padded {
+ ValueType vals[unique] = {};
+ } padded;
+
+ auto& vals = padded.vals;
+
+ std::vector<std::atomic_ref<ValueType>> refs;
+ refs.reserve(total);
+ for (std::size_t i = 0; i != repetitions; ++i) {
+ for (auto& val : vals) {
+ refs.push_back(std::atomic_ref<ValueType>(val));
+ }
+ }
+
+ using std::execution::par;
+
+ auto load = [](const std::atomic_ref<ValueType>& ref) { return static_cast<int>(ref.load()); };
+ auto xchg0 = [](std::atomic_ref<ValueType>& ref) { return static_cast<int>(ref.exchange(0)); };
+
+ int (*inc)(std::atomic_ref<ValueType> & ref);
+ if constexpr (AddViaCas) {
+ inc = [](std::atomic_ref<ValueType>& ref) {
+ for (;;) {
+ ValueType e = ref.load();
+ ValueType d = static_cast<ValueType>(static_cast<int>(e) + 1);
+ if (ref.compare_exchange_weak(e, d)) {
+ return static_cast<int>(e);
+ }
+ }
+ };
+ } else {
+ inc = [](std::atomic_ref<ValueType>& ref) { return static_cast<int>(ref.fetch_add(1)); };
+ }
+
+ assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0);
+ assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, inc) == 0);
+ assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions);
+ assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, inc) == range);
+ assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions * 2);
+ assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, xchg0) == range * 2);
+ assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0);
+}
+
+#include <intrin.h>
+
+int main() {
+ test_ops<false, char>();
+ test_ops<false, signed char>();
+ test_ops<false, unsigned char>();
+ test_ops<false, short>();
+ test_ops<false, unsigned short>();
+ test_ops<false, int>();
+ test_ops<false, unsigned int>();
+ test_ops<false, long>();
+ test_ops<false, unsigned long>();
+ test_ops<false, long long>();
+ test_ops<false, unsigned long long>();
+ test_ops<false, float>();
+ test_ops<false, double>();
+ test_ops<false, long double>();
+
+ test_ops<true, char>();
+ test_ops<true, signed char>();
+ test_ops<true, unsigned char>();
+ test_ops<true, short>();
+ test_ops<true, unsigned short>();
+ test_ops<true, int>();
+ test_ops<true, unsigned int>();
+ test_ops<true, long>();
+ test_ops<true, unsigned long>();
+ test_ops<true, long long>();
+ test_ops<true, unsigned long long>();
+ test_ops<true, float>();
+ test_ops<true, double>();
+ test_ops<true, long double>();
+ test_ops<true, bigint>();
+ test_ops<true, int128>();
+}