SSE2 vectorization for bitset::to_string

stl/inc/bitset

@@ -18,6 +18,15 @@ _STL_DISABLE_CLANG_WARNINGS
 #pragma push_macro("new")
 #undef new
 
+#if _USE_STD_VECTOR_ALGORITHMS
+extern "C" {
+__declspec(noalias) void __stdcall __std_bitset_to_string_1(
+ char* _Dest, const void* _Src, size_t _Size_bits, char _Elem0, char _Elem1) noexcept;
+__declspec(noalias) void __stdcall __std_bitset_to_string_2(
+ wchar_t* _Dest, const void* _Src, size_t _Size_bits, wchar_t _Elem0, wchar_t _Elem1) noexcept;
+} // extern "C"
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
 _STD_BEGIN
 _EXPORT_STD template <size_t _Bits>
 class bitset { // store fixed-length sequence of Boolean elements
@@ -348,6 +357,23 @@ public:
  // convert bitset to string
  basic_string<_Elem, _Tr, _Alloc> _Str;
  _Str._Resize_and_overwrite(_Bits, [this, _Elem0, _Elem1](_Elem* _Buf, size_t _Len) {
+#if _USE_STD_VECTOR_ALGORITHMS
+ constexpr size_t _Bitset_vector_threshold = 32;
+ if constexpr (_Bits >= _Bitset_vector_threshold && is_integral_v<_Elem> && sizeof(_Elem) <= 2) {
+ if (!_Is_constant_evaluated()) {
+ if constexpr (sizeof(_Elem) == 1) {
+ __std_bitset_to_string_1(reinterpret_cast<char*>(_Buf), _Array, _Len, static_cast<char>(_Elem0),
+ static_cast<char>(_Elem1));
+ } else {
+ _STL_INTERNAL_STATIC_ASSERT(sizeof(_Elem) == 2);
+ __std_bitset_to_string_2(reinterpret_cast<wchar_t*>(_Buf), _Array, _Len,
+ static_cast<wchar_t>(_Elem0), static_cast<wchar_t>(_Elem1));
+ }
+ return _Len;
+ }
+ }
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
  for (size_t _Pos = 0; _Pos < _Len; ++_Pos) {
  _Buf[_Pos] = _Subscript(_Len - 1 - _Pos) ? _Elem1 : _Elem0;
  }

stl/src/vector_algorithms.cpp

@@ -7,6 +7,7 @@
 
 #if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC
 #include <cstdint>
+#include <cstring>
 #ifndef _M_ARM64EC
 #include <intrin.h>
 #include <isa_availability.h>
@@ -1563,5 +1564,114 @@ __declspec(noalias) size_t
  return __std_count_trivial_impl<_Find_traits_8>(_First, _Last, _Val);
 }
 
+} // extern "C"
+
+#ifndef _M_ARM64EC
+namespace {
+ __m128i __forceinline _Bitset_to_string_1_step(const uint16_t _Val, const __m128i _Px0, const __m128i _Px1) {
+ const __m128i _Vx0 = _mm_cvtsi32_si128(_Val);
+ const __m128i _Vx1 = _mm_unpacklo_epi8(_Vx0, _Vx0);
+ const __m128i _Vx2 = _mm_unpacklo_epi8(_Vx1, _Vx1);
+ const __m128i _Vx3 = _mm_shuffle_epi32(_Vx2, _MM_SHUFFLE(0, 0, 1, 1));
+ const __m128i _Msk = _mm_and_si128(_Vx3, _mm_set1_epi64x(0x0102040810204080));
+ const __m128i _Ex0 = _mm_cmpeq_epi8(_Msk, _mm_setzero_si128());
+ const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1);
+ return _Ex1;
+ }
+
+ __m128i __forceinline _Bitset_to_string_2_step(const uint8_t _Val, const __m128i _Px0, const __m128i _Px1) {
+ const __m128i _Vx = _mm_set1_epi16(_Val);
+ const __m128i _Msk = _mm_and_si128(_Vx, _mm_set_epi64x(0x0001000200040008, 0x0010002000400080));
+ const __m128i _Ex0 = _mm_cmpeq_epi16(_Msk, _mm_setzero_si128());
+ const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1);
+ return _Ex1;
+ }
+} // unnamed namespace
+#endif // !defined(_M_ARM64EC)
+
+extern "C" {
+
+__declspec(noalias) void __stdcall __std_bitset_to_string_1(
+ char* const _Dest, const void* _Src, size_t _Size_bits, const char _Elem0, const char _Elem1) noexcept {
+#ifndef _M_ARM64EC
+ if (_Use_sse2()) {
+ const __m128i _Px0 = _mm_set1_epi8(_Elem0 ^ _Elem1);
+ const __m128i _Px1 = _mm_set1_epi8(_Elem1);
+ if (_Size_bits >= 16) {
+ char* _Pos = _Dest + _Size_bits;
+ _Size_bits &= 0xF;
+ char* const _Stop_at = _Dest + _Size_bits;
+ do {
+ uint16_t _Val;
+ memcpy(&_Val, _Src, 2);
+ const __m128i _Elems = _Bitset_to_string_1_step(_Val, _Px0, _Px1);
+ _Pos -= 16;
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(_Pos), _Elems);
+ _Advance_bytes(_Src, 2);
+ } while (_Pos != _Stop_at);
+ }
+
+ if (_Size_bits > 0) {
+ __assume(_Size_bits < 16);
+ uint16_t _Val;
+ if (_Size_bits > 8) {
+ memcpy(&_Val, _Src, 2);
+ } else {
+ _Val = *reinterpret_cast<const uint8_t*>(_Src);
+ }
+ const __m128i _Elems = _Bitset_to_string_1_step(_Val, _Px0, _Px1);
+ char _Tmp[16];
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(_Tmp), _Elems);
+ const char* const _Tmpd = _Tmp + (16 - _Size_bits);
+ for (size_t _Ix = 0; _Ix < _Size_bits; ++_Ix) {
+ _Dest[_Ix] = _Tmpd[_Ix];
+ }
+ }
+ }
+#endif // !defined(_M_ARM64EC)
+ const auto _Arr = reinterpret_cast<const uint8_t*>(_Src);
+ for (size_t _Ix = 0; _Ix < _Size_bits; ++_Ix) {
+ _Dest[_Size_bits - 1 - _Ix] = ((_Arr[_Ix >> 3] >> (_Ix & 7)) & 1) != 0 ? _Elem1 : _Elem0;
+ }
+}
+
+__declspec(noalias) void __stdcall __std_bitset_to_string_2(
+ wchar_t* const _Dest, const void* _Src, size_t _Size_bits, const wchar_t _Elem0, const wchar_t _Elem1) noexcept {
+#ifndef _M_ARM64EC
+ if (_Use_sse2()) {
+ const __m128i _Px0 = _mm_set1_epi16(_Elem0 ^ _Elem1);
+ const __m128i _Px1 = _mm_set1_epi16(_Elem1);
+ if (_Size_bits >= 8) {
+ wchar_t* _Pos = _Dest + _Size_bits;
+ _Size_bits &= 0x7;
+ wchar_t* const _Stop_at = _Dest + _Size_bits;
+ do {
+ const uint8_t _Val = *reinterpret_cast<const uint8_t*>(_Src);
+ const __m128i _Elems = _Bitset_to_string_2_step(_Val, _Px0, _Px1);
+ _Pos -= 8;
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(_Pos), _Elems);
+ _Advance_bytes(_Src, 1);
+ } while (_Pos != _Stop_at);
+ }
+
+ if (_Size_bits > 0) {
+ __assume(_Size_bits < 8);
+ const uint8_t _Val = *reinterpret_cast<const uint8_t*>(_Src);
+ const __m128i _Elems = _Bitset_to_string_2_step(_Val, _Px0, _Px1);
+ wchar_t _Tmp[8];
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(_Tmp), _Elems);
+ const wchar_t* const _Tmpd = _Tmp + (8 - _Size_bits);
+ for (size_t _Ix = 0; _Ix < _Size_bits; ++_Ix) {
+ _Dest[_Ix] = _Tmpd[_Ix];
+ }
+ }
+ }
+#endif // !defined(_M_ARM64EC)
+ const auto _Arr = reinterpret_cast<const uint8_t*>(_Src);
+ for (size_t _Ix = 0; _Ix < _Size_bits; ++_Ix) {
+ _Dest[_Size_bits - 1 - _Ix] = ((_Arr[_Ix >> 3] >> (_Ix & 7)) & 1) != 0 ? _Elem1 : _Elem0;
+ }
+}
+
 } // extern "C"
 #endif // defined(_M_IX86) || defined(_M_X64)

tests/std/tests/VSO_0000000_vector_algorithms/test.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 #include <algorithm>
+#include <bitset>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -12,6 +13,7 @@
 #include <limits>
 #include <list>
 #include <random>
+#include <string>
 #include <type_traits>
 #include <vector>
 
@@ -443,6 +445,73 @@ void test_one_container() {
  test_two_containers<Container, list<int>>();
 }
 
+void test_bitset(mt19937_64& gen) {
+ assert(bitset<0>(0x0ULL).to_string() == "");
+ assert(bitset<0>(0xFEDCBA9876543210ULL).to_string() == "");
+ assert(bitset<15>(0x6789ULL).to_string() == "110011110001001");
+ assert(bitset<15>(0xFEDCBA9876543210ULL).to_string() == "011001000010000");
+ assert(bitset<32>(0xABCD1234ULL).to_string() == "10101011110011010001001000110100");
+ assert(bitset<32>(0xFEDCBA9876543210ULL).to_string() == "01110110010101000011001000010000");
+ assert(bitset<45>(0x1701D1729FFFULL).to_string() == "101110000000111010001011100101001111111111111");
+ assert(bitset<45>(0xFEDCBA9876543210ULL).to_string() == "110101001100001110110010101000011001000010000");
+ assert(bitset<64>(0xFEDCBA9876543210ULL).to_string()
+ == "1111111011011100101110101001100001110110010101000011001000010000");
+ assert(bitset<75>(0xFEDCBA9876543210ULL).to_string()
+ == "000000000001111111011011100101110101001100001110110010101000011001000010000");
+
+ assert(bitset<0>(0x0ULL).to_string<wchar_t>() == L"");
+ assert(bitset<0>(0xFEDCBA9876543210ULL).to_string<wchar_t>() == L"");
+ assert(bitset<15>(0x6789ULL).to_string<wchar_t>() == L"110011110001001");
+ assert(bitset<15>(0xFEDCBA9876543210ULL).to_string<wchar_t>() == L"011001000010000");
+ assert(bitset<32>(0xABCD1234ULL).to_string<wchar_t>() == L"10101011110011010001001000110100");
+ assert(bitset<32>(0xFEDCBA9876543210ULL).to_string<wchar_t>() == L"01110110010101000011001000010000");
+ assert(bitset<45>(0x1701D1729FFFULL).to_string<wchar_t>() == L"101110000000111010001011100101001111111111111");
+ assert(bitset<45>(0xFEDCBA9876543210ULL).to_string<wchar_t>() == L"110101001100001110110010101000011001000010000");
+ assert(bitset<64>(0xFEDCBA9876543210ULL).to_string<wchar_t>()
+ == L"1111111011011100101110101001100001110110010101000011001000010000");
+ assert(bitset<75>(0xFEDCBA9876543210ULL).to_string<wchar_t>()
+ == L"000000000001111111011011100101110101001100001110110010101000011001000010000");
+
+ assert(bitset<64>(0xFEDCBA9876543210ULL).to_string('o', 'x')
+ == "xxxxxxxoxxoxxxooxoxxxoxoxooxxooooxxxoxxooxoxoxooooxxooxooooxoooo");
+ assert(bitset<64>(0xFEDCBA9876543210ULL).to_string<wchar_t>(L'o', L'x')
+ == L"xxxxxxxoxxoxxxooxoxxxoxoxooxxooooxxxoxxooxoxoxooooxxooxooooxoooo");
+
+#ifdef __cpp_lib_char8_t
+ assert(bitset<75>(0xFEDCBA9876543210ULL).to_string<char8_t>()
+ == u8"000000000001111111011011100101110101001100001110110010101000011001000010000");
+#endif // __cpp_lib_char8_t
+ assert(bitset<75>(0xFEDCBA9876543210ULL).to_string<char16_t>()
+ == u"000000000001111111011011100101110101001100001110110010101000011001000010000");
+ assert(bitset<75>(0xFEDCBA9876543210ULL).to_string<char32_t>()
+ == U"000000000001111111011011100101110101001100001110110010101000011001000010000"); // not vectorized
+
+ {
+ constexpr size_t N = 2048;
+
+ string str;
+ wstring wstr;
+ str.reserve(N);
+ wstr.reserve(N);
+
+ while (str.size() != N) {
+ uint64_t random_value = gen();
+
+ for (int bits = 0; bits < 64; ++bits) {
+ const auto character = '0' + (random_value & 1);
+ str.push_back(static_cast<char>(character));
+ wstr.push_back(static_cast<wchar_t>(character));
+ random_value >>= 1;
+ }
+ }
+
+ const bitset<N> b(str);
+
+ assert(b.to_string() == str);
+ assert(b.to_string<wchar_t>() == wstr);
+ }
+}
+
 void test_various_containers() {
  test_one_container<vector<int>>(); // contiguous, vectorizable
  test_one_container<deque<int>>(); // random-access, not vectorizable
@@ -524,20 +593,24 @@ int main() {
 
  test_vector_algorithms(gen);
  test_various_containers();
+ test_bitset(gen);
 #ifndef _M_CEE_PURE
 #if defined(_M_IX86) || defined(_M_X64)
  disable_instructions(__ISA_AVAILABLE_AVX2);
  test_vector_algorithms(gen);
  test_various_containers();
+ test_bitset(gen);
 
  disable_instructions(__ISA_AVAILABLE_SSE42);
  test_vector_algorithms(gen);
  test_various_containers();
+ test_bitset(gen);
 #endif // defined(_M_IX86) || defined(_M_X64)
 #if defined(_M_IX86)
  disable_instructions(__ISA_AVAILABLE_SSE2);
  test_vector_algorithms(gen);
  test_various_containers();
+ test_bitset(gen);
 #endif // defined(_M_IX86)
 #endif // _M_CEE_PURE
 }