[libc++] Replace __compressed_pair with [[no_unique_address]]

Author: philnik777

libcxx/include/__config

@@ -168,6 +168,8 @@
 // pointer from 16 to 8. This changes the output of std::string::max_size,
 // which makes it ABI breaking
 #    define _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
+// Don't add padding from __compressed_pair
+#    define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 #  elif _LIBCPP_ABI_VERSION == 1
 #    if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support

libcxx/include/__functional/function.h

@@ -138,45 +138,46 @@ class __default_alloc_func;
 
 template <class _Fp, class _Ap, class _Rp, class... _ArgTypes>
 class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> {
-  __compressed_pair<_Fp, _Ap> __f_;
+  _LIBCPP_COMPRESSED_PAIR(_Fp, __func_, _Ap, __alloc_);
 
 public:
   typedef _LIBCPP_NODEBUG _Fp _Target;
   typedef _LIBCPP_NODEBUG _Ap _Alloc;
 
-  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_.first(); }
+  _LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __func_; }
 
   // WIN32 APIs may define __allocator, so use __get_allocator instead.
-  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __f_.second(); }
+  _LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __alloc_; }
 
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f)
-      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple()) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f) : __func_(std::move(__f)), __alloc_() {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(__a)) {}
+  _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a) : __func_(__f), __alloc_(__a) {}
 
   _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(std::move(__a))) {}
+      : __func_(__f), __alloc_(std::move(__a)) {}
 
   _LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a)
-      : __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple(std::move(__a))) {}
+      : __func_(std::move(__f)), __alloc_(std::move(__a)) {}
 
   _LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
     typedef __invoke_void_return_wrapper<_Rp> _Invoker;
-    return _Invoker::__call(__f_.first(), std::forward<_ArgTypes>(__arg)...);
+    return _Invoker::__call(__func_, std::forward<_ArgTypes>(__arg)...);
   }
 
   _LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const {
     typedef allocator_traits<_Alloc> __alloc_traits;
     typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA;
-    _AA __a(__f_.second());
+    _AA __a(__alloc_);
     typedef __allocator_destructor<_AA> _Dp;
     unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
-    ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a));
+    ::new ((void*)__hold.get()) __alloc_func(__func_, _Alloc(__a));
     return __hold.release();
   }
 
-  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); }
+  _LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT {
+    __func_.~_Fp();
+    __alloc_.~_Alloc();
+  }
 
   _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) {
     typedef allocator_traits<_Alloc> __alloc_traits;

libcxx/include/__hash_table

@@ -11,161 +11,39 @@
 #define _LIBCPP___MEMORY_COMPRESSED_PAIR_H
 
 #include <__config>
-#include <__fwd/get.h>
-#include <__fwd/tuple.h>
-#include <__tuple/tuple_indices.h>
-#include <__type_traits/decay.h>
-#include <__type_traits/dependent_type.h>
-#include <__type_traits/enable_if.h>
-#include <__type_traits/is_default_constructible.h>
+#include <__type_traits/datasizeof.h>
 #include <__type_traits/is_empty.h>
 #include <__type_traits/is_final.h>
-#include <__type_traits/is_same.h>
-#include <__type_traits/is_swappable.h>
-#include <__utility/forward.h>
-#include <__utility/move.h>
-#include <__utility/piecewise_construct.h>
-#include <cstddef>
+#include <__type_traits/is_object.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
-_LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// Tag used to default initialize one or both of the pair's elements.
-struct __default_init_tag {};
-struct __value_init_tag {};
-
-template <class _Tp, int _Idx, bool _CanBeEmptyBase = is_empty<_Tp>::value && !__libcpp_is_final<_Tp>::value>
-struct __compressed_pair_elem {
-  using _ParamT         = _Tp;
-  using reference       = _Tp&;
-  using const_reference = const _Tp&;
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_() {}
-
-  template <class _Up, class = __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value> >
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
-      : __value_(std::forward<_Up>(__u)) {}
-
-#ifndef _LIBCPP_CXX03_LANG
-  template <class... _Args, size_t... _Indices>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair_elem(
-      piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
-      : __value_(std::forward<_Args>(std::get<_Indices>(__args))...) {}
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return __value_; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return __value_; }
-
-private:
-  _Tp __value_;
-};
-
-template <class _Tp, int _Idx>
-struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp {
-  using _ParamT         = _Tp;
-  using reference       = _Tp&;
-  using const_reference = const _Tp&;
-  using __value_type    = _Tp;
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem() = default;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_type() {}
-
-  template <class _Up, class = __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value> >
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
-      : __value_type(std::forward<_Up>(__u)) {}
-
-#ifndef _LIBCPP_CXX03_LANG
-  template <class... _Args, size_t... _Indices>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17
-  __compressed_pair_elem(piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
-      : __value_type(std::forward<_Args>(std::get<_Indices>(__args))...) {}
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return *this; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return *this; }
-};
-
-template <class _T1, class _T2>
-class __compressed_pair : private __compressed_pair_elem<_T1, 0>, private __compressed_pair_elem<_T2, 1> {
-public:
-  // NOTE: This static assert should never fire because __compressed_pair
-  // is *almost never* used in a scenario where it's possible for T1 == T2.
-  // (The exception is std::function where it is possible that the function
-  //  object and the allocator have the same type).
-  static_assert(
-      (!is_same<_T1, _T2>::value),
-      "__compressed_pair cannot be instantiated when T1 and T2 are the same type; "
-      "The current implementation is NOT ABI-compatible with the previous implementation for this configuration");
-
-  using _Base1 _LIBCPP_NODEBUG = __compressed_pair_elem<_T1, 0>;
-  using _Base2 _LIBCPP_NODEBUG = __compressed_pair_elem<_T2, 1>;
-
-  template <bool _Dummy = true,
-            class       = __enable_if_t< __dependent_type<is_default_constructible<_T1>, _Dummy>::value &&
-                                   __dependent_type<is_default_constructible<_T2>, _Dummy>::value > >
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair()
-      : _Base1(__value_init_tag()), _Base2(__value_init_tag()) {}
-
-  template <class _U1, class _U2>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair(_U1&& __t1, _U2&& __t2)
-      : _Base1(std::forward<_U1>(__t1)), _Base2(std::forward<_U2>(__t2)) {}
-
-#ifndef _LIBCPP_CXX03_LANG
-  template <class... _Args1, class... _Args2>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair(
-      piecewise_construct_t __pc, tuple<_Args1...> __first_args, tuple<_Args2...> __second_args)
-      : _Base1(__pc, std::move(__first_args), typename __make_tuple_indices<sizeof...(_Args1)>::type()),
-        _Base2(__pc, std::move(__second_args), typename __make_tuple_indices<sizeof...(_Args2)>::type()) {}
-#endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base1::reference first() _NOEXCEPT {
-    return static_cast<_Base1&>(*this).__get();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base1::const_reference first() const _NOEXCEPT {
-    return static_cast<_Base1 const&>(*this).__get();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base2::reference second() _NOEXCEPT {
-    return static_cast<_Base2&>(*this).__get();
-  }
+#ifndef _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base2::const_reference second() const _NOEXCEPT {
-    return static_cast<_Base2 const&>(*this).__get();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base1* __get_first_base(__compressed_pair* __pair) _NOEXCEPT {
-    return static_cast<_Base1*>(__pair);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base2* __get_second_base(__compressed_pair* __pair) _NOEXCEPT {
-    return static_cast<_Base2*>(__pair);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void swap(__compressed_pair& __x)
-      _NOEXCEPT_(__is_nothrow_swappable<_T1>::value&& __is_nothrow_swappable<_T2>::value) {
-    using std::swap;
-    swap(first(), __x.first());
-    swap(second(), __x.second());
-  }
+template <class _ToPad>
+class __compressed_pair_padding {
+  char __padding_[(!is_object<_ToPad>::value || (is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value))
+                      ? 0
+                      : sizeof(_ToPad) - __libcpp_datasizeof<_ToPad>::value];
 };
 
-template <class _T1, class _T2>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
-swap(__compressed_pair<_T1, _T2>& __x, __compressed_pair<_T1, _T2>& __y)
-    _NOEXCEPT_(__is_nothrow_swappable<_T1>::value&& __is_nothrow_swappable<_T2>::value) {
-  __x.swap(__y);
-}
+#  define _LIBCPP_COMPRESSED_PAIR_PADDING(T, Name) _LIBCPP_NO_UNIQUE_ADDRESS __compressed_pair_padding<T> Name
+#  define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2)                                                                \
+    [[__gnu__::__aligned__(_LIBCPP_ALIGNOF(T2))]] _LIBCPP_NO_UNIQUE_ADDRESS T1 Name1;                                  \
+    _LIBCPP_COMPRESSED_PAIR_PADDING(T1, Name1##_padding_);                                                             \
+    _LIBCPP_NO_UNIQUE_ADDRESS T2 Name2;                                                                                \
+    _LIBCPP_COMPRESSED_PAIR_PADDING(T2, Name2##_padding_)
+#else
+#  define _LIBCPP_COMPRESSED_PAIR_PADDING(T, Name)
+#  define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2)                                                                \
+    _LIBCPP_NO_UNIQUE_ADDRESS T1 Name1;                                                                                \
+    _LIBCPP_NO_UNIQUE_ADDRESS T2 Name2;
+#endif // _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 
 _LIBCPP_END_NAMESPACE_STD
 
-_LIBCPP_POP_MACROS
-
 #endif // _LIBCPP___MEMORY_COMPRESSED_PAIR_H

libcxx/include/__memory/compressed_pair.h

@@ -196,11 +196,17 @@ class _LIBCPP_EXPORTED_FROM_ABI __shared_weak_count : private __shared_count {
 
 template <class _Tp, class _Dp, class _Alloc>
 class __shared_ptr_pointer : public __shared_weak_count {
-  __compressed_pair<__compressed_pair<_Tp, _Dp>, _Alloc> __data_;
+  [[using __gnu__: __aligned__(_LIBCPP_ALIGNOF(_Dp)),
+    __aligned__(_LIBCPP_ALIGNOF(_Alloc))]] _LIBCPP_NO_UNIQUE_ADDRESS _Tp __ptr_;
+  _LIBCPP_COMPRESSED_PAIR_PADDING(_Tp, __ptr_padding_);
+  _LIBCPP_NO_UNIQUE_ADDRESS _Dp __deleter_;
+  _LIBCPP_COMPRESSED_PAIR_PADDING(_Dp, __deleter_padding_);
+  _LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
+  _LIBCPP_COMPRESSED_PAIR_PADDING(_Alloc, __alloc_padding_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __shared_ptr_pointer(_Tp __p, _Dp __d, _Alloc __a)
-      : __data_(__compressed_pair<_Tp, _Dp>(__p, std::move(__d)), std::move(__a)) {}
+      : __ptr_(__p), __deleter_(std::move(__d)), __alloc_(std::move(__a)) {}
 
 #ifndef _LIBCPP_HAS_NO_RTTI
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL const void* __get_deleter(const type_info&) const _NOEXCEPT override;
@@ -215,15 +221,15 @@ class __shared_ptr_pointer : public __shared_weak_count {
 
 template <class _Tp, class _Dp, class _Alloc>
 const void* __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__get_deleter(const type_info& __t) const _NOEXCEPT {
-  return __t == typeid(_Dp) ? std::addressof(__data_.first().second()) : nullptr;
+  return __t == typeid(_Dp) ? std::addressof(__deleter_) : nullptr;
 }
 
 #endif // _LIBCPP_HAS_NO_RTTI
 
 template <class _Tp, class _Dp, class _Alloc>
 void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared() _NOEXCEPT {
-  __data_.first().second()(__data_.first().first());
-  __data_.first().second().~_Dp();
+  __deleter_(__ptr_);
+  __deleter_.~_Dp();
 }
 
 template <class _Tp, class _Dp, class _Alloc>
@@ -232,8 +238,8 @@ void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared_weak() _NOEXCEPT {
   typedef allocator_traits<_Al> _ATraits;
   typedef pointer_traits<typename _ATraits::pointer> _PTraits;
 
-  _Al __a(__data_.second());
-  __data_.second().~_Alloc();
+  _Al __a(__alloc_);
+  __alloc_.~_Alloc();
   __a.deallocate(_PTraits::pointer_to(*this), 1);
 }
 
@@ -299,28 +305,19 @@ struct __shared_ptr_emplace : __shared_weak_count {
   // with Allocator construction for _Tp. To allow implementing P0674 in C++20,
   // we now use a properly aligned char buffer while making sure that we maintain
   // the same layout that we had when we used a compressed pair.
-  using _CompressedPair = __compressed_pair<_Alloc, _Tp>;
-  struct _ALIGNAS_TYPE(_CompressedPair) _Storage {
-    char __blob_[sizeof(_CompressedPair)];
+  struct [[using __gnu__: __aligned__(_LIBCPP_ALIGNOF(_Alloc)), __aligned__(_LIBCPP_ALIGNOF(_Tp))]] _Storage {
+    union {
+      struct {
+        _LIBCPP_COMPRESSED_PAIR(_Alloc, __alloc_, _Tp, __elem_);
+      };
+    };
 
     _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) { ::new ((void*)__get_alloc()) _Alloc(std::move(__a)); }
     _LIBCPP_HIDE_FROM_ABI ~_Storage() { __get_alloc()->~_Alloc(); }
-    _LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT {
-      _CompressedPair* __as_pair                = reinterpret_cast<_CompressedPair*>(__blob_);
-      typename _CompressedPair::_Base1* __first = _CompressedPair::__get_first_base(__as_pair);
-      _Alloc* __alloc                           = reinterpret_cast<_Alloc*>(__first);
-      return __alloc;
-    }
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT {
-      _CompressedPair* __as_pair                 = reinterpret_cast<_CompressedPair*>(__blob_);
-      typename _CompressedPair::_Base2* __second = _CompressedPair::__get_second_base(__as_pair);
-      _Tp* __elem                                = reinterpret_cast<_Tp*>(__second);
-      return __elem;
-    }
+    _LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT { return std::addressof(__alloc_); }
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT { return std::addressof(__elem_); }
   };
 
-  static_assert(_LIBCPP_ALIGNOF(_Storage) == _LIBCPP_ALIGNOF(_CompressedPair), "");
-  static_assert(sizeof(_Storage) == sizeof(_CompressedPair), "");
   _Storage __storage_;
 };