1
1
use super :: key:: { Key , LazyKey , get, set} ;
2
2
use super :: { abort_on_dtor_unwind, guard} ;
3
- use crate :: alloc:: Layout ;
3
+ use crate :: alloc:: { self , Layout } ;
4
4
use crate :: cell:: Cell ;
5
5
use crate :: marker:: PhantomData ;
6
- use crate :: ptr;
6
+ use crate :: mem:: ManuallyDrop ;
7
+ use crate :: ops:: Deref ;
8
+ use crate :: panic:: { AssertUnwindSafe , catch_unwind, resume_unwind} ;
9
+ use crate :: ptr:: { self , NonNull } ;
7
10
8
11
#[ doc( hidden) ]
9
12
#[ allow_internal_unstable( thread_local_internals) ]
@@ -110,6 +113,60 @@ pub const fn value_align<T: 'static>() -> usize {
110
113
crate :: mem:: align_of :: < Value < T > > ( )
111
114
}
112
115
116
+ /// Equivalent to `Box<Value<T>>`, but potentially over-aligned.
117
+ struct AlignedBox < T : ' static , const ALIGN : usize > {
118
+ ptr : NonNull < Value < T > > ,
119
+ }
120
+
121
+ impl < T : ' static , const ALIGN : usize > AlignedBox < T , ALIGN > {
122
+ #[ inline]
123
+ fn new ( v : Value < T > ) -> Self {
124
+ let layout = Layout :: new :: < Value < T > > ( ) . align_to ( ALIGN ) . unwrap ( ) ;
125
+
126
+ let ptr: * mut Value < T > = ( unsafe { alloc:: alloc ( layout) } ) . cast ( ) ;
127
+ let Some ( ptr) = NonNull :: new ( ptr) else {
128
+ alloc:: handle_alloc_error ( layout) ;
129
+ } ;
130
+ unsafe { ptr. write ( v) } ;
131
+ Self { ptr }
132
+ }
133
+
134
+ #[ inline]
135
+ fn into_raw ( b : Self ) -> * mut Value < T > {
136
+ let md = ManuallyDrop :: new ( b) ;
137
+ md. ptr . as_ptr ( )
138
+ }
139
+
140
+ #[ inline]
141
+ unsafe fn from_raw ( ptr : * mut Value < T > ) -> Self {
142
+ Self { ptr : unsafe { NonNull :: new_unchecked ( ptr) } }
143
+ }
144
+ }
145
+
146
+ impl < T : ' static , const ALIGN : usize > Deref for AlignedBox < T , ALIGN > {
147
+ type Target = Value < T > ;
148
+
149
+ #[ inline]
150
+ fn deref ( & self ) -> & Self :: Target {
151
+ unsafe { & * ( self . ptr . as_ptr ( ) ) }
152
+ }
153
+ }
154
+
155
+ impl < T : ' static , const ALIGN : usize > Drop for AlignedBox < T , ALIGN > {
156
+ #[ inline]
157
+ fn drop ( & mut self ) {
158
+ let layout = Layout :: new :: < Value < T > > ( ) . align_to ( ALIGN ) . unwrap ( ) ;
159
+
160
+ unsafe {
161
+ let unwind_result = catch_unwind ( AssertUnwindSafe ( || self . ptr . drop_in_place ( ) ) ) ;
162
+ alloc:: dealloc ( self . ptr . as_ptr ( ) . cast ( ) , layout) ;
163
+ if let Err ( payload) = unwind_result {
164
+ resume_unwind ( payload) ;
165
+ }
166
+ }
167
+ }
168
+ }
169
+
113
170
impl < T : ' static , const ALIGN : usize > Storage < T , ALIGN > {
114
171
pub const fn new ( ) -> Storage < T , ALIGN > {
115
172
Storage { key : LazyKey :: new ( Some ( destroy_value :: < T , ALIGN > ) ) , marker : PhantomData }
@@ -148,15 +205,11 @@ impl<T: 'static, const ALIGN: usize> Storage<T, ALIGN> {
148
205
return ptr:: null ( ) ;
149
206
}
150
207
151
- // Manually allocate with the requested alignment
152
- let layout = Layout :: new :: < Value < T > > ( ) . align_to ( ALIGN ) . unwrap ( ) ;
153
- let ptr: * mut Value < T > = ( unsafe { crate :: alloc:: alloc ( layout) } ) . cast ( ) ;
154
- if ptr. is_null ( ) {
155
- crate :: alloc:: handle_alloc_error ( layout) ;
156
- }
157
- unsafe {
158
- ptr. write ( Value { value : i. and_then ( Option :: take) . unwrap_or_else ( f) , key } ) ;
159
- }
208
+ let value = AlignedBox :: < T , ALIGN > :: new ( Value {
209
+ value : i. and_then ( Option :: take) . unwrap_or_else ( f) ,
210
+ key,
211
+ } ) ;
212
+ let ptr = AlignedBox :: into_raw ( value) ;
160
213
161
214
// SAFETY:
162
215
// * key came from a `LazyKey` and is thus correct.
@@ -174,10 +227,7 @@ impl<T: 'static, const ALIGN: usize> Storage<T, ALIGN> {
174
227
// initializer has already returned and the next scope only starts
175
228
// after we return the pointer. Therefore, there can be no references
176
229
// to the old value.
177
- unsafe {
178
- old. drop_in_place ( ) ;
179
- crate :: alloc:: dealloc ( old. cast ( ) , layout) ;
180
- }
230
+ drop ( unsafe { AlignedBox :: < T , ALIGN > :: from_raw ( old) } ) ;
181
231
}
182
232
183
233
// SAFETY: We just created this value above.
@@ -196,22 +246,13 @@ unsafe extern "C" fn destroy_value<T: 'static, const ALIGN: usize>(ptr: *mut u8)
196
246
// Note that to prevent an infinite loop we reset it back to null right
197
247
// before we return from the destructor ourselves.
198
248
abort_on_dtor_unwind ( || {
199
- let value_ptr: * mut Value < T > = ptr. cast ( ) ;
200
- unsafe {
201
- let key = ( * value_ptr) . key ;
202
-
203
- // SAFETY: `key` is the TLS key `ptr` was stored under.
204
- set ( key, ptr:: without_provenance_mut ( 1 ) ) ;
205
-
206
- // drop and deallocate the value
207
- let layout =
208
- Layout :: from_size_align_unchecked ( crate :: mem:: size_of :: < Value < T > > ( ) , ALIGN ) ;
209
- value_ptr. drop_in_place ( ) ;
210
- crate :: alloc:: dealloc ( ptr, layout) ;
211
-
212
- // SAFETY: `key` is the TLS key `ptr` was stored under.
213
- set ( key, ptr:: null_mut ( ) ) ;
214
- } ;
249
+ let ptr = unsafe { AlignedBox :: < T , ALIGN > :: from_raw ( ptr as * mut Value < T > ) } ;
250
+ let key = ptr. key ;
251
+ // SAFETY: `key` is the TLS key `ptr` was stored under.
252
+ unsafe { set ( key, ptr:: without_provenance_mut ( 1 ) ) } ;
253
+ drop ( ptr) ;
254
+ // SAFETY: `key` is the TLS key `ptr` was stored under.
255
+ unsafe { set ( key, ptr:: null_mut ( ) ) } ;
215
256
// Make sure that the runtime cleanup will be performed
216
257
// after the next round of TLS destruction.
217
258
guard:: enable ( ) ;
0 commit comments