@@ -27,8 +27,8 @@ pub struct Drain<
2727 drain_len : usize ,
2828 // index into the logical array, not the physical one (always lies in [0..deque.len))
2929 idx : usize ,
30- // number of elements after the drain range
31- tail_len : usize ,
30+ // number of elements remaining after dropping the drain
31+ new_len : usize ,
3232 remaining : usize ,
3333 // Needed to make Drain covariant over T
3434 _marker : PhantomData < & ' a T > ,
@@ -41,12 +41,12 @@ impl<'a, T, A: Allocator> Drain<'a, T, A> {
4141 drain_len : usize ,
4242 ) -> Self {
4343 let orig_len = mem:: replace ( & mut deque. len , drain_start) ;
44- let tail_len = orig_len - drain_start - drain_len;
44+ let new_len = orig_len - drain_len;
4545 Drain {
4646 deque : NonNull :: from ( deque) ,
4747 drain_len,
4848 idx : drain_start,
49- tail_len ,
49+ new_len ,
5050 remaining : drain_len,
5151 _marker : PhantomData ,
5252 }
@@ -79,7 +79,7 @@ impl<T: fmt::Debug, A: Allocator> fmt::Debug for Drain<'_, T, A> {
7979 f. debug_tuple ( "Drain" )
8080 . field ( & self . drain_len )
8181 . field ( & self . idx )
82- . field ( & self . tail_len )
82+ . field ( & self . new_len )
8383 . field ( & self . remaining )
8484 . finish ( )
8585 }
@@ -95,9 +95,26 @@ impl<T, A: Allocator> Drop for Drain<'_, T, A> {
9595 fn drop ( & mut self ) {
9696 struct DropGuard < ' r , ' a , T , A : Allocator > ( & ' r mut Drain < ' a , T , A > ) ;
9797
98+ let guard = DropGuard ( self ) ;
99+
100+ if mem:: needs_drop :: < T > ( ) && guard. 0 . remaining != 0 {
101+ unsafe {
102+ // SAFETY: We just checked that `self.remaining != 0`.
103+ let ( front, back) = guard. 0 . as_slices ( ) ;
104+ // since idx is a logical index, we don't need to worry about wrapping.
105+ guard. 0 . idx += front. len ( ) ;
106+ guard. 0 . remaining -= front. len ( ) ;
107+ ptr:: drop_in_place ( front) ;
108+ guard. 0 . remaining = 0 ;
109+ ptr:: drop_in_place ( back) ;
110+ }
111+ }
112+
113+ // Dropping `guard` handles moving the remaining elements into place.
98114 impl < ' r , ' a , T , A : Allocator > Drop for DropGuard < ' r , ' a , T , A > {
115+ #[ inline]
99116 fn drop ( & mut self ) {
100- if self . 0 . remaining != 0 {
117+ if mem :: needs_drop :: < T > ( ) && self . 0 . remaining != 0 {
101118 unsafe {
102119 // SAFETY: We just checked that `self.remaining != 0`.
103120 let ( front, back) = self . 0 . as_slices ( ) ;
@@ -108,70 +125,111 @@ impl<T, A: Allocator> Drop for Drain<'_, T, A> {
108125
109126 let source_deque = unsafe { self . 0 . deque . as_mut ( ) } ;
110127
111- let drain_start = source_deque. len ( ) ;
112128 let drain_len = self . 0 . drain_len ;
113- let drain_end = drain_start + drain_len;
114-
115- let orig_len = self . 0 . tail_len + drain_end;
129+ let new_len = self . 0 . new_len ;
116130
117131 if T :: IS_ZST {
118132 // no need to copy around any memory if T is a ZST
119- source_deque. len = orig_len - drain_len ;
133+ source_deque. len = new_len ;
120134 return ;
121135 }
122136
123- let head_len = drain_start ;
124- let tail_len = self . 0 . tail_len ;
137+ let head_len = source_deque . len ; // #elements in front of the drain
138+ let tail_len = new_len - head_len ; // #elements behind the drain
125139
126- match ( head_len, tail_len) {
127- ( 0 , 0 ) => {
128- source_deque. head = 0 ;
129- source_deque. len = 0 ;
130- }
131- ( 0 , _) => {
132- source_deque. head = source_deque. to_physical_idx ( drain_len) ;
133- source_deque. len = orig_len - drain_len;
134- }
135- ( _, 0 ) => {
136- source_deque. len = orig_len - drain_len;
137- }
138- _ => unsafe {
139- if head_len <= tail_len {
140- source_deque. wrap_copy (
141- source_deque. head ,
142- source_deque. to_physical_idx ( drain_len) ,
143- head_len,
144- ) ;
145- source_deque. head = source_deque. to_physical_idx ( drain_len) ;
146- source_deque. len = orig_len - drain_len;
140+ // Next, we will fill the hole left by the drain with as few writes as possible.
141+ // The code below handles the following control flow and reduces the amount of
142+ // branches under the assumption that `head_len == 0 || tail_len == 0`, i.e.
143+ // draining at the front or at the back of the dequeue is especially common.
144+ //
145+ // H = "head index" = `deque.head`
146+ // h = elements in front of the drain
147+ // d = elements in the drain
148+ // t = elements behind the drain
149+ //
150+ // Note that the buffer may wrap at any point and the wrapping is handled by
151+ // `wrap_copy` and `to_physical_idx`.
152+ //
153+ // Case 1: if `head_len == 0 && tail_len == 0`
154+ // Everything was drained, reset the head index back to 0.
155+ // H
156+ // [ . . . . . d d d d . . . . . ]
157+ // H
158+ // [ . . . . . . . . . . . . . . ]
159+ //
160+ // Case 2: else if `tail_len == 0`
161+ // Don't move data or the head index.
162+ // H
163+ // [ . . . h h h h d d d d . . . ]
164+ // H
165+ // [ . . . h h h h . . . . . . . ]
166+ //
167+ // Case 3: else if `head_len == 0`
168+ // Don't move data, but move the head index.
169+ // H
170+ // [ . . . d d d d t t t t . . . ]
171+ // H
172+ // [ . . . . . . . t t t t . . . ]
173+ //
174+ // Case 4: else if `tail_len <= head_len`
175+ // Move data, but not the head index.
176+ // H
177+ // [ . . h h h h d d d d t t . . ]
178+ // H
179+ // [ . . h h h h t t . . . . . . ]
180+ //
181+ // Case 5: else
182+ // Move data and the head index.
183+ // H
184+ // [ . . h h d d d d t t t t . . ]
185+ // H
186+ // [ . . . . . . h h t t t t . . ]
187+
188+ // When draining at the front (`.drain(..n)`) or at the back (`.drain(n..)`),
189+ // we don't need to copy any data. The number of elements copied would be 0.
190+ if head_len != 0 && tail_len != 0 {
191+ join_head_and_tail_wrapping ( source_deque, drain_len, head_len, tail_len) ;
192+ // Marking this function as cold helps LLVM to eliminate it entirely if
193+ // this branch is never taken.
194+ // We use `#[cold]` instead of `#[inline(never)]`, because inlining this
195+ // function into the general case (`.drain(n..m)`) is fine.
196+ // See `tests/codegen/vecdeque-drain.rs` for a test.
197+ #[ cold]
198+ fn join_head_and_tail_wrapping < T , A : Allocator > (
199+ source_deque : & mut VecDeque < T , A > ,
200+ drain_len : usize ,
201+ head_len : usize ,
202+ tail_len : usize ,
203+ ) {
204+ // Pick whether to move the head or the tail here.
205+ let ( src, dst, len) ;
206+ if head_len < tail_len {
207+ src = source_deque. head ;
208+ dst = source_deque. to_physical_idx ( drain_len) ;
209+ len = head_len;
147210 } else {
148- source_deque. wrap_copy (
149- source_deque. to_physical_idx ( head_len + drain_len) ,
150- source_deque. to_physical_idx ( head_len) ,
151- tail_len,
152- ) ;
153- source_deque. len = orig_len - drain_len;
211+ src = source_deque. to_physical_idx ( head_len + drain_len) ;
212+ dst = source_deque. to_physical_idx ( head_len) ;
213+ len = tail_len;
214+ } ;
215+
216+ unsafe {
217+ source_deque. wrap_copy ( src, dst, len) ;
154218 }
155- } ,
219+ }
156220 }
157- }
158- }
159221
160- let guard = DropGuard ( self ) ;
161- if guard. 0 . remaining != 0 {
162- unsafe {
163- // SAFETY: We just checked that `self.remaining != 0`.
164- let ( front, back) = guard. 0 . as_slices ( ) ;
165- // since idx is a logical index, we don't need to worry about wrapping.
166- guard. 0 . idx += front. len ( ) ;
167- guard. 0 . remaining -= front. len ( ) ;
168- ptr:: drop_in_place ( front) ;
169- guard. 0 . remaining = 0 ;
170- ptr:: drop_in_place ( back) ;
222+ if new_len == 0 {
223+ // Special case: If the entire dequeue was drained, reset the head back to 0,
224+ // like `.clear()` does.
225+ source_deque. head = 0 ;
226+ } else if head_len < tail_len {
227+ // If we moved the head above, then we need to adjust the head index here.
228+ source_deque. head = source_deque. to_physical_idx ( drain_len) ;
229+ }
230+ source_deque. len = new_len;
171231 }
172232 }
173-
174- // Dropping `guard` handles moving the remaining elements into place.
175233 }
176234}
177235
0 commit comments