Skip to content

Commit f8a0286

Browse files
nnethercotennethercote
committed
Speed up SipHasher128.
The current code in `SipHasher128::short_write` is inefficient. It uses `u8to64_le` (which is complex and slow) to extract just the right number of bytes of the input into a u64 and pad the result with zeroes. It then left-shifts that value in order to bitwise-OR it with `self.tail`. For example, imagine we have a u32 input 0xIIHH_GGFF and only need three bytes to fill up `self.tail`. The current code uses `u8to64_le` to construct 0x0000_0000_00HH_GGFF, which is just 0xIIHH_GGFF with the 0xII removed and zero-extended to a u64. The code then left-shifts that value by five bytes -- discarding the 0x00 byte that replaced the 0xII byte! -- to give 0xHHGG_FF00_0000_0000. It then then ORs that value with self.tail. There's a much simpler way to do it: zero-extend to u64 first, then left shift. E.g. 0xIIHH_GGFF is zero-extended to 0x0000_0000_IIHH_GGFF, and then left-shifted to 0xHHGG_FF00_0000_0000. We don't have to take time to exclude the unneeded 0xII byte, because it just gets shifted out anyway! It also avoids multiple occurrences of `unsafe`. There's a similar story with the setting of `self.tail` at the method's end. The current code uses `u8to64_le` to extract the remaining part of the input, but the same effect can be achieved more quickly with a right shift on the zero-extended input. All that works on little-endian. It doesn't work for big-endian, but we can just do a `to_le` before calling `short_write` and then it works. This commit changes `SipHasher128` to use the simpler shift-based approach. The code is also smaller, which means that `short_write` is now inlined where previously it wasn't, which makes things faster again. This gives big speed-ups for all incremental builds, especially "baseline" incremental builds.
1 parent a19edd6 commit f8a0286

File tree

1 file changed

+72
-39
lines changed

1 file changed

+72
-39
lines changed

‎src/librustc_data_structures/sip128.rs

+72-39
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ use std::cmp;
44
use std::hash::Hasher;
55
use std::mem;
66
use std::ptr;
7-
use std::slice;
87

98
#[cfg(test)]
109
mod tests;
@@ -122,42 +121,76 @@ impl SipHasher128 {
122121
self.state.v1 ^= 0xee;
123122
}
124123

125-
// Specialized write function that is only valid for buffers with len <= 8.
126-
// It's used to force inlining of write_u8 and write_usize, those would normally be inlined
127-
// except for composite types (that includes slices and str hashing because of delimiter).
128-
// Without this extra push the compiler is very reluctant to inline delimiter writes,
129-
// degrading performance substantially for the most common use cases.
124+
// A specialized write function for values with size <= 8.
125+
//
126+
// The hashing of multi-byte integers depends on endianness. E.g.:
127+
// - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])`
128+
// - big-endian: `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])`
129+
//
130+
// This function does the right thing for little-endian hardware. On
131+
// big-endian hardware `x` must be byte-swapped first to give the right
132+
// behaviour. After any byte-swapping, the input must be zero-extended to
133+
// 64-bits. The caller is responsible for the byte-swapping and
134+
// zero-extension.
130135
#[inline]
131-
fn short_write(&mut self, msg: &[u8]) {
132-
debug_assert!(msg.len() <= 8);
133-
let length = msg.len();
134-
self.length += length;
136+
fn short_write<T>(&mut self, _x: T, x: u64) {
137+
let size = mem::size_of::<T>();
138+
self.length += size;
139+
140+
// The original number must be zero-extended, not sign-extended.
141+
debug_assert!(if size < 8 { x >> (8 * size) == 0 } else { true });
135142

143+
// The number of bytes needed to fill `self.tail`.
136144
let needed = 8 - self.ntail;
137-
let fill = cmp::min(length, needed);
138-
if fill == 8 {
139-
self.tail = unsafe { load_int_le!(msg, 0, u64) };
140-
} else {
141-
self.tail |= unsafe { u8to64_le(msg, 0, fill) } << (8 * self.ntail);
142-
if length < needed {
143-
self.ntail += length;
144-
return;
145-
}
145+
146+
// SipHash parses the input stream as 8-byte little-endian integers.
147+
// Inputs are put into `self.tail` until 8 bytes of data have been
148+
// collected, and then that word is processed.
149+
//
150+
// For example, imagine that `self.tail` is 0x0000_00EE_DDCC_BBAA,
151+
// `self.ntail` is 5 (because 5 bytes have been put into `self.tail`),
152+
// and `needed` is therefore 3.
153+
//
154+
// - Scenario 1, `self.write_u8(0xFF)`: we have already zero-extended
155+
// the input to 0x0000_0000_0000_00FF. We now left-shift it five
156+
// bytes, giving 0x0000_FF00_0000_0000. We then bitwise-OR that value
157+
// into `self.tail`, resulting in 0x0000_FFEE_DDCC_BBAA.
158+
// (Zero-extension of the original input is critical in this scenario
159+
// because we don't want the high two bytes of `self.tail` to be
160+
// touched by the bitwise-OR.) `self.tail` is not yet full, so we
161+
// return early, after updating `self.ntail` to 6.
162+
//
163+
// - Scenario 2, `self.write_u32(0xIIHH_GGFF)`: we have already
164+
// zero-extended the input to 0x0000_0000_IIHH_GGFF. We now
165+
// left-shift it five bytes, giving 0xHHGG_FF00_0000_0000. We then
166+
// bitwise-OR that value into `self.tail`, resulting in
167+
// 0xHHGG_FFEE_DDCC_BBAA. `self.tail` is now full, and we can use it
168+
// to update `self.state`. (As mentioned above, this assumes a
169+
// little-endian machine; on a big-endian machine we would have
170+
// byte-swapped 0xIIHH_GGFF in the caller, giving 0xFFGG_HHII, and we
171+
// would then end up bitwise-ORing 0xGGHH_II00_0000_0000 into
172+
// `self.tail`).
173+
//
174+
self.tail |= x << (8 * self.ntail);
175+
if size < needed {
176+
self.ntail += size;
177+
return;
146178
}
179+
180+
// `self.tail` is full, process it.
147181
self.state.v3 ^= self.tail;
148182
Sip24Rounds::c_rounds(&mut self.state);
149183
self.state.v0 ^= self.tail;
150184

151-
// Buffered tail is now flushed, process new input.
152-
self.ntail = length - needed;
153-
self.tail = unsafe { u8to64_le(msg, needed, self.ntail) };
154-
}
155-
156-
#[inline(always)]
157-
fn short_write_gen<T>(&mut self, x: T) {
158-
let bytes =
159-
unsafe { slice::from_raw_parts(&x as *const T as *const u8, mem::size_of::<T>()) };
160-
self.short_write(bytes);
185+
// Continuing scenario 2: we have one byte left over from the input. We
186+
// set `self.ntail` to 1 and `self.tail` to `0x0000_0000_IIHH_GGFF >>
187+
// 8*3`, which is 0x0000_0000_0000_00II. (Or on a big-endian machine
188+
// the prior byte-swapping would leave us with 0x0000_0000_0000_00FF.)
189+
//
190+
// The `if` is needed to avoid shifting by 64 bits, which Rust
191+
// complains about.
192+
self.ntail = size - needed;
193+
self.tail = if needed < 8 { x >> (8 * needed) } else { 0 };
161194
}
162195

163196
#[inline]
@@ -182,52 +215,52 @@ impl SipHasher128 {
182215
impl Hasher for SipHasher128 {
183216
#[inline]
184217
fn write_u8(&mut self, i: u8) {
185-
self.short_write_gen(i);
218+
self.short_write(i, i as u64);
186219
}
187220

188221
#[inline]
189222
fn write_u16(&mut self, i: u16) {
190-
self.short_write_gen(i);
223+
self.short_write(i, i.to_le() as u64);
191224
}
192225

193226
#[inline]
194227
fn write_u32(&mut self, i: u32) {
195-
self.short_write_gen(i);
228+
self.short_write(i, i.to_le() as u64);
196229
}
197230

198231
#[inline]
199232
fn write_u64(&mut self, i: u64) {
200-
self.short_write_gen(i);
233+
self.short_write(i, i.to_le() as u64);
201234
}
202235

203236
#[inline]
204237
fn write_usize(&mut self, i: usize) {
205-
self.short_write_gen(i);
238+
self.short_write(i, i.to_le() as u64);
206239
}
207240

208241
#[inline]
209242
fn write_i8(&mut self, i: i8) {
210-
self.short_write_gen(i);
243+
self.short_write(i, i as u8 as u64);
211244
}
212245

213246
#[inline]
214247
fn write_i16(&mut self, i: i16) {
215-
self.short_write_gen(i);
248+
self.short_write(i, (i as u16).to_le() as u64);
216249
}
217250

218251
#[inline]
219252
fn write_i32(&mut self, i: i32) {
220-
self.short_write_gen(i);
253+
self.short_write(i, (i as u32).to_le() as u64);
221254
}
222255

223256
#[inline]
224257
fn write_i64(&mut self, i: i64) {
225-
self.short_write_gen(i);
258+
self.short_write(i, (i as u64).to_le() as u64);
226259
}
227260

228261
#[inline]
229262
fn write_isize(&mut self, i: isize) {
230-
self.short_write_gen(i);
263+
self.short_write(i, (i as usize).to_le() as u64);
231264
}
232265

233266
#[inline]

0 commit comments

Comments
 (0)