diff --git a/Cargo.lock b/Cargo.lock index d22a91073394a..24397c22ef6a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2063,6 +2063,7 @@ dependencies = [ "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "chalk-engine 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "fmt_macros 0.0.0", diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml index a572b6bf919e1..04eb68143a6eb 100644 --- a/src/librustc/Cargo.toml +++ b/src/librustc/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["dylib"] [dependencies] arena = { path = "../libarena" } bitflags = "1.0" +cfg-if = "0.1.2" fmt_macros = { path = "../libfmt_macros" } graphviz = { path = "../libgraphviz" } jobserver = "0.1" diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 4c94c993ab405..26379ee1ad4ac 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -16,6 +16,7 @@ use smallvec::SmallVec; use rustc_data_structures::sync::{Lrc, Lock}; use std::env; use std::hash::Hash; +use std::mem; use ty::{self, TyCtxt}; use util::common::{ProfileQueriesMsg, profq_msg}; @@ -208,8 +209,7 @@ impl DepGraph { self.with_task_impl(key, cx, arg, false, task, |key| OpenTask::Regular(Lock::new(RegularOpenTask { node: key, - reads: SmallVec::new(), - read_set: Default::default(), + read_set: OrderedDepIndexSet::new(), })), |data, key, task| data.borrow_mut().complete_task(key, task)) } @@ -352,8 +352,7 @@ impl DepGraph { if let Some(ref data) = self.data { let (result, open_task) = ty::tls::with_context(|icx| { let task = OpenTask::Anon(Lock::new(AnonOpenTask { - reads: SmallVec::new(), - read_set: Default::default(), + read_set: OrderedDepIndexSet::new(), })); let r = { @@ -949,8 +948,7 @@ impl CurrentDepGraph { if let OpenTask::Regular(task) = task { let RegularOpenTask { node, - read_set: _, - reads + read_set, } = task.into_inner(); assert_eq!(node, key); @@ -961,22 +959,22 @@ impl CurrentDepGraph { // when called for LOCAL_CRATE) or they depend on a CrateMetadata // node. if cfg!(debug_assertions) { - if node.kind.is_input() && reads.len() > 0 && + if node.kind.is_input() && read_set.reads.len() > 0 && // FIXME(mw): Special case for DefSpan until Spans are handled // better in general. node.kind != DepKind::DefSpan && - reads.iter().any(|&i| { + read_set.reads.iter().any(|&i| { !(self.nodes[i].kind == DepKind::CrateMetadata || self.nodes[i].kind == DepKind::Krate) }) { bug!("Input node {:?} with unexpected reads: {:?}", node, - reads.iter().map(|&i| self.nodes[i]).collect::>()) + read_set.reads.iter().map(|&i| self.nodes[i]).collect::>()) } } - self.alloc_node(node, reads) + self.alloc_node(node, read_set.reads) } else { bug!("complete_task() - Expected regular task to be popped") } @@ -985,18 +983,17 @@ impl CurrentDepGraph { fn pop_anon_task(&mut self, kind: DepKind, task: OpenTask) -> DepNodeIndex { if let OpenTask::Anon(task) = task { let AnonOpenTask { - read_set: _, - reads + read_set, } = task.into_inner(); debug_assert!(!kind.is_input()); let mut fingerprint = self.anon_id_seed; let mut hasher = StableHasher::new(); - for &read in reads.iter() { + for &read in read_set.reads.iter() { let read_dep_node = self.nodes[read]; - ::std::mem::discriminant(&read_dep_node.kind).hash(&mut hasher); + mem::discriminant(&read_dep_node.kind).hash(&mut hasher); // Fingerprint::combine() is faster than sending Fingerprint // through the StableHasher (at least as long as StableHasher @@ -1014,7 +1011,7 @@ impl CurrentDepGraph { if let Some(&index) = self.node_to_node_index.get(&target_dep_node) { index } else { - self.alloc_node(target_dep_node, reads) + self.alloc_node(target_dep_node, read_set.reads) } } else { bug!("pop_anon_task() - Expected anonymous task to be popped") @@ -1040,9 +1037,8 @@ impl CurrentDepGraph { OpenTask::Regular(ref task) => { let mut task = task.lock(); self.total_read_count += 1; - if task.read_set.insert(source) { - task.reads.push(source); + if task.read_set.insert(source) { if cfg!(debug_assertions) { if let Some(ref forbidden_edge) = self.forbidden_edge { let target = &task.node; @@ -1059,10 +1055,7 @@ impl CurrentDepGraph { } } OpenTask::Anon(ref task) => { - let mut task = task.lock(); - if task.read_set.insert(source) { - task.reads.push(source); - } + task.lock().read_set.insert(source); } OpenTask::Ignore | OpenTask::EvalAlways { .. } => { // ignore @@ -1088,13 +1081,11 @@ impl CurrentDepGraph { pub struct RegularOpenTask { node: DepNode, - reads: SmallVec<[DepNodeIndex; 8]>, - read_set: FxHashSet, + read_set: OrderedDepIndexSet, } pub struct AnonOpenTask { - reads: SmallVec<[DepNodeIndex; 8]>, - read_set: FxHashSet, + read_set: OrderedDepIndexSet, } pub enum OpenTask { @@ -1106,6 +1097,87 @@ pub enum OpenTask { }, } +struct OrderedDepIndexSet { + reads: SmallVec<[DepNodeIndex; 8]>, + read_set: FxHashSet, +} + +impl OrderedDepIndexSet { + fn new() -> OrderedDepIndexSet { + OrderedDepIndexSet { + reads: SmallVec::from_buf_and_len([DepNodeIndex::INVALID; 8], 0), + read_set: Default::default(), + } + } +} + +cfg_if! { + if #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(stage0)))] { + impl OrderedDepIndexSet { + + #[inline(always)] + fn insert(&mut self, dep_node_index: DepNodeIndex) -> bool { + unsafe { + self.insert_impl(dep_node_index) + } + } + + #[target_feature(enable = "sse2")] + unsafe fn insert_impl(&mut self, dep_node_index: DepNodeIndex) -> bool { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + if self.reads.len() <= self.reads.inline_size() { + debug_assert!(dep_node_index != DepNodeIndex::INVALID); + debug_assert!(mem::size_of::() == 4); + debug_assert!(self.reads.capacity() == 8); + + let ptr = self.reads.as_slice().as_ptr() as *const __m128i; + let data1 = _mm_loadu_si128(ptr); + let data2 = _mm_loadu_si128(ptr.offset(1)); + let cmp = _mm_set1_epi32(dep_node_index.as_u32() as i32); + + if (_mm_movemask_epi8(_mm_cmpeq_epi32(cmp, data1)) | + _mm_movemask_epi8(_mm_cmpeq_epi32(cmp, data2))) != 0 { + // Already contained + false + } else { + self.reads.push(dep_node_index); + + if self.reads.len() > self.reads.inline_size() { + self.read_set.extend(self.reads.iter().cloned()); + } + true + } + } else { + if self.read_set.insert(dep_node_index) { + self.reads.push(dep_node_index); + true + } else { + false + } + } + } + } + } else { + impl OrderedDepIndexSet { + #[inline(always)] + fn insert(&mut self, dep_node_index: DepNodeIndex) -> bool { + if self.read_set.insert(dep_node_index) { + self.reads.push(dep_node_index); + true + } else { + false + } + } + } + } +} + // A data structure that stores Option values as a contiguous // array, using one u32 per entry. struct DepNodeColorMap { diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index 4324cfc7b5f10..73a1fe608aa24 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -69,6 +69,7 @@ #![feature(in_band_lifetimes)] #![feature(crate_visibility_modifier)] #![feature(transpose_result)] +#![cfg_attr(not(stage0), feature(stdsimd))] #![recursion_limit="512"] @@ -108,6 +109,8 @@ extern crate backtrace; #[macro_use] extern crate smallvec; +#[macro_use] +extern crate cfg_if; // Note that librustc doesn't actually depend on these crates, see the note in // `Cargo.toml` for this crate about why these are here.