Auto merge of #57948 - Zoxc:parallel, r=michaelwoerister

Use multiple threads by default. Limits tests to one thread. Do some renaming.

r? @michaelwoerister

Author: bors

Cargo.lock

@@ -2234,6 +2234,7 @@ dependencies = [
  "jobserver 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "polonius-engine 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-rayon 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",

config.toml.example

@@ -317,8 +317,8 @@
 # Whether to always use incremental compilation when building rustc
 #incremental = false
 
-# Build rustc with experimental parallelization
-#experimental-parallel-queries = false
+# Build a multi-threaded rustc
+#parallel-compiler = false
 
 # The default linker that will be hard-coded into the generated compiler for
 # targets that don't specify linker explicitly in their target specifications.

src/bootstrap/bin/rustc.rs

@@ -284,8 +284,8 @@ fn main() {
         }
     }
 
-    if env::var_os("RUSTC_PARALLEL_QUERIES").is_some() {
-        cmd.arg("--cfg").arg("parallel_queries");
+    if env::var_os("RUSTC_PARALLEL_COMPILER").is_some() {
+        cmd.arg("--cfg").arg("parallel_compiler");
     }
 
     if env::var_os("RUSTC_DENY_WARNINGS").is_some() && env::var_os("RUSTC_EXTERNAL_TOOL").is_none()

src/bootstrap/compile.rs

@@ -554,8 +554,8 @@ pub fn rustc_cargo_env(builder: &Builder, cargo: &mut Command) {
     if let Some(ref s) = builder.config.rustc_default_linker {
         cargo.env("CFG_DEFAULT_LINKER", s);
     }
-    if builder.config.rustc_parallel_queries {
-        cargo.env("RUSTC_PARALLEL_QUERIES", "1");
+    if builder.config.rustc_parallel {
+        cargo.env("RUSTC_PARALLEL_COMPILER", "1");
     }
     if builder.config.rust_verify_llvm_ir {
         cargo.env("RUSTC_VERIFY_LLVM_IR", "1");

src/bootstrap/config.rs

@@ -97,7 +97,7 @@ pub struct Config {
     pub rust_debuginfo_only_std: bool,
     pub rust_debuginfo_tools: bool,
     pub rust_rpath: bool,
-    pub rustc_parallel_queries: bool,
+    pub rustc_parallel: bool,
     pub rustc_default_linker: Option<String>,
     pub rust_optimize_tests: bool,
     pub rust_debuginfo_tests: bool,
@@ -298,7 +298,7 @@ struct Rust {
     debuginfo_lines: Option<bool>,
     debuginfo_only_std: Option<bool>,
     debuginfo_tools: Option<bool>,
-    experimental_parallel_queries: Option<bool>,
+    parallel_compiler: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
     channel: Option<String>,
@@ -557,7 +557,7 @@ impl Config {
             set(&mut config.lld_enabled, rust.lld);
             set(&mut config.lldb_enabled, rust.lldb);
             set(&mut config.llvm_tools_enabled, rust.llvm_tools);
-            config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
+            config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);
             config.save_toolstates = rust.save_toolstates.clone().map(PathBuf::from);

src/bootstrap/configure.py

@@ -35,7 +35,7 @@ def v(*args):
 o("docs", "build.docs", "build standard library documentation")
 o("compiler-docs", "build.compiler-docs", "build compiler documentation")
 o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
-o("experimental-parallel-queries", "rust.experimental-parallel-queries", "build rustc with experimental parallelization")
+o("parallel-compiler", "rust.parallel-compiler", "build a multi-threaded rustc")
 o("test-miri", "rust.test-miri", "run miri's test suite")
 o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata")
 o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests")

src/ci/run.sh

@@ -82,7 +82,7 @@ fi
 SCCACHE_IDLE_TIMEOUT=10800 sccache --start-server || true
 
 if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then
-  $SRC/configure --enable-experimental-parallel-queries
+  $SRC/configure --enable-parallel-compiler
   CARGO_INCREMENTAL=0 python2.7 ../x.py check
   rm -f config.toml
   rm -rf build

src/librustc/Cargo.toml

@@ -15,6 +15,7 @@ fmt_macros = { path = "../libfmt_macros" }
 graphviz = { path = "../libgraphviz" }
 jobserver = "0.1"
 lazy_static = "1.0.0"
+num_cpus = "1.0"
 scoped-tls = { version = "0.1.1", features = ["nightly"] }
 log = { version = "0.4", features = ["release_max_level_info", "std"] }
 polonius-engine = "0.6.2"

src/librustc/dep_graph/graph.rs

@@ -580,7 +580,7 @@ impl DepGraph {
     ) -> Option<DepNodeIndex> {
         debug!("try_mark_previous_green({:?}) - BEGIN", dep_node);
 
-        #[cfg(not(parallel_queries))]
+        #[cfg(not(parallel_compiler))]
         {
             debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node));
             debug_assert!(data.colors.get(prev_dep_node_index).is_none());
@@ -743,7 +743,7 @@ impl DepGraph {
 
         // ... and finally storing a "Green" entry in the color map.
         // Multiple threads can all write the same color here
-        #[cfg(not(parallel_queries))]
+        #[cfg(not(parallel_compiler))]
         debug_assert!(data.colors.get(prev_dep_node_index).is_none(),
                       "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
                       insertion for {:?}", dep_node);
@@ -766,7 +766,7 @@ impl DepGraph {
         did_allocation: bool,
         diagnostics: Vec<Diagnostic>,
     ) {
-        if did_allocation || !cfg!(parallel_queries) {
+        if did_allocation || !cfg!(parallel_compiler) {
             // Only the thread which did the allocation emits the error messages
             let handle = tcx.sess.diagnostic();
 
@@ -778,7 +778,7 @@ impl DepGraph {
                 DiagnosticBuilder::new_diagnostic(handle, diagnostic).emit();
             }
 
-            #[cfg(parallel_queries)]
+            #[cfg(parallel_compiler)]
             {
                 // Mark the diagnostics and emitted and wake up waiters
                 data.emitted_diagnostics.lock().insert(dep_node_index);

src/librustc/lib.rs

@@ -70,6 +70,7 @@ extern crate core;
 extern crate fmt_macros;
 extern crate getopts;
 extern crate graphviz;
+extern crate num_cpus;
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate scoped_tls;
 #[cfg(windows)]

src/librustc/session/config.rs

@@ -1194,8 +1194,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "prints the llvm optimization passes being run"),
     ast_json: bool = (false, parse_bool, [UNTRACKED],
         "print the AST as JSON and halt"),
-    query_threads: Option<usize> = (None, parse_opt_uint, [UNTRACKED],
-        "execute queries on a thread pool with N threads"),
+    threads: Option<usize> = (None, parse_opt_uint, [UNTRACKED],
+        "use a thread pool with N threads"),
     ast_json_noexpand: bool = (false, parse_bool, [UNTRACKED],
         "print the pre-expansion AST as JSON and halt"),
     ls: bool = (false, parse_bool, [UNTRACKED],
@@ -1986,17 +1986,17 @@ pub fn build_session_options_and_crate_config(
         }
     }
 
-    if debugging_opts.query_threads == Some(0) {
+    if debugging_opts.threads == Some(0) {
         early_error(
             error_format,
-            "Value for query threads must be a positive nonzero integer",
+            "Value for threads must be a positive nonzero integer",
         );
     }
 
-    if debugging_opts.query_threads.unwrap_or(1) > 1 && debugging_opts.fuel.is_some() {
+    if debugging_opts.threads.unwrap_or(1) > 1 && debugging_opts.fuel.is_some() {
         early_error(
             error_format,
-            "Optimization fuel is incompatible with multiple query threads",
+            "Optimization fuel is incompatible with multiple threads",
         );
     }
 

src/librustc/session/mod.rs

@@ -877,7 +877,7 @@ impl Session {
         let mut ret = true;
         if let Some(ref c) = self.optimization_fuel_crate {
             if c == crate_name {
-                assert_eq!(self.query_threads(), 1);
+                assert_eq!(self.threads(), 1);
                 let mut fuel = self.optimization_fuel.lock();
                 ret = fuel.remaining != 0;
                 if fuel.remaining == 0 && !fuel.out_of_fuel {
@@ -890,7 +890,7 @@ impl Session {
         }
         if let Some(ref c) = self.print_fuel_crate {
             if c == crate_name {
-                assert_eq!(self.query_threads(), 1);
+                assert_eq!(self.threads(), 1);
                 self.print_fuel.fetch_add(1, SeqCst);
             }
         }
@@ -899,14 +899,14 @@ impl Session {
 
     /// Returns the number of query threads that should be used for this
     /// compilation
-    pub fn query_threads_from_opts(opts: &config::Options) -> usize {
-        opts.debugging_opts.query_threads.unwrap_or(1)
+    pub fn threads_from_opts(opts: &config::Options) -> usize {
+        opts.debugging_opts.threads.unwrap_or(::num_cpus::get())
     }
 
     /// Returns the number of query threads that should be used for this
     /// compilation
-    pub fn query_threads(&self) -> usize {
-        Self::query_threads_from_opts(&self.opts)
+    pub fn threads(&self) -> usize {
+        Self::threads_from_opts(&self.opts)
     }
 
     /// Returns the number of codegen units that should be used for this

src/librustc/ty/context.rs

@@ -1823,10 +1823,10 @@ pub mod tls {
     use rustc_data_structures::thin_vec::ThinVec;
     use dep_graph::TaskDeps;
 
-    #[cfg(not(parallel_queries))]
+    #[cfg(not(parallel_compiler))]
     use std::cell::Cell;
 
-    #[cfg(parallel_queries)]
+    #[cfg(parallel_compiler)]
     use rayon_core;
 
     /// This is the implicit state of rustc. It contains the current
@@ -1859,28 +1859,28 @@ pub mod tls {
     /// Sets Rayon's thread local variable which is preserved for Rayon jobs
     /// to `value` during the call to `f`. It is restored to its previous value after.
     /// This is used to set the pointer to the new ImplicitCtxt.
-    #[cfg(parallel_queries)]
+    #[cfg(parallel_compiler)]
     #[inline]
     fn set_tlv<F: FnOnce() -> R, R>(value: usize, f: F) -> R {
         rayon_core::tlv::with(value, f)
     }
 
     /// Gets Rayon's thread local variable which is preserved for Rayon jobs.
     /// This is used to get the pointer to the current ImplicitCtxt.
-    #[cfg(parallel_queries)]
+    #[cfg(parallel_compiler)]
     #[inline]
     fn get_tlv() -> usize {
         rayon_core::tlv::get()
     }
 
     /// A thread local variable which stores a pointer to the current ImplicitCtxt
-    #[cfg(not(parallel_queries))]
+    #[cfg(not(parallel_compiler))]
     thread_local!(static TLV: Cell<usize> = Cell::new(0));
 
     /// Sets TLV to `value` during the call to `f`.
     /// It is restored to its previous value after.
     /// This is used to set the pointer to the new ImplicitCtxt.
-    #[cfg(not(parallel_queries))]
+    #[cfg(not(parallel_compiler))]
     #[inline]
     fn set_tlv<F: FnOnce() -> R, R>(value: usize, f: F) -> R {
         let old = get_tlv();
@@ -1890,7 +1890,7 @@ pub mod tls {
     }
 
     /// This is used to get the pointer to the current ImplicitCtxt.
-    #[cfg(not(parallel_queries))]
+    #[cfg(not(parallel_compiler))]
     fn get_tlv() -> usize {
         TLV.with(|tlv| tlv.get())
     }