Skip to content

Commit

Permalink
Auto merge of #97911 - dtolnay:numcpu, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history
Revert "remove num_cpus dependency" in rustc and update cargo

Fixes #97549. This PR reverts #94524 and does a Cargo update to pull in rust-lang/cargo#10737.

Rust 1.61.0 has a regression in which it misidentifies the number of available CPUs in some environments, leading to enormously increased memory usage and failing builds. In between Rust 1.60 and 1.61 both rustc and cargo replaced some uses of `num_cpus` with `available_parallelism`, which eliminated support for cgroupv1, still apparently in common use. This PR switches both rustc and cargo back to using `num_cpus` in order to support environments where the available parallelism is controlled by cgroupv1. Both can use `available_parallism` again once it handles cgroupv1 (if ever).

I have confirmed that the rustc part of this PR fixes the memory usage regression in my non-Cargo environment, and others have confirmed in #97549 that the Cargo regression was at fault for the memory usage regression in their environments.
  • Loading branch information
bors committed Jun 9, 2022
2 parents d7b8d77 + fbc86e0 commit 420c970
Show file tree
Hide file tree
Showing 10 changed files with 13 additions and 8 deletions.
4 changes: 4 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ dependencies = [
"hex 0.4.2",
"ignore",
"libc",
"num_cpus",
"once_cell",
"opener",
"pretty_assertions 0.7.2",
Expand Down Expand Up @@ -250,6 +251,7 @@ dependencies = [
"anyhow",
"flate2",
"hex 0.4.2",
"num_cpus",
"rayon",
"serde",
"serde_json",
Expand Down Expand Up @@ -349,6 +351,7 @@ dependencies = [
"libgit2-sys",
"log",
"memchr",
"num_cpus",
"opener",
"openssl",
"os_info",
Expand Down Expand Up @@ -4419,6 +4422,7 @@ name = "rustc_session"
version = "0.0.0"
dependencies = [
"getopts",
"num_cpus",
"rustc_ast",
"rustc_data_structures",
"rustc_errors",
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_session/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,6 @@ rustc_serialize = { path = "../rustc_serialize" }
rustc_data_structures = { path = "../rustc_data_structures" }
rustc_span = { path = "../rustc_span" }
rustc_fs_util = { path = "../rustc_fs_util" }
num_cpus = "1.0"
rustc_ast = { path = "../rustc_ast" }
rustc_lint_defs = { path = "../rustc_lint_defs" }
2 changes: 1 addition & 1 deletion compiler/rustc_session/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@ mod parse {
pub(crate) fn parse_threads(slot: &mut usize, v: Option<&str>) -> bool {
match v.and_then(|s| s.parse().ok()) {
Some(0) => {
*slot = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get);
*slot = ::num_cpus::get();
true
}
Some(i) => {
Expand Down
1 change: 1 addition & 0 deletions src/bootstrap/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ test = false
[dependencies]
cmake = "0.1.38"
filetime = "0.2"
num_cpus = "1.0"
getopts = "0.2.19"
cc = "1.0.69"
libc = "0.2"
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1418,7 +1418,7 @@ fn set<T>(field: &mut T, val: Option<T>) {

fn threads_from_config(v: u32) -> u32 {
match v {
0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32,
0 => num_cpus::get() as u32,
n => n,
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/flags.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
let j_msg = format!(
"number of jobs to run in parallel; \
defaults to {} (this host's logical CPU count)",
std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
num_cpus::get()
);
opts.optopt("j", "jobs", &j_msg, "JOBS");
opts.optflag("h", "help", "print this help message");
Expand Down
4 changes: 1 addition & 3 deletions src/bootstrap/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1032,9 +1032,7 @@ impl Build {
/// Returns the number of parallel jobs that have been configured for this
/// build.
fn jobs(&self) -> u32 {
self.config.jobs.unwrap_or_else(|| {
std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32
})
self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32)
}

fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> {
Expand Down
1 change: 1 addition & 0 deletions src/tools/build-manifest/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ tar = "0.4.29"
sha2 = "0.10.1"
rayon = "1.5.1"
hex = "0.4.2"
num_cpus = "1.13.0"
2 changes: 1 addition & 1 deletion src/tools/build-manifest/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ fn main() {
let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
} else {
std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
num_cpus::get()
};
rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
Expand Down
2 changes: 1 addition & 1 deletion src/tools/cargo

0 comments on commit 420c970

Please sign in to comment.