Unrolled build for rust-lang#135058

Rollup merge of rust-lang#135058 - onur-ozkan:path-resolution, r=jieyouxu

refactor bootstrap path resolution

Previously we removed paths as soon as we found the first intersection, which made it impossible to find other intersecting paths (and that is the reason of rust-lang#135022).

This patch changes that by marking the intersecting paths instead, so we can collect them all and remove them together when needed. Which means, `x build compiler` would compile anything that ends or starts with `"compiler"` instead of picking the first matching `Step` from `builder::get_step_descriptions`.

Fixes rust-lang#135022

Author: rust-timer

src/bootstrap/src/core/build_steps/compile.rs

@@ -93,7 +93,7 @@ impl Step for Std {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.crate_or_deps("sysroot").path("library")
+        run.crate_or_deps("sysroot").path("library").alias("core")
     }
 
     fn make_run(run: RunConfig<'_>) {

src/bootstrap/src/core/build_steps/doc.rs

@@ -574,7 +574,10 @@ impl Step for Std {
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         let builder = run.builder;
-        run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs)
+        run.crate_or_deps("sysroot")
+            .path("library")
+            .alias("core")
+            .default_condition(builder.config.docs)
     }
 
     fn make_run(run: RunConfig<'_>) {

src/bootstrap/src/core/builder/mod.rs

@@ -3,7 +3,7 @@ mod cargo;
 use std::any::{Any, type_name};
 use std::cell::{Cell, RefCell};
 use std::collections::BTreeSet;
-use std::fmt::{Debug, Write};
+use std::fmt::{self, Debug, Write};
 use std::hash::Hash;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
@@ -271,16 +271,17 @@ impl PathSet {
     /// This is used for `StepDescription::krate`, which passes all matching crates at once to
     /// `Step::make_run`, rather than calling it many times with a single crate.
     /// See `tests.rs` for examples.
-    fn intersection_removing_matches(&self, needles: &mut Vec<PathBuf>, module: Kind) -> PathSet {
+    fn intersection_removing_matches(&self, needles: &mut [CLIStepPath], module: Kind) -> PathSet {
         let mut check = |p| {
-            for (i, n) in needles.iter().enumerate() {
-                let matched = Self::check(p, n, module);
+            let mut result = false;
+            for n in needles.iter_mut() {
+                let matched = Self::check(p, &n.path, module);
                 if matched {
-                    needles.remove(i);
-                    return true;
+                    n.will_be_executed = true;
+                    result = true;
                 }
             }
-            false
+            result
         };
         match self {
             PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()),
@@ -361,6 +362,32 @@ fn remap_paths(paths: &mut Vec<PathBuf>) {
     paths.append(&mut add);
 }
 
+#[derive(Clone, PartialEq)]
+struct CLIStepPath {
+    path: PathBuf,
+    will_be_executed: bool,
+}
+
+#[cfg(test)]
+impl CLIStepPath {
+    fn will_be_executed(mut self, will_be_executed: bool) -> Self {
+        self.will_be_executed = will_be_executed;
+        self
+    }
+}
+
+impl Debug for CLIStepPath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.path.display())
+    }
+}
+
+impl From<PathBuf> for CLIStepPath {
+    fn from(path: PathBuf) -> Self {
+        Self { path, will_be_executed: false }
+    }
+}
+
 impl StepDescription {
     fn from<S: Step>(kind: Kind) -> StepDescription {
         StepDescription {
@@ -478,7 +505,8 @@ impl StepDescription {
             return;
         }
 
-        let mut path_lookup: Vec<(PathBuf, bool)> =
+        let mut paths: Vec<CLIStepPath> = paths.into_iter().map(|p| p.into()).collect();
+        let mut path_lookup: Vec<(CLIStepPath, bool)> =
             paths.clone().into_iter().map(|p| (p, false)).collect();
 
         // List of `(usize, &StepDescription, Vec<PathSet>)` where `usize` is the closest index of a path
@@ -518,8 +546,10 @@ impl StepDescription {
             }
         }
 
+        paths.retain(|p| !p.will_be_executed);
+
         if !paths.is_empty() {
-            eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths,);
+            eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths);
             eprintln!(
                 "HELP: run `x.py {} --help --verbose` to show a list of available paths",
                 builder.kind.as_str()
@@ -682,7 +712,7 @@ impl<'a> ShouldRun<'a> {
     /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?)
     fn pathset_for_paths_removing_matches(
         &self,
-        paths: &mut Vec<PathBuf>,
+        paths: &mut [CLIStepPath],
         kind: Kind,
     ) -> Vec<PathSet> {
         let mut sets = vec![];
@@ -825,12 +855,8 @@ impl<'a> Builder<'a> {
         match kind {
             Kind::Build => describe!(
                 compile::Std,
-                // FIXME(#135022): `compile::Assemble` **must** come before `compile::Rustc` after
-                // `PathSet` also permits prefix-matching, because `compile::Rustc` can consume the
-                // `"compiler"` path filter first, causing `compile::Assemble` to no longer run when
-                // the user writes `./x build compiler --stage 0`.
-                compile::Assemble,
                 compile::Rustc,
+                compile::Assemble,
                 compile::CodegenBackend,
                 compile::StartupObjects,
                 tool::BuildManifest,
@@ -929,14 +955,10 @@ impl<'a> Builder<'a> {
                 test::Rustdoc,
                 test::CoverageRunRustdoc,
                 test::Pretty,
-                test::Crate,
-                test::CrateLibrustc,
-                // The cranelift and gcc tests need to be listed after the
-                // compiler unit tests (CrateLibrustc) so that they don't
-                // hijack the whole `compiler` directory during path matching.
-                // <https://github.com/rust-lang/rust/pull/134919>
                 test::CodegenCranelift,
                 test::CodegenGCC,
+                test::Crate,
+                test::CrateLibrustc,
                 test::CrateRustdoc,
                 test::CrateRustdocJsonTypes,
                 test::CrateBootstrap,

src/bootstrap/src/core/builder/tests.rs

@@ -108,13 +108,37 @@ fn test_intersection() {
     };
     let library_set = set(&["library/core", "library/alloc", "library/std"]);
     let mut command_paths = vec![
-        PathBuf::from("library/core"),
-        PathBuf::from("library/alloc"),
-        PathBuf::from("library/stdarch"),
+        CLIStepPath::from(PathBuf::from("library/core")),
+        CLIStepPath::from(PathBuf::from("library/alloc")),
+        CLIStepPath::from(PathBuf::from("library/stdarch")),
     ];
     let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build);
     assert_eq!(subset, set(&["library/core", "library/alloc"]),);
-    assert_eq!(command_paths, vec![PathBuf::from("library/stdarch")]);
+    assert_eq!(command_paths, vec![
+        CLIStepPath::from(PathBuf::from("library/core")).will_be_executed(true),
+        CLIStepPath::from(PathBuf::from("library/alloc")).will_be_executed(true),
+        CLIStepPath::from(PathBuf::from("library/stdarch")).will_be_executed(false),
+    ]);
+}
+
+#[test]
+fn test_resolve_parent_and_subpaths() {
+    let set = |paths: &[&str]| {
+        PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect())
+    };
+
+    let mut command_paths = vec![
+        CLIStepPath::from(PathBuf::from("src/tools/miri")),
+        CLIStepPath::from(PathBuf::from("src/tools/miri/cargo-miri")),
+    ];
+
+    let library_set = set(&["src/tools/miri", "src/tools/miri/cargo-miri"]);
+    library_set.intersection_removing_matches(&mut command_paths, Kind::Build);
+
+    assert_eq!(command_paths, vec![
+        CLIStepPath::from(PathBuf::from("src/tools/miri")).will_be_executed(true),
+        CLIStepPath::from(PathBuf::from("src/tools/miri/cargo-miri")).will_be_executed(true),
+    ]);
 }
 
 #[test]