Auto merge of #112270 - matthiaskrgr:rollup-umplsb6, r=matthiaskrgr

Rollup of 6 pull requests

Successful merges:

 - #109093 (add `#[doc(alias="flatmap")]` to `Option::and_then`)
 - #110701 (Fix `x test core` when download-rustc is enabled)
 - #111982 (Revert "Enable incremental independent of stage")
 - #112158 (Add portable-simd mention)
 - #112172 (doc: improve explanation)
 - #112178 (Fix bug where private item with intermediate doc hidden re-export was not inlined)

r? `@ghost`
`@rustbot` modify labels: rollup

library/core/src/option.rs

@@ -1402,6 +1402,7 @@ impl<T> Option<T> {
     /// let item_2_0 = arr_2d.get(2).and_then(|row| row.get(0));
     /// assert_eq!(item_2_0, None);
     /// ```
+    #[doc(alias = "flatmap")]
     #[inline]
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn and_then<U, F>(self, f: F) -> Option<U>

library/std/src/sync/once.rs

@@ -91,7 +91,7 @@ impl Once {
     /// return).
     ///
     /// If the given closure recursively invokes `call_once` on the same [`Once`]
-    /// instance the exact behavior is not specified, allowed outcomes are
+    /// instance, the exact behavior is not specified: allowed outcomes are
     /// a panic or a deadlock.
     ///
     /// # Examples

src/bootstrap/builder.rs

@@ -995,7 +995,7 @@ impl<'a> Builder<'a> {
     }
 
     pub fn sysroot(&self, compiler: Compiler) -> Interned<PathBuf> {
-        self.ensure(compile::Sysroot { compiler })
+        self.ensure(compile::Sysroot::new(compiler))
     }
 
     /// Returns the libdir where the standard library and other artifacts are
@@ -1786,7 +1786,10 @@ impl<'a> Builder<'a> {
             cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1");
         }
 
-        if self.config.incremental {
+        // Ignore incremental modes except for stage0, since we're
+        // not guaranteeing correctness across builds if the compiler
+        // is changing under your feet.
+        if self.config.incremental && compiler.stage == 0 {
             cargo.env("CARGO_INCREMENTAL", "1");
         } else {
             // Don't rely on any default setting for incr. comp. in Cargo

src/bootstrap/compile.rs

@@ -40,11 +40,18 @@ pub struct Std {
     ///
     /// This shouldn't be used from other steps; see the comment on [`Rustc`].
     crates: Interned<Vec<String>>,
+    /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself,
+    /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps.
+    force_recompile: bool,
 }
 
 impl Std {
     pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
-        Self { target, compiler, crates: Default::default() }
+        Self { target, compiler, crates: Default::default(), force_recompile: false }
+    }
+
+    pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self {
+        Self { target, compiler, crates: Default::default(), force_recompile: true }
     }
 }
 
@@ -77,6 +84,7 @@ impl Step for Std {
             compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
             target: run.target,
             crates: make_run_crates(&run, "library"),
+            force_recompile: false,
         });
     }
 
@@ -89,11 +97,20 @@ impl Step for Std {
         let target = self.target;
         let compiler = self.compiler;
 
-        // When using `download-rustc`, we already have artifacts for the host available
-        // (they were copied in `impl Step for Sysroot`). Don't recompile them.
-        // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
-        // so its artifacts can't be reused.
-        if builder.download_rustc() && compiler.stage != 0 && target == builder.build.build {
+        // When using `download-rustc`, we already have artifacts for the host available. Don't
+        // recompile them.
+        if builder.download_rustc() && target == builder.build.build
+            // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so
+            // its artifacts can't be reused.
+            && compiler.stage != 0
+            // This check is specific to testing std itself; see `test::Std` for more details.
+            && !self.force_recompile
+        {
+            cp_rustc_component_to_ci_sysroot(
+                builder,
+                compiler,
+                builder.config.ci_rust_std_contents(),
+            );
             return;
         }
 
@@ -428,6 +445,8 @@ struct StdLink {
     pub target: TargetSelection,
     /// Not actually used; only present to make sure the cache invalidation is correct.
     crates: Interned<Vec<String>>,
+    /// See [`Std::force_recompile`].
+    force_recompile: bool,
 }
 
 impl StdLink {
@@ -437,6 +456,7 @@ impl StdLink {
             target_compiler: std.compiler,
             target: std.target,
             crates: std.crates,
+            force_recompile: std.force_recompile,
         }
     }
 }
@@ -460,8 +480,24 @@ impl Step for StdLink {
         let compiler = self.compiler;
         let target_compiler = self.target_compiler;
         let target = self.target;
-        let libdir = builder.sysroot_libdir(target_compiler, target);
-        let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
+
+        // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`.
+        let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() {
+            // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too
+            let lib = builder.sysroot_libdir_relative(self.compiler);
+            let sysroot = builder.ensure(crate::compile::Sysroot {
+                compiler: self.compiler,
+                force_recompile: self.force_recompile,
+            });
+            let libdir = sysroot.join(lib).join("rustlib").join(target.triple).join("lib");
+            let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host.triple).join("lib");
+            (INTERNER.intern_path(libdir), INTERNER.intern_path(hostdir))
+        } else {
+            let libdir = builder.sysroot_libdir(target_compiler, target);
+            let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
+            (libdir, hostdir)
+        };
+
         add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
     }
 }
@@ -594,6 +630,25 @@ impl Step for StartupObjects {
     }
 }
 
+fn cp_rustc_component_to_ci_sysroot(
+    builder: &Builder<'_>,
+    compiler: Compiler,
+    contents: Vec<String>,
+) {
+    let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false });
+
+    let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc");
+    for file in contents {
+        let src = ci_rustc_dir.join(&file);
+        let dst = sysroot.join(file);
+        if src.is_dir() {
+            t!(fs::create_dir_all(dst));
+        } else {
+            builder.copy(&src, &dst);
+        }
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Rustc {
     pub target: TargetSelection,
@@ -653,18 +708,11 @@ impl Step for Rustc {
         if builder.download_rustc() && compiler.stage != 0 {
             // Copy the existing artifacts instead of rebuilding them.
             // NOTE: this path is only taken for tools linking to rustc-dev (including ui-fulldeps tests).
-            let sysroot = builder.ensure(Sysroot { compiler });
-
-            let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc");
-            for file in builder.config.rustc_dev_contents() {
-                let src = ci_rustc_dir.join(&file);
-                let dst = sysroot.join(file);
-                if src.is_dir() {
-                    t!(fs::create_dir_all(dst));
-                } else {
-                    builder.copy(&src, &dst);
-                }
-            }
+            cp_rustc_component_to_ci_sysroot(
+                builder,
+                compiler,
+                builder.config.ci_rustc_dev_contents(),
+            );
             return;
         }
 
@@ -1225,6 +1273,14 @@ pub fn compiler_file(
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Sysroot {
     pub compiler: Compiler,
+    /// See [`Std::force_recompile`].
+    force_recompile: bool,
+}
+
+impl Sysroot {
+    pub(crate) fn new(compiler: Compiler) -> Self {
+        Sysroot { compiler, force_recompile: false }
+    }
 }
 
 impl Step for Sysroot {
@@ -1247,6 +1303,8 @@ impl Step for Sysroot {
         let sysroot_dir = |stage| {
             if stage == 0 {
                 host_dir.join("stage0-sysroot")
+            } else if self.force_recompile && stage == compiler.stage {
+                host_dir.join(format!("stage{stage}-test-sysroot"))
             } else if builder.download_rustc() && compiler.stage != builder.top_stage {
                 host_dir.join("ci-rustc-sysroot")
             } else {
@@ -1286,14 +1344,19 @@ impl Step for Sysroot {
             // 2. The sysroot is deleted and recreated between each invocation, so running `x test
             //    ui-fulldeps && x test ui` can't cause failures.
             let mut filtered_files = Vec::new();
-            // Don't trim directories or files that aren't loaded per-target; they can't cause conflicts.
-            let suffix = format!("lib/rustlib/{}/lib", compiler.host);
-            for path in builder.config.rustc_dev_contents() {
-                let path = Path::new(&path);
-                if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) {
-                    filtered_files.push(path.file_name().unwrap().to_owned());
+            let mut add_filtered_files = |suffix, contents| {
+                for path in contents {
+                    let path = Path::new(&path);
+                    if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) {
+                        filtered_files.push(path.file_name().unwrap().to_owned());
+                    }
                 }
-            }
+            };
+            let suffix = format!("lib/rustlib/{}/lib", compiler.host);
+            add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents());
+            // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the
+            // newly compiled std, not the downloaded std.
+            add_filtered_files("lib", builder.config.ci_rust_std_contents());
 
             let filtered_extensions = [OsStr::new("rmeta"), OsStr::new("rlib"), OsStr::new("so")];
             let ci_rustc_dir = builder.ci_rustc_dir(builder.config.build);
@@ -1411,7 +1474,8 @@ impl Step for Assemble {
 
         // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
         if builder.download_rustc() {
-            let sysroot = builder.ensure(Sysroot { compiler: target_compiler });
+            let sysroot =
+                builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false });
             // Ensure that `libLLVM.so` ends up in the newly created target directory,
             // so that tools using `rustc_private` can use it.
             dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot);

src/bootstrap/download.rs

@@ -271,11 +271,8 @@ impl Config {
         // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding
         // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow.
         // Cache the entries when we extract it so we only have to read it once.
-        let mut recorded_entries = if dst.ends_with("ci-rustc") && pattern == "rustc-dev" {
-            Some(BufWriter::new(t!(File::create(dst.join(".rustc-dev-contents")))))
-        } else {
-            None
-        };
+        let mut recorded_entries =
+            if dst.ends_with("ci-rustc") { recorded_entries(dst, pattern) } else { None };
 
         for member in t!(tar.entries()) {
             let mut member = t!(member);
@@ -332,6 +329,17 @@ impl Config {
     }
 }
 
+fn recorded_entries(dst: &Path, pattern: &str) -> Option<BufWriter<File>> {
+    let name = if pattern == "rustc-dev" {
+        ".rustc-dev-contents"
+    } else if pattern.starts_with("rust-std") {
+        ".rust-std-contents"
+    } else {
+        return None;
+    };
+    Some(BufWriter::new(t!(File::create(dst.join(name)))))
+}
+
 enum DownloadSource {
     CI,
     Dist,
@@ -382,11 +390,20 @@ impl Config {
         Some(rustfmt_path)
     }
 
-    pub(crate) fn rustc_dev_contents(&self) -> Vec<String> {
+    pub(crate) fn ci_rust_std_contents(&self) -> Vec<String> {
+        self.ci_component_contents(".rust-std-contents")
+    }
+
+    pub(crate) fn ci_rustc_dev_contents(&self) -> Vec<String> {
+        self.ci_component_contents(".rustc-dev-contents")
+    }
+
+    fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
         assert!(self.download_rustc());
         let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc");
-        let rustc_dev_contents_file = t!(File::open(ci_rustc_dir.join(".rustc-dev-contents")));
-        t!(BufReader::new(rustc_dev_contents_file).lines().collect())
+        let stamp_file = ci_rustc_dir.join(stamp_file);
+        let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
+        t!(BufReader::new(contents_file).lines().collect())
     }
 
     pub(crate) fn download_ci_rustc(&self, commit: &str) {

src/bootstrap/test.rs

@@ -2204,7 +2204,8 @@ impl Step for Crate {
         let target = self.target;
         let mode = self.mode;
 
-        builder.ensure(compile::Std::new(compiler, target));
+        // See [field@compile::Std::force_recompile].
+        builder.ensure(compile::Std::force_recompile(compiler, target));
         builder.ensure(RemoteCopyLibs { compiler, target });
 
         // If we're not doing a full bootstrap but we're testing a stage2
@@ -2218,6 +2219,16 @@ impl Step for Crate {
         match mode {
             Mode::Std => {
                 compile::std_cargo(builder, target, compiler.stage, &mut cargo);
+                // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`,
+                // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`.
+                // Override it.
+                if builder.download_rustc() {
+                    let sysroot = builder
+                        .out
+                        .join(compiler.host.triple)
+                        .join(format!("stage{}-test-sysroot", compiler.stage));
+                    cargo.env("RUSTC_SYSROOT", sysroot);
+                }
             }
             Mode::Rustc => {
                 compile::rustc_cargo(builder, &mut cargo, target, compiler.stage);
@@ -2269,6 +2280,11 @@ impl Step for CrateRustdoc {
             // isn't really necessary.
             builder.compiler_for(builder.top_stage, target, target)
         };
+        // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when
+        // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from
+        // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this
+        // explicitly to make sure it ends up in the stage2 sysroot.
+        builder.ensure(compile::Std::new(compiler, target));
         builder.ensure(compile::Rustc::new(compiler, target));
 
         let mut cargo = tool::prepare_tool_cargo(
@@ -2320,7 +2336,13 @@ impl Step for CrateRustdoc {
         dylib_path.insert(0, PathBuf::from(&*libdir));
         cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-        let _guard = builder.msg(builder.kind, compiler.stage, "rustdoc", compiler.host, target);
+        let _guard = builder.msg_sysroot_tool(
+            builder.kind,
+            compiler.stage,
+            "rustdoc",
+            compiler.host,
+            target,
+        );
         run_cargo_test(
             cargo,
             &[],

src/librustdoc/clean/mod.rs

@@ -2180,7 +2180,8 @@ fn get_all_import_attributes<'hir>(
             // This is the "original" reexport so we get all its attributes without filtering them.
             attrs = import_attrs.iter().map(|attr| (Cow::Borrowed(attr), Some(def_id))).collect();
             first = false;
-        } else {
+        // We don't add attributes of an intermediate re-export if it has `#[doc(hidden)]`.
+        } else if !cx.tcx.is_doc_hidden(def_id) {
             add_without_unwanted_attributes(&mut attrs, import_attrs, is_inline, Some(def_id));
         }
     }

src/librustdoc/visit_ast.rs

@@ -246,7 +246,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
         glob: bool,
         please_inline: bool,
     ) -> bool {
-        debug!("maybe_inline_local res: {:?}", res);
+        debug!("maybe_inline_local (renamed: {renamed:?}) res: {res:?}");
 
         if renamed == Some(kw::Underscore) {
             // We never inline `_` reexports.
@@ -308,6 +308,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> {
                     .cache
                     .effective_visibilities
                     .is_directly_public(tcx, item_def_id.to_def_id()) &&
+                !tcx.is_doc_hidden(item_def_id) &&
                 !inherits_doc_hidden(tcx, item_def_id, None)
             {
                 // The imported item is public and not `doc(hidden)` so no need to inline it.