Auto merge of #122362 - Zoxc:rustc_driver_static_std, r=oli-obk,lqd,b…

…jorn3,Kobzol

Link `std` statically in `rustc_driver`

This makes `rustc_driver` statically link to `std`. This is done by not passing `-C prefer-dynamic` when building `rustc_driver`. However building `rustc-main` won't work currently as it tries to dynamically link to both `rustc_driver` and `std` resulting in a crate graph with `std` duplicated. To fix that new command line option `-Z prefer_deps_of_dynamic` is added which prevents linking to a dylib if there's a static variant of it already statically linked into another dylib dependency.

The main motivation for this change is to enable `#[global_allocator]` to be used in `rustc_driver` allowing overriding the allocator used in rustc on all platforms.

---

Instead of adding `-Z prefer_deps_of_dynamic`, this PR is changed to crate opt-in to the linking change via the `rustc_private` feature instead, as that would be typically needed to link to `rustc_driver` anyway.

---

try-job: aarch64-apple
try-job: x86_64-msvc
try-job: i686-mingw
try-job: dist-x86_64-msvc
try-job: aarch64-gnu

compiler/rustc/src/main.rs

@@ -1,3 +1,6 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to `rustc_driver`.
+#![feature(rustc_private)]
+
 // A note about jemalloc: rustc uses jemalloc when built for CI and
 // distribution. The obvious way to do this is with the `#[global_allocator]`
 // mechanism. However, for complicated reasons (see

compiler/rustc_metadata/messages.ftl

@@ -41,6 +41,9 @@ metadata_crate_dep_multiple =
 metadata_crate_dep_not_static =
     `{$crate_name}` was unavailable as a static crate, preventing fully static linking
 
+metadata_crate_dep_rustc_driver =
+    `feature(rustc_private)` is needed to link to the compiler's `rustc_driver` library
+
 metadata_crate_location_unknown_type =
     extern location for {$crate_name} is of an unknown type: {$path}
 

compiler/rustc_metadata/src/dependency_format.rs

@@ -51,20 +51,22 @@
 //! Additionally, the algorithm is geared towards finding *any* solution rather
 //! than finding a number of solutions (there are normally quite a few).
 
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::CrateNum;
 use rustc_middle::bug;
 use rustc_middle::middle::dependency_format::{Dependencies, DependencyList, Linkage};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::CrateType;
 use rustc_session::cstore::CrateDepKind;
 use rustc_session::cstore::LinkagePreference::{self, RequireDynamic, RequireStatic};
+use rustc_span::sym;
 use tracing::info;
 
 use crate::creader::CStore;
 use crate::errors::{
     BadPanicStrategy, CrateDepMultiple, IncompatiblePanicInDropStrategy, LibRequired,
-    NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcLibRequired, TwoPanicRuntimes,
+    NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcDriverHelp, RustcLibRequired,
+    TwoPanicRuntimes,
 };
 
 pub(crate) fn calculate(tcx: TyCtxt<'_>) -> Dependencies {
@@ -160,25 +162,49 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         Linkage::Dynamic | Linkage::IncludedFromDylib => {}
     }
 
+    let all_dylibs = || {
+        tcx.crates(()).iter().filter(|&&cnum| {
+            !tcx.dep_kind(cnum).macros_only() && tcx.used_crate_source(cnum).dylib.is_some()
+        })
+    };
+
+    let mut upstream_in_dylibs = FxHashSet::default();
+
+    if tcx.features().rustc_private {
+        // We need this to prevent users of `rustc_driver` from linking dynamically to `std`
+        // which does not work as `std` is also statically linked into `rustc_driver`.
+
+        // Find all libraries statically linked to upstream dylibs.
+        for &cnum in all_dylibs() {
+            let deps = tcx.dylib_dependency_formats(cnum);
+            for &(depnum, style) in deps.iter() {
+                if let RequireStatic = style {
+                    upstream_in_dylibs.insert(depnum);
+                }
+            }
+        }
+    }
+
     let mut formats = FxHashMap::default();
 
     // Sweep all crates for found dylibs. Add all dylibs, as well as their
     // dependencies, ensuring there are no conflicts. The only valid case for a
     // dependency to be relied upon twice is for both cases to rely on a dylib.
-    for &cnum in tcx.crates(()).iter() {
-        if tcx.dep_kind(cnum).macros_only() {
+    for &cnum in all_dylibs() {
+        if upstream_in_dylibs.contains(&cnum) {
+            info!("skipping dylib: {}", tcx.crate_name(cnum));
+            // If this dylib is also available statically linked to another dylib
+            // we try to use that instead.
             continue;
         }
+
         let name = tcx.crate_name(cnum);
-        let src = tcx.used_crate_source(cnum);
-        if src.dylib.is_some() {
-            info!("adding dylib: {}", name);
-            add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static);
-            let deps = tcx.dylib_dependency_formats(cnum);
-            for &(depnum, style) in deps.iter() {
-                info!("adding {:?}: {}", style, tcx.crate_name(depnum));
-                add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static);
-            }
+        info!("adding dylib: {}", name);
+        add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static);
+        let deps = tcx.dylib_dependency_formats(cnum);
+        for &(depnum, style) in deps.iter() {
+            info!("adding {:?}: {}", style, tcx.crate_name(depnum));
+            add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static);
         }
     }
 
@@ -268,12 +294,15 @@ fn add_library(
             // This error is probably a little obscure, but I imagine that it
             // can be refined over time.
             if link2 != link || link == RequireStatic {
+                let linking_to_rustc_driver = tcx.sess.psess.unstable_features.is_nightly_build()
+                    && tcx.crates(()).iter().any(|&cnum| tcx.crate_name(cnum) == sym::rustc_driver);
                 tcx.dcx().emit_err(CrateDepMultiple {
                     crate_name: tcx.crate_name(cnum),
                     non_static_deps: unavailable_as_static
                         .drain(..)
                         .map(|cnum| NonStaticCrateDep { crate_name: tcx.crate_name(cnum) })
                         .collect(),
+                    rustc_driver_help: linking_to_rustc_driver.then_some(RustcDriverHelp),
                 });
             }
         }

compiler/rustc_metadata/src/errors.rs

@@ -38,6 +38,8 @@ pub struct CrateDepMultiple {
     pub crate_name: Symbol,
     #[subdiagnostic]
     pub non_static_deps: Vec<NonStaticCrateDep>,
+    #[subdiagnostic]
+    pub rustc_driver_help: Option<RustcDriverHelp>,
 }
 
 #[derive(Subdiagnostic)]
@@ -46,6 +48,10 @@ pub struct NonStaticCrateDep {
     pub crate_name: Symbol,
 }
 
+#[derive(Subdiagnostic)]
+#[help(metadata_crate_dep_rustc_driver)]
+pub struct RustcDriverHelp;
+
 #[derive(Diagnostic)]
 #[diag(metadata_two_panic_runtimes)]
 pub struct TwoPanicRuntimes {

compiler/rustc_span/src/symbol.rs

@@ -1619,6 +1619,7 @@ symbols! {
         rustc_dirty,
         rustc_do_not_const_check,
         rustc_doc_primitive,
+        rustc_driver,
         rustc_dummy,
         rustc_dump_def_parents,
         rustc_dump_item_bounds,

src/bootstrap/src/bin/rustc.rs

@@ -89,6 +89,25 @@ fn main() {
         rustc_real
     };
 
+    // Get the name of the crate we're compiling, if any.
+    let crate_name = parse_value_from_args(&orig_args, "--crate-name");
+
+    // When statically linking `std` into `rustc_driver`, remove `-C prefer-dynamic`
+    if env::var("RUSTC_LINK_STD_INTO_RUSTC_DRIVER").unwrap() == "1"
+        && crate_name == Some("rustc_driver")
+        && stage != "0"
+    {
+        if let Some(pos) = args.iter().enumerate().position(|(i, a)| {
+            a == "-C" && args.get(i + 1).map(|a| a == "prefer-dynamic").unwrap_or(false)
+        }) {
+            args.remove(pos);
+            args.remove(pos);
+        }
+        if let Some(pos) = args.iter().position(|a| a == "-Cprefer-dynamic") {
+            args.remove(pos);
+        }
+    }
+
     let mut cmd = match env::var_os("RUSTC_WRAPPER_REAL") {
         Some(wrapper) if !wrapper.is_empty() => {
             let mut cmd = Command::new(wrapper);
@@ -99,9 +118,6 @@ fn main() {
     };
     cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-    // Get the name of the crate we're compiling, if any.
-    let crate_name = parse_value_from_args(&orig_args, "--crate-name");
-
     if let Some(crate_name) = crate_name {
         if let Some(target) = env::var_os("RUSTC_TIME") {
             if target == "all"

src/bootstrap/src/core/builder.rs

@@ -1106,6 +1106,12 @@ impl<'a> Builder<'a> {
         StepDescription::run(v, self, paths);
     }
 
+    /// Returns if `std` should be statically linked into `rustc_driver`.
+    /// It's currently not done on `windows-gnu` due to linker bugs.
+    pub fn link_std_into_rustc_driver(&self, target: TargetSelection) -> bool {
+        !target.triple.ends_with("-windows-gnu")
+    }
+
     /// Obtain a compiler at a given stage and for a given host (i.e., this is the target that the
     /// compiler will run on, *not* the target it will build code for). Explicitly does not take
     /// `Compiler` since all `Compiler` instances are meant to be obtained through this function,
@@ -2162,10 +2168,18 @@ impl<'a> Builder<'a> {
         // When we build Rust dylibs they're all intended for intermediate
         // usage, so make sure we pass the -Cprefer-dynamic flag instead of
         // linking all deps statically into the dylib.
-        if matches!(mode, Mode::Std | Mode::Rustc) {
+        if matches!(mode, Mode::Std) {
+            rustflags.arg("-Cprefer-dynamic");
+        }
+        if matches!(mode, Mode::Rustc) && !self.link_std_into_rustc_driver(target) {
             rustflags.arg("-Cprefer-dynamic");
         }
 
+        cargo.env(
+            "RUSTC_LINK_STD_INTO_RUSTC_DRIVER",
+            if self.link_std_into_rustc_driver(target) { "1" } else { "0" },
+        );
+
         // When building incrementally we default to a lower ThinLTO import limit
         // (unless explicitly specified otherwise). This will produce a somewhat
         // slower code but give way better compile times.

src/doc/unstable-book/src/language-features/rustc-private.md

@@ -0,0 +1,11 @@
+# `rustc_private`
+
+The tracking issue for this feature is: [#27812]
+
+[#27812]: https://github.com/rust-lang/rust/issues/27812
+
+------------------------
+
+This feature allows access to unstable internal compiler crates.
+
+Additionally it changes the linking behavior of crates which have this feature enabled. It will prevent linking to a dylib if there's a static variant of it already statically linked into another dylib dependency. This is required to successfully link to `rustc_driver`.

src/tools/clippy/src/main.rs

@@ -1,3 +1,6 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to
+// `rustc_driver`.
+#![feature(rustc_private)]
 // warn on lints, that are included in `rust-lang/rust`s bootstrap
 #![warn(rust_2018_idioms, unused_lifetimes)]
 

src/tools/clippy/tests/compile-test.rs

@@ -1,3 +1,6 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to
+// `rustc_driver`.
+#![feature(rustc_private)]
 #![warn(rust_2018_idioms, unused_lifetimes)]
 #![allow(unused_extern_crates)]
 

src/tools/rustdoc/main.rs

@@ -1,3 +1,6 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to `rustc_driver`.
+#![feature(rustc_private)]
+
 fn main() {
     rustdoc::main()
 }

src/tools/rustfmt/src/git-rustfmt/main.rs

@@ -1,3 +1,7 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to
+// `rustc_driver`.
+#![feature(rustc_private)]
+
 #[macro_use]
 extern crate tracing;