rustc: Link LLVM directly into rustc again

This commit builds on rust-lang#65501 continue to simplify the build system and
compiler now that we no longer have multiple LLVM backends to ship by
default. Here this switches the compiler back to what it once was long
long ago, which is linking LLVM directly to the compiler rather than
dynamically loading it at runtime. The `codegen-backends` directory of
the sysroot no longer exists and all relevant support in the build
system is removed. Note that `rustc` still supports a dynamically loaded
codegen backend as it did previously, it just no longer supports
dynamically loaded codegen backends in its own sysroot.

Additionally as part of this the `librustc_codegen_llvm` crate now once
again explicitly depends on all of its crates instead of implicitly
loading them through the sysroot. This involved filling out its
`Cargo.toml` and deleting all the now-unnecessary `extern crate`
annotations in the header of the crate. (this in turn required adding a
number of imports for names of macros too).

The end results of this change are:

* Rustbuild's build process for the compiler as all the "oh don't forget
  the codegen backend" checks can be easily removed.
* Building `rustc_codegen_llvm` is much simpler since it's simply
  another compiler crate.
* Managing the dependencies of `rustc_codegen_llvm` is much simpler since
  it's "just another `Cargo.toml` to edit"
* The build process should be a smidge faster because there's more
  parallelism in the main rustc build step rather than splitting
  `librustc_codegen_llvm` out to its own step.
* The compiler is expected to be slightly faster by default because the
  codegen backend does not need to be dynamically loaded.
* Disabling LLVM as part of rustbuild is still supported, supporting
  multiple codegen backends is still supported, and dynamic loading of a
  codegen backend is still supported.

Author: alexcrichton

Cargo.lock

@@ -3435,7 +3435,25 @@ dependencies = [
 name = "rustc_codegen_llvm"
 version = "0.0.0"
 dependencies = [
+ "bitflags",
+ "flate2",
+ "libc",
+ "log",
+ "rustc",
+ "rustc-demangle",
+ "rustc_codegen_ssa",
+ "rustc_codegen_utils",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_fs_util",
+ "rustc_incremental",
+ "rustc_index",
  "rustc_llvm",
+ "rustc_target",
+ "smallvec 0.6.10",
+ "syntax",
+ "syntax_expand",
+ "syntax_pos",
 ]
 
 [[package]]
@@ -3596,7 +3614,12 @@ dependencies = [
  "log",
  "once_cell",
  "rustc",
+<<<<<<< HEAD
  "rustc-rayon",
+=======
+ "rustc-rayon 0.3.0",
+ "rustc_codegen_llvm",
+>>>>>>> rustc: Link LLVM directly into rustc again
  "rustc_codegen_ssa",
  "rustc_codegen_utils",
  "rustc_data_structures",

config.toml.example

@@ -379,9 +379,6 @@
 # and currently the only standard option supported is `"llvm"`
 #codegen-backends = ["llvm"]
 
-# This is the name of the directory in which codegen backends will get installed
-#codegen-backends-dir = "codegen-backends"
-
 # Indicates whether LLD will be compiled and made available in the sysroot for
 # rustc to execute.
 #lld = false

src/bootstrap/builder.rs

@@ -339,7 +339,6 @@ impl<'a> Builder<'a> {
             Kind::Build => describe!(
                 compile::Std,
                 compile::Rustc,
-                compile::CodegenBackend,
                 compile::StartupObjects,
                 tool::BuildManifest,
                 tool::Rustbook,
@@ -364,7 +363,6 @@ impl<'a> Builder<'a> {
             Kind::Check | Kind::Clippy | Kind::Fix => describe!(
                 check::Std,
                 check::Rustc,
-                check::CodegenBackend,
                 check::Rustdoc
             ),
             Kind::Test => describe!(
@@ -632,11 +630,6 @@ impl<'a> Builder<'a> {
         self.ensure(Libdir { compiler, target })
     }
 
-    pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf {
-        self.sysroot_libdir(compiler, compiler.host)
-            .with_file_name(self.config.rust_codegen_backends_dir.clone())
-    }
-
     /// Returns the compiler's libdir where it stores the dynamic libraries that
     /// it itself links against.
     ///
@@ -707,15 +700,6 @@ impl<'a> Builder<'a> {
         }
     }
 
-    /// Gets the paths to all of the compiler's codegen backends.
-    fn codegen_backends(&self, compiler: Compiler) -> impl Iterator<Item = PathBuf> {
-        fs::read_dir(self.sysroot_codegen_backends(compiler))
-            .into_iter()
-            .flatten()
-            .filter_map(Result::ok)
-            .map(|entry| entry.path())
-    }
-
     pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
         self.ensure(tool::Rustdoc { compiler })
     }
@@ -759,12 +743,6 @@ impl<'a> Builder<'a> {
         let mut cargo = Command::new(&self.initial_cargo);
         let out_dir = self.stage_out(compiler, mode);
 
-        // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
-        // so we need to explicitly clear out if they've been updated.
-        for backend in self.codegen_backends(compiler) {
-            self.clear_if_dirty(&out_dir, &backend);
-        }
-
         if cmd == "doc" || cmd == "rustdoc" {
             let my_out = match mode {
                 // This is the intended out directory for compiler documentation.

src/bootstrap/check.rs

@@ -1,11 +1,10 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
-use crate::compile::{run_cargo, std_cargo, rustc_cargo, rustc_cargo_env,
-                     add_to_sysroot};
+use crate::compile::{run_cargo, std_cargo, rustc_cargo, add_to_sysroot};
 use crate::builder::{RunConfig, Builder, Kind, ShouldRun, Step};
 use crate::tool::{prepare_tool_cargo, SourceType};
 use crate::{Compiler, Mode};
-use crate::cache::{INTERNER, Interned};
+use crate::cache::Interned;
 use std::path::PathBuf;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -97,7 +96,7 @@ impl Step for Rustc {
 
         let mut cargo = builder.cargo(compiler, Mode::Rustc, target,
             cargo_subcommand(builder.kind));
-        rustc_cargo(builder, &mut cargo);
+        rustc_cargo(builder, &mut cargo, target);
 
         builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
         run_cargo(builder,
@@ -113,55 +112,6 @@ impl Step for Rustc {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
-pub struct CodegenBackend {
-    pub target: Interned<String>,
-    pub backend: Interned<String>,
-}
-
-impl Step for CodegenBackend {
-    type Output = ();
-    const ONLY_HOSTS: bool = true;
-    const DEFAULT: bool = true;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.all_krates("rustc_codegen_llvm")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        let backend = run.builder.config.rust_codegen_backends.get(0);
-        let backend = backend.cloned().unwrap_or_else(|| {
-            INTERNER.intern_str("llvm")
-        });
-        run.builder.ensure(CodegenBackend {
-            target: run.target,
-            backend,
-        });
-    }
-
-    fn run(self, builder: &Builder<'_>) {
-        let compiler = builder.compiler(0, builder.config.build);
-        let target = self.target;
-        let backend = self.backend;
-
-        builder.ensure(Rustc { target });
-
-        let mut cargo = builder.cargo(compiler, Mode::Codegen, target,
-            cargo_subcommand(builder.kind));
-        cargo.arg("--manifest-path").arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
-        rustc_cargo_env(builder, &mut cargo);
-
-        // We won't build LLVM if it's not available, as it shouldn't affect `check`.
-
-        run_cargo(builder,
-                  cargo,
-                  args(builder.kind),
-                  &codegen_backend_stamp(builder, compiler, target, backend),
-                  vec![],
-                  true);
-    }
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Rustdoc {
     pub target: Interned<String>,
@@ -231,16 +181,6 @@ pub fn librustc_stamp(
     builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp")
 }
 
-/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular
-/// compiler for the specified target and backend.
-fn codegen_backend_stamp(builder: &Builder<'_>,
-                         compiler: Compiler,
-                         target: Interned<String>,
-                         backend: Interned<String>) -> PathBuf {
-    builder.cargo_out(compiler, Mode::Codegen, target)
-         .join(format!(".librustc_codegen_llvm-{}-check.stamp", backend))
-}
-
 /// Cargo's output path for rustdoc in a given stage, compiled by a particular
 /// compiler for the specified target.
 pub fn rustdoc_stamp(