Auto merge of #53673 - michaelwoerister:incr-thinlto-2000, r=alexcric…

…hton Enable ThinLTO with incremental compilation. This is an updated version of #52309. This PR allows `rustc` to use (local) ThinLTO and incremental compilation at the same time. In theory this should allow for getting compile-time improvements for small changes while keeping the runtime performance of the generated code roughly the same as when compiling non-incrementally. The difference to #52309 is that this version also caches the pre-LTO version of LLVM bitcode. This allows for another layer of caching: 1. if the module itself has changed, we have to re-codegen and re-optimize. 2. if the module itself has not changed, but a module it imported from during ThinLTO has, we don't need to re-codegen and don't need to re-run the first optimization phase. Only the second (i.e. ThinLTO-) optimization phase is re-run. 3. if neither the module itself nor any of its imports have changed then we can re-use the final, post-ThinLTO version of the module. (We might have to load its pre-ThinLTO version though so it's available for other modules to import from)
rust-lang · Sep 3, 2018 · ee73f80 · ee73f80
2 parents 591a17d + 21d05f6
commit ee73f80
Show file tree

Hide file tree

Showing 17 changed files with 647 additions and 297 deletions.
diff --git a/src/Cargo.lock b/src/Cargo.lock
@@ -1198,6 +1198,15 @@ dependencies = [
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "memmap"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.2.1"
@@ -2029,6 +2038,7 @@ name = "rustc_codegen_llvm"
 version = "0.0.0"
 dependencies = [
  "cc 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_llvm 0.0.0",
@@ -3151,6 +3161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 "checksum mdbook 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "90b5a8d7e341ceee5db3882a06078d42661ddcfa2b3687319cc5da76ec4e782f"
 "checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d"
+"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
 "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
 "checksum minifier 0.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "9908ed7c62f990c21ab41fdca53a864a3ada0da69d8729c4de727b397e27bc11"
 "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4"

diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs
@@ -878,7 +878,7 @@ pub struct WorkProduct {
  pub saved_files: Vec<(WorkProductFileKind, String)>,
 }
 
-#[derive(Clone, Copy, Debug, RustcEncodable, RustcDecodable)]
+#[derive(Clone, Copy, Debug, RustcEncodable, RustcDecodable, PartialEq)]
 pub enum WorkProductFileKind {
  Object,
  Bytecode,

diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs
@@ -68,7 +68,7 @@ pub enum OptLevel {
  SizeMin, // -Oz
 }
 
-#[derive(Clone, Copy, PartialEq, Hash)]
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
 pub enum Lto {
  /// Don't do any LTO whatsoever
  No,

diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs
@@ -580,11 +580,6 @@ impl Session {
  return config::Lto::No;
  }
 
- // Right now ThinLTO isn't compatible with incremental compilation.
- if self.opts.incremental.is_some() {
- return config::Lto::No;
- }
-
  // Now we're in "defaults" territory. By default we enable ThinLTO for
  // optimized compiles (anything greater than O0).
  match self.opts.optimize {
@@ -1177,8 +1172,18 @@ pub fn build_session_(
 // commandline argument, you can do so here.
 fn validate_commandline_args_with_session_available(sess: &Session) {
 
- if sess.lto() != Lto::No && sess.opts.incremental.is_some() {
- sess.err("can't perform LTO when compiling incrementally");
+ if sess.opts.incremental.is_some() {
+ match sess.lto() {
+ Lto::Yes |
+ Lto::Thin |
+ Lto::Fat => {
+ sess.err("can't perform LTO when compiling incrementally");
+ }
+ Lto::ThinLocal |
+ Lto::No => {
+ // This is fine
+ }
+ }
  }
 
  // Since we don't know if code in an rlib will be linked to statically or

diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs
@@ -716,12 +716,6 @@ impl<'tcx> QueryDescription<'tcx> for queries::codegen_unit<'tcx> {
  }
 }
 
-impl<'tcx> QueryDescription<'tcx> for queries::compile_codegen_unit<'tcx> {
- fn describe(_tcx: TyCtxt, _: InternedString) -> String {
- "compile_codegen_unit".to_string()
- }
-}
-
 impl<'tcx> QueryDescription<'tcx> for queries::output_filenames<'tcx> {
  fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
  "output_filenames".to_string()

diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs
@@ -28,7 +28,7 @@ use middle::lib_features::LibFeatures;
 use middle::lang_items::{LanguageItems, LangItem};
 use middle::exported_symbols::{SymbolExportLevel, ExportedSymbol};
 use mir::interpret::ConstEvalResult;
-use mir::mono::{CodegenUnit, Stats};
+use mir::mono::CodegenUnit;
 use mir;
 use mir::interpret::GlobalId;
 use session::{CompileResult, CrateDisambiguator};
@@ -525,7 +525,6 @@ define_queries! { <'tcx>
  -> (Arc<DefIdSet>, Arc<Vec<Arc<CodegenUnit<'tcx>>>>),
  [] fn is_codegened_item: IsCodegenedItem(DefId) -> bool,
  [] fn codegen_unit: CodegenUnit(InternedString) -> Arc<CodegenUnit<'tcx>>,
- [] fn compile_codegen_unit: CompileCodegenUnit(InternedString) -> Stats,
  },
 
  Other {

diff --git a/src/librustc_codegen_llvm/Cargo.toml b/src/librustc_codegen_llvm/Cargo.toml
@@ -14,6 +14,7 @@ cc = "1.0.1"
 num_cpus = "1.0"
 rustc-demangle = "0.1.4"
 rustc_llvm = { path = "../librustc_llvm" }
+memmap = "0.6"
 
 [features]
 # This is used to convince Cargo to separately cache builds of `rustc_codegen_llvm`