Auto merge of #45399 - alexcrichton:compress-parallel, r=michaelwoerister

rustc: Move bytecode compression into codegen

This commit moves compression of the bytecode from the `link` module to the
`write` module, namely allowing it to be (a) cached by incremental compilation
and (b) produced in parallel. The parallelization may show up as some nice wins
during normal compilation and the caching in incremental mode should be
beneficial for incremental compiles! (no more need to recompress the entire
crate's bitcode on all builds)

Author: bors

src/librustc/dep_graph/graph.rs

@@ -12,7 +12,6 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher,
                                            StableHashingContextProvider};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::indexed_vec::{Idx, IndexVec};
-use session::config::OutputType;
 use std::cell::{Ref, RefCell};
 use std::env;
 use std::hash::Hash;
@@ -647,7 +646,14 @@ impl DepGraph {
 pub struct WorkProduct {
     pub cgu_name: String,
     /// Saved files associated with this CGU
-    pub saved_files: Vec<(OutputType, String)>,
+    pub saved_files: Vec<(WorkProductFileKind, String)>,
+}
+
+#[derive(Clone, Copy, Debug, RustcEncodable, RustcDecodable)]
+pub enum WorkProductFileKind {
+    Object,
+    Bytecode,
+    BytecodeCompressed,
 }
 
 pub(super) struct CurrentDepGraph {

src/librustc/dep_graph/mod.rs

@@ -21,6 +21,7 @@ mod serialized;
 pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig};
 pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId, label_strs};
 pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex, DepNodeColor};
+pub use self::graph::WorkProductFileKind;
 pub use self::prev::PreviousDepGraph;
 pub use self::query::DepGraphQuery;
 pub use self::safe::AssertDepGraphSafe;

src/librustc_incremental/persist/work_product.rs

@@ -11,29 +11,33 @@
 //! This module contains files for saving intermediate work-products.
 
 use persist::fs::*;
-use rustc::dep_graph::{WorkProduct, WorkProductId, DepGraph};
+use rustc::dep_graph::{WorkProduct, WorkProductId, DepGraph, WorkProductFileKind};
 use rustc::session::Session;
-use rustc::session::config::OutputType;
 use rustc::util::fs::link_or_copy;
 use std::path::PathBuf;
 use std::fs as std_fs;
 
 pub fn save_trans_partition(sess: &Session,
                             dep_graph: &DepGraph,
                             cgu_name: &str,
-                            files: &[(OutputType, PathBuf)]) {
+                            files: &[(WorkProductFileKind, PathBuf)]) {
     debug!("save_trans_partition({:?},{:?})",
            cgu_name,
            files);
     if sess.opts.incremental.is_none() {
-        return;
+        return
     }
     let work_product_id = WorkProductId::from_cgu_name(cgu_name);
 
     let saved_files: Option<Vec<_>> =
         files.iter()
              .map(|&(kind, ref path)| {
-                 let file_name = format!("cgu-{}.{}", cgu_name, kind.extension());
+                 let extension = match kind {
+                     WorkProductFileKind::Object => "o",
+                     WorkProductFileKind::Bytecode => "bc",
+                     WorkProductFileKind::BytecodeCompressed => "bc-compressed",
+                 };
+                 let file_name = format!("cgu-{}.{}", cgu_name, extension);
                  let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name);
                  match link_or_copy(path, &path_in_incr_dir) {
                      Ok(_) => Some((kind, file_name)),

src/librustc_trans/back/link.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use super::archive::{ArchiveBuilder, ArchiveConfig};
-use super::bytecode::{self, RLIB_BYTECODE_EXTENSION};
+use super::bytecode::RLIB_BYTECODE_EXTENSION;
 use super::linker::Linker;
 use super::command::Command;
 use super::rpath::RPathConfig;
@@ -37,7 +37,7 @@ use std::env;
 use std::ffi::OsString;
 use std::fmt;
 use std::fs::{self, File};
-use std::io::{self, Read, Write, BufWriter};
+use std::io::{self, Write, BufWriter};
 use std::path::{Path, PathBuf};
 use std::process::{Output, Stdio};
 use std::str;
@@ -126,14 +126,6 @@ fn command_path(sess: &Session) -> OsString {
     env::join_paths(new_path).unwrap()
 }
 
-fn metadata_obj(outputs: &OutputFilenames) -> PathBuf {
-    outputs.temp_path(OutputType::Object, Some(METADATA_MODULE_NAME))
-}
-
-fn allocator_obj(outputs: &OutputFilenames) -> PathBuf {
-    outputs.temp_path(OutputType::Object, Some(ALLOCATOR_MODULE_NAME))
-}
-
 pub fn remove(sess: &Session, path: &Path) {
     match fs::remove_file(path) {
         Ok(..) => {}
@@ -175,13 +167,23 @@ pub fn link_binary(sess: &Session,
     // Remove the temporary object file and metadata if we aren't saving temps
     if !sess.opts.cg.save_temps {
         if sess.opts.output_types.should_trans() {
-            for obj in trans.modules.iter() {
-                remove(sess, &obj.object);
+            for obj in trans.modules.iter().filter_map(|m| m.object.as_ref()) {
+                remove(sess, obj);
             }
         }
-        remove(sess, &metadata_obj(outputs));
-        if trans.allocator_module.is_some() {
-            remove(sess, &allocator_obj(outputs));
+        for obj in trans.modules.iter().filter_map(|m| m.bytecode_compressed.as_ref()) {
+            remove(sess, obj);
+        }
+        if let Some(ref obj) = trans.metadata_module.object {
+            remove(sess, obj);
+        }
+        if let Some(ref allocator) = trans.allocator_module {
+            if let Some(ref obj) = allocator.object {
+                remove(sess, obj);
+            }
+            if let Some(ref bc) = allocator.bytecode_compressed {
+                remove(sess, bc);
+            }
         }
     }
 
@@ -256,8 +258,8 @@ fn link_binary_output(sess: &Session,
                       crate_type: config::CrateType,
                       outputs: &OutputFilenames,
                       crate_name: &str) -> Vec<PathBuf> {
-    for module in trans.modules.iter() {
-        check_file_is_writeable(&module.object, sess);
+    for obj in trans.modules.iter().filter_map(|m| m.object.as_ref()) {
+        check_file_is_writeable(obj, sess);
     }
 
     let tmpdir = match TempDir::new("rustc") {
@@ -280,20 +282,14 @@ fn link_binary_output(sess: &Session,
                 link_rlib(sess,
                           trans,
                           RlibFlavor::Normal,
-                          outputs,
                           &out_filename,
                           tmpdir.path()).build();
             }
             config::CrateTypeStaticlib => {
-                link_staticlib(sess,
-                               trans,
-                               outputs,
-                               &out_filename,
-                               tmpdir.path());
+                link_staticlib(sess, trans, &out_filename, tmpdir.path());
             }
             _ => {
-                link_natively(sess, crate_type, &out_filename,
-                              trans, outputs, tmpdir.path());
+                link_natively(sess, crate_type, &out_filename, trans, tmpdir.path());
             }
         }
         out_filenames.push(out_filename);
@@ -349,14 +345,13 @@ enum RlibFlavor {
 fn link_rlib<'a>(sess: &'a Session,
                  trans: &CrateTranslation,
                  flavor: RlibFlavor,
-                 outputs: &OutputFilenames,
                  out_filename: &Path,
                  tmpdir: &Path) -> ArchiveBuilder<'a> {
     info!("preparing rlib to {:?}", out_filename);
     let mut ab = ArchiveBuilder::new(archive_config(sess, out_filename, None));
 
-    for module in trans.modules.iter() {
-        ab.add_file(&module.object);
+    for obj in trans.modules.iter().filter_map(|m| m.object.as_ref()) {
+        ab.add_file(obj);
     }
 
     // Note that in this loop we are ignoring the value of `lib.cfg`. That is,
@@ -421,56 +416,9 @@ fn link_rlib<'a>(sess: &'a Session,
             ab.add_file(&metadata);
 
             // For LTO purposes, the bytecode of this library is also inserted
-            // into the archive.  If codegen_units > 1, we insert each of the
-            // bitcode files.
-            for module in trans.modules.iter() {
-                // Note that we make sure that the bytecode filename in the
-                // archive is never exactly 16 bytes long by adding a 16 byte
-                // extension to it. This is to work around a bug in LLDB that
-                // would cause it to crash if the name of a file in an archive
-                // was exactly 16 bytes.
-                let bc_filename = module.object.with_extension("bc");
-                let bc_encoded_filename = tmpdir.join({
-                    module.object.with_extension(RLIB_BYTECODE_EXTENSION).file_name().unwrap()
-                });
-
-                let mut bc_data = Vec::new();
-                match fs::File::open(&bc_filename).and_then(|mut f| {
-                    f.read_to_end(&mut bc_data)
-                }) {
-                    Ok(..) => {}
-                    Err(e) => sess.fatal(&format!("failed to read bytecode: {}",
-                                                 e))
-                }
-
-                let encoded = bytecode::encode(&module.llmod_id, &bc_data);
-
-                let mut bc_file_deflated = match fs::File::create(&bc_encoded_filename) {
-                    Ok(file) => file,
-                    Err(e) => {
-                        sess.fatal(&format!("failed to create compressed \
-                                             bytecode file: {}", e))
-                    }
-                };
-
-                match bc_file_deflated.write_all(&encoded) {
-                    Ok(()) => {}
-                    Err(e) => {
-                        sess.fatal(&format!("failed to write compressed \
-                                             bytecode: {}", e));
-                    }
-                };
-
-                ab.add_file(&bc_encoded_filename);
-
-                // See the bottom of back::write::run_passes for an explanation
-                // of when we do and don't keep .#module-name#.bc files around.
-                let user_wants_numbered_bitcode =
-                        sess.opts.output_types.contains_key(&OutputType::Bitcode) &&
-                        sess.codegen_units() > 1;
-                if !sess.opts.cg.save_temps && !user_wants_numbered_bitcode {
-                    remove(sess, &bc_filename);
-                }
+            // into the archive.
+            for bytecode in trans.modules.iter().filter_map(|m| m.bytecode_compressed.as_ref()) {
+                ab.add_file(bytecode);
             }
 
             // After adding all files to the archive, we need to update the
@@ -482,8 +430,11 @@ fn link_rlib<'a>(sess: &'a Session,
         }
 
         RlibFlavor::StaticlibBase => {
-            if trans.allocator_module.is_some() {
-                ab.add_file(&allocator_obj(outputs));
+            let obj = trans.allocator_module
+                .as_ref()
+                .and_then(|m| m.object.as_ref());
+            if let Some(obj) = obj {
+                ab.add_file(obj);
             }
         }
     }
@@ -505,13 +456,11 @@ fn link_rlib<'a>(sess: &'a Session,
 // metadata file).
 fn link_staticlib(sess: &Session,
                   trans: &CrateTranslation,
-                  outputs: &OutputFilenames,
                   out_filename: &Path,
                   tempdir: &Path) {
     let mut ab = link_rlib(sess,
                            trans,
                            RlibFlavor::StaticlibBase,
-                           outputs,
                            out_filename,
                            tempdir);
     let mut all_native_libs = vec![];
@@ -616,7 +565,6 @@ fn link_natively(sess: &Session,
                  crate_type: config::CrateType,
                  out_filename: &Path,
                  trans: &CrateTranslation,
-                 outputs: &OutputFilenames,
                  tmpdir: &Path) {
     info!("preparing {:?} to {:?}", crate_type, out_filename);
     let flavor = sess.linker_flavor();
@@ -656,7 +604,7 @@ fn link_natively(sess: &Session,
     {
         let mut linker = trans.linker_info.to_linker(cmd, &sess);
         link_args(&mut *linker, sess, crate_type, tmpdir,
-                  out_filename, outputs, trans);
+                  out_filename, trans);
         cmd = linker.finalize();
     }
     if let Some(args) = sess.target.target.options.late_link_args.get(&flavor) {
@@ -878,7 +826,6 @@ fn link_args(cmd: &mut Linker,
              crate_type: config::CrateType,
              tmpdir: &Path,
              out_filename: &Path,
-             outputs: &OutputFilenames,
              trans: &CrateTranslation) {
 
     // The default library location, we need this to find the runtime.
@@ -889,8 +836,8 @@ fn link_args(cmd: &mut Linker,
     let t = &sess.target.target;
 
     cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path));
-    for module in trans.modules.iter() {
-        cmd.add_object(&module.object);
+    for obj in trans.modules.iter().filter_map(|m| m.object.as_ref()) {
+        cmd.add_object(obj);
     }
     cmd.output_filename(out_filename);
 
@@ -913,11 +860,16 @@ fn link_args(cmd: &mut Linker,
     // object file, so we link that in here.
     if crate_type == config::CrateTypeDylib ||
        crate_type == config::CrateTypeProcMacro {
-        cmd.add_object(&metadata_obj(outputs));
+        if let Some(obj) = trans.metadata_module.object.as_ref() {
+            cmd.add_object(obj);
+        }
     }
 
-    if trans.allocator_module.is_some() {
-        cmd.add_object(&allocator_obj(outputs));
+    let obj = trans.allocator_module
+        .as_ref()
+        .and_then(|m| m.object.as_ref());
+    if let Some(obj) = obj {
+        cmd.add_object(obj);
     }
 
     // Try to strip as much out of the generated object by removing unused
@@ -1185,9 +1137,9 @@ fn add_upstream_rust_crates(cmd: &mut Linker,
 
         for f in archive.src_files() {
             if f.ends_with(RLIB_BYTECODE_EXTENSION) || f == METADATA_FILENAME {
-                    archive.remove_file(&f);
-                    continue
-                }
+                archive.remove_file(&f);
+                continue
+            }
         }
 
         archive.build();

src/librustc_trans/back/lto.rs

@@ -343,8 +343,7 @@ fn thin_lto(diag_handler: &Handler,
             info!("local module: {} - {}", i, module.llmod_id);
             let llvm = module.llvm().expect("can't lto pretranslated module");
             let name = CString::new(module.llmod_id.clone()).unwrap();
-            let buffer = llvm::LLVMRustThinLTOBufferCreate(llvm.llmod);
-            let buffer = ThinBuffer(buffer);
+            let buffer = ThinBuffer::new(llvm.llmod);
             thin_modules.push(llvm::ThinLTOModule {
                 identifier: name.as_ptr(),
                 data: buffer.data().as_ptr(),
@@ -499,13 +498,13 @@ unsafe impl Send for ModuleBuffer {}
 unsafe impl Sync for ModuleBuffer {}
 
 impl ModuleBuffer {
-    fn new(m: ModuleRef) -> ModuleBuffer {
+    pub fn new(m: ModuleRef) -> ModuleBuffer {
         ModuleBuffer(unsafe {
             llvm::LLVMRustModuleBufferCreate(m)
         })
     }
 
-    fn data(&self) -> &[u8] {
+    pub fn data(&self) -> &[u8] {
         unsafe {
             let ptr = llvm::LLVMRustModuleBufferPtr(self.0);
             let len = llvm::LLVMRustModuleBufferLen(self.0);
@@ -545,13 +544,20 @@ impl Drop for ThinData {
     }
 }
 
-struct ThinBuffer(*mut llvm::ThinLTOBuffer);
+pub struct ThinBuffer(*mut llvm::ThinLTOBuffer);
 
 unsafe impl Send for ThinBuffer {}
 unsafe impl Sync for ThinBuffer {}
 
 impl ThinBuffer {
-    fn data(&self) -> &[u8] {
+    pub fn new(m: ModuleRef) -> ThinBuffer {
+        unsafe {
+            let buffer = llvm::LLVMRustThinLTOBufferCreate(m);
+            ThinBuffer(buffer)
+        }
+    }
+
+    pub fn data(&self) -> &[u8] {
         unsafe {
             let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _;
             let len = llvm::LLVMRustThinLTOBufferLen(self.0);

src/librustc_trans/back/write.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use back::lto;
+use back::bytecode::{self, RLIB_BYTECODE_EXTENSION};
+use back::lto::{self, ModuleBuffer, ThinBuffer};
 use back::link::{self, get_linker, remove};
 use back::linker::LinkerInfo;
 use back::symbol_export::ExportedSymbols;
 use base;
 use consts;
 use rustc_incremental::{save_trans_partition, in_incr_comp_dir};
-use rustc::dep_graph::DepGraph;
+use rustc::dep_graph::{DepGraph, WorkProductFileKind};
 use rustc::middle::cstore::{LinkMeta, EncodedMetadata};
 use rustc::session::config::{self, OutputFilenames, OutputType, OutputTypes, Passes, SomePasses,
                              AllPasses, Sanitizer};
@@ -44,7 +45,7 @@ use rustc_demangle;
 
 use std::any::Any;
 use std::ffi::{CString, CStr};
-use std::fs;
+use std::fs::{self, File};
 use std::io;
 use std::io::Write;
 use std::mem;
@@ -228,6 +229,7 @@ pub struct ModuleConfig {
     // Flags indicating which outputs to produce.
     emit_no_opt_bc: bool,
     emit_bc: bool,
+    emit_bc_compressed: bool,
     emit_lto_bc: bool,
     emit_ir: bool,
     emit_asm: bool,
@@ -257,6 +259,7 @@ impl ModuleConfig {
 
             emit_no_opt_bc: false,
             emit_bc: false,
+            emit_bc_compressed: false,
             emit_lto_bc: false,
             emit_ir: false,
             emit_asm: false,
@@ -627,20 +630,34 @@ unsafe fn codegen(cgcx: &CodegenContext,
     let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
     let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name);
 
-    if write_bc {
-        let bc_out_c = path2cstr(&bc_out);
-        if llvm::LLVMRustThinLTOAvailable() {
-            with_codegen(tm, llmod, config.no_builtins, |cpm| {
-                llvm::LLVMRustWriteThinBitcodeToFile(
-                    cpm,
-                    llmod,
-                    bc_out_c.as_ptr(),
-                )
-            });
+
+    if write_bc || config.emit_bc_compressed {
+        let thin;
+        let old;
+        let data = if llvm::LLVMRustThinLTOAvailable() {
+            thin = ThinBuffer::new(llmod);
+            thin.data()
         } else {
-            llvm::LLVMWriteBitcodeToFile(llmod, bc_out_c.as_ptr());
+            old = ModuleBuffer::new(llmod);
+            old.data()
+        };
+        timeline.record("make-bc");
+
+        if write_bc {
+            if let Err(e) = File::create(&bc_out).and_then(|mut f| f.write_all(data)) {
+                diag_handler.err(&format!("failed to write bytecode: {}", e));
+            }
+            timeline.record("write-bc");
+        }
+
+        if config.emit_bc_compressed {
+            let dst = bc_out.with_extension(RLIB_BYTECODE_EXTENSION);
+            let data = bytecode::encode(&mtrans.llmod_id, data);
+            if let Err(e) = File::create(&dst).and_then(|mut f| f.write_all(&data)) {
+                diag_handler.err(&format!("failed to write bytecode: {}", e));
+            }
+            timeline.record("compress-bc");
         }
-        timeline.record("bc");
     }
 
     time(config.time_passes, &format!("codegen passes [{}]", module_name.unwrap()),
@@ -736,6 +753,7 @@ unsafe fn codegen(cgcx: &CodegenContext,
     drop(handlers);
     Ok(mtrans.into_compiled_module(config.emit_obj,
                                    config.emit_bc,
+                                   config.emit_bc_compressed,
                                    &cgcx.output_filenames))
 }
 
@@ -822,11 +840,12 @@ pub fn start_async_translation(tcx: TyCtxt,
         allocator_config.emit_bc = true;
     }
 
-    // Emit bitcode files for the crate if we're emitting an rlib.
-    // Whenever an rlib is created, the bitcode is inserted into the
-    // archive in order to allow LTO against it.
+    // Emit compressed bitcode files for the crate if we're emitting an rlib.
+    // Whenever an rlib is created, the bitcode is inserted into the archive in
+    // order to allow LTO against it.
     if need_crate_bitcode_for_rlib(sess) {
-        modules_config.emit_bc = true;
+        modules_config.emit_bc_compressed = true;
+        allocator_config.emit_bc_compressed = true;
     }
 
     for output_type in output_types_override.keys() {
@@ -906,29 +925,25 @@ pub fn start_async_translation(tcx: TyCtxt,
 
 fn copy_module_artifacts_into_incr_comp_cache(sess: &Session,
                                               dep_graph: &DepGraph,
-                                              compiled_modules: &CompiledModules,
-                                              crate_output: &OutputFilenames) {
+                                              compiled_modules: &CompiledModules) {
     if sess.opts.incremental.is_none() {
         return;
     }
 
     for module in compiled_modules.modules.iter() {
         let mut files = vec![];
 
-        if module.emit_obj {
-            let path = crate_output.temp_path(OutputType::Object, Some(&module.name));
-            files.push((OutputType::Object, path));
+        if let Some(ref path) = module.object {
+            files.push((WorkProductFileKind::Object, path.clone()));
         }
-
-        if module.emit_bc {
-            let path = crate_output.temp_path(OutputType::Bitcode, Some(&module.name));
-            files.push((OutputType::Bitcode, path));
+        if let Some(ref path) = module.bytecode {
+            files.push((WorkProductFileKind::Bytecode, path.clone()));
+        }
+        if let Some(ref path) = module.bytecode_compressed {
+            files.push((WorkProductFileKind::BytecodeCompressed, path.clone()));
         }
 
-        save_trans_partition(sess,
-                             dep_graph,
-                             &module.name,
-                             &files);
+        save_trans_partition(sess, dep_graph, &module.name, &files);
     }
 }
 
@@ -1032,8 +1047,6 @@ fn produce_final_output_artifacts(sess: &Session,
         // well.
 
         // Specific rules for keeping .#module-name#.bc:
-        //  - If we're building an rlib (`needs_crate_bitcode`), then keep
-        //    it.
         //  - If the user requested bitcode (`user_wants_bitcode`), and
         //    codegen_units > 1, then keep it.
         //  - If the user requested bitcode but codegen_units == 1, then we
@@ -1043,41 +1056,37 @@ fn produce_final_output_artifacts(sess: &Session,
         // If you change how this works, also update back::link::link_rlib,
         // where .#module-name#.bc files are (maybe) deleted after making an
         // rlib.
-        let needs_crate_bitcode = need_crate_bitcode_for_rlib(sess);
         let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
 
-        let keep_numbered_bitcode = needs_crate_bitcode ||
-                (user_wants_bitcode && sess.codegen_units() > 1);
+        let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units() > 1;
 
         let keep_numbered_objects = needs_crate_object ||
                 (user_wants_objects && sess.codegen_units() > 1);
 
         for module in compiled_modules.modules.iter() {
-            let module_name = Some(&module.name[..]);
-
-            if module.emit_obj && !keep_numbered_objects {
-                let path = crate_output.temp_path(OutputType::Object, module_name);
-                remove(sess, &path);
+            if let Some(ref path) = module.object {
+                if !keep_numbered_objects {
+                    remove(sess, path);
+                }
             }
 
-            if module.emit_bc && !keep_numbered_bitcode {
-                let path = crate_output.temp_path(OutputType::Bitcode, module_name);
-                remove(sess, &path);
+            if let Some(ref path) = module.bytecode {
+                if !keep_numbered_bitcode {
+                    remove(sess, path);
+                }
             }
         }
 
-        if compiled_modules.metadata_module.emit_bc && !user_wants_bitcode {
-            let path = crate_output.temp_path(OutputType::Bitcode,
-                                              Some(&compiled_modules.metadata_module.name));
-            remove(sess, &path);
-        }
-
-        if let Some(ref allocator_module) = compiled_modules.allocator_module {
-            if allocator_module.emit_bc && !user_wants_bitcode {
-                let path = crate_output.temp_path(OutputType::Bitcode,
-                                                  Some(&allocator_module.name));
+        if !user_wants_bitcode {
+            if let Some(ref path) = compiled_modules.metadata_module.bytecode {
                 remove(sess, &path);
             }
+
+            if let Some(ref allocator_module) = compiled_modules.allocator_module {
+                if let Some(ref path) = allocator_module.bytecode {
+                    remove(sess, path);
+                }
+            }
         }
     }
 
@@ -1149,8 +1158,28 @@ fn execute_work_item(cgcx: &CodegenContext,
                                         .as_ref()
                                         .unwrap();
         let name = &mtrans.name;
+        let mut object = None;
+        let mut bytecode = None;
+        let mut bytecode_compressed = None;
         for (kind, saved_file) in wp.saved_files {
-            let obj_out = cgcx.output_filenames.temp_path(kind, Some(name));
+            let obj_out = match kind {
+                WorkProductFileKind::Object => {
+                    let path = cgcx.output_filenames.temp_path(OutputType::Object, Some(name));
+                    object = Some(path.clone());
+                    path
+                }
+                WorkProductFileKind::Bytecode => {
+                    let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name));
+                    bytecode = Some(path.clone());
+                    path
+                }
+                WorkProductFileKind::BytecodeCompressed => {
+                    let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name))
+                        .with_extension(RLIB_BYTECODE_EXTENSION);
+                    bytecode_compressed = Some(path.clone());
+                    path
+                }
+            };
             let source_file = in_incr_comp_dir(&incr_comp_session_dir,
                                                &saved_file);
             debug!("copying pre-existing module `{}` from {:?} to {}",
@@ -1167,16 +1196,18 @@ fn execute_work_item(cgcx: &CodegenContext,
                 }
             }
         }
-        let object = cgcx.output_filenames.temp_path(OutputType::Object, Some(name));
+        assert_eq!(object.is_some(), config.emit_obj);
+        assert_eq!(bytecode.is_some(), config.emit_bc);
+        assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed);
 
         Ok(WorkItemResult::Compiled(CompiledModule {
-            object,
             llmod_id: mtrans.llmod_id.clone(),
             name: module_name,
             kind: ModuleKind::Regular,
             pre_existing: true,
-            emit_bc: config.emit_bc,
-            emit_obj: config.emit_obj,
+            object,
+            bytecode,
+            bytecode_compressed,
         }))
     } else {
         debug!("llvm-optimizing {:?}", module_name);
@@ -2053,8 +2084,7 @@ impl OngoingCrateTranslation {
 
         copy_module_artifacts_into_incr_comp_cache(sess,
                                                    dep_graph,
-                                                   &compiled_modules,
-                                                   &self.output_filenames);
+                                                   &compiled_modules);
         produce_final_output_artifacts(sess,
                                        &compiled_modules,
                                        &self.output_filenames);
@@ -2075,6 +2105,7 @@ impl OngoingCrateTranslation {
 
             modules: compiled_modules.modules,
             allocator_module: compiled_modules.allocator_module,
+            metadata_module: compiled_modules.metadata_module,
         };
 
         if self.no_integrated_as {

src/librustc_trans/lib.rs

@@ -64,6 +64,7 @@ extern crate serialize;
 extern crate cc; // Used to locate MSVC
 
 pub use base::trans_crate;
+use back::bytecode::RLIB_BYTECODE_EXTENSION;
 
 pub use metadata::LlvmMetadataLoader;
 pub use llvm_util::{init, target_features, print_version, print_passes, print, enable_llvm_debug};
@@ -90,7 +91,7 @@ mod diagnostics;
 
 pub mod back {
     mod archive;
-    mod bytecode;
+    pub mod bytecode;
     mod command;
     pub(crate) mod linker;
     pub mod link;
@@ -227,21 +228,37 @@ impl ModuleTranslation {
     pub fn into_compiled_module(self,
                                 emit_obj: bool,
                                 emit_bc: bool,
+                                emit_bc_compressed: bool,
                                 outputs: &OutputFilenames) -> CompiledModule {
         let pre_existing = match self.source {
             ModuleSource::Preexisting(_) => true,
             ModuleSource::Translated(_) => false,
         };
-        let object = outputs.temp_path(OutputType::Object, Some(&self.name));
+        let object = if emit_obj {
+            Some(outputs.temp_path(OutputType::Object, Some(&self.name)))
+        } else {
+            None
+        };
+        let bytecode = if emit_bc {
+            Some(outputs.temp_path(OutputType::Bitcode, Some(&self.name)))
+        } else {
+            None
+        };
+        let bytecode_compressed = if emit_bc_compressed {
+            Some(outputs.temp_path(OutputType::Bitcode, Some(&self.name))
+                    .with_extension(RLIB_BYTECODE_EXTENSION))
+        } else {
+            None
+        };
 
         CompiledModule {
             llmod_id: self.llmod_id,
             name: self.name.clone(),
             kind: self.kind,
             pre_existing,
-            emit_obj,
-            emit_bc,
             object,
+            bytecode,
+            bytecode_compressed,
         }
     }
 }
@@ -250,11 +267,11 @@ impl ModuleTranslation {
 pub struct CompiledModule {
     pub name: String,
     pub llmod_id: String,
-    pub object: PathBuf,
     pub kind: ModuleKind,
     pub pre_existing: bool,
-    pub emit_obj: bool,
-    pub emit_bc: bool,
+    pub object: Option<PathBuf>,
+    pub bytecode: Option<PathBuf>,
+    pub bytecode_compressed: Option<PathBuf>,
 }
 
 pub enum ModuleSource {
@@ -289,6 +306,7 @@ pub struct CrateTranslation {
     pub crate_name: Symbol,
     pub modules: Vec<CompiledModule>,
     allocator_module: Option<CompiledModule>,
+    metadata_module: CompiledModule,
     pub link: rustc::middle::cstore::LinkMeta,
     pub metadata: rustc::middle::cstore::EncodedMetadata,
     windows_subsystem: Option<String>,