auto merge of rust-lang#15670 : epdtry/rust/fast-archive-builder, r=alexcrichton

When rustc produces an rlib, it includes the contents of each static library required by the crate.  Currently each static library is added individually, by extracting the library with `ar x` and adding the objects to the rlib using `ar r`.  Each `ar r` has significant overhead - it appears to scan through the full contents of the rlib before adding the new files.  This patch avoids most of the overhead by adding all library objects (and other rlib components) at once using a single `ar r`.

When building `librustc` (on Linux, using GNU ar), this patch gives a 60-80% reduction in linking time, from 90s to 10s one machine I tried and 25s to 8s on another.  (Though `librustc` is a bit of a special case - it's a very large crate, so the rlib is large to begin with, and it also relies on a total of 45 static libraries due to the way LLVM is organized.)  More reasonable crates such as `libstd` and `libcore` also get a small reduction in linking time (just from adding metadata, bitcode, and object code in one `ar` invocation instead of three), but this is not very noticeable since the time there is small to begin with (around 1s).

Author: bors

src/librustc/back/link.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use super::archive::{Archive, ArchiveConfig, METADATA_FILENAME};
+use super::archive::{Archive, ArchiveBuilder, ArchiveConfig, METADATA_FILENAME};
 use super::rpath;
 use super::rpath::RPathConfig;
 use super::svh::Svh;
@@ -983,7 +983,7 @@ fn link_binary_output(sess: &Session,
 
     match crate_type {
         config::CrateTypeRlib => {
-            link_rlib(sess, Some(trans), &obj_filename, &out_filename);
+            link_rlib(sess, Some(trans), &obj_filename, &out_filename).build();
         }
         config::CrateTypeStaticlib => {
             link_staticlib(sess, &obj_filename, &out_filename);
@@ -1019,7 +1019,7 @@ fn archive_search_paths(sess: &Session) -> Vec<Path> {
 fn link_rlib<'a>(sess: &'a Session,
                  trans: Option<&CrateTranslation>, // None == no metadata/bytecode
                  obj_filename: &Path,
-                 out_filename: &Path) -> Archive<'a> {
+                 out_filename: &Path) -> ArchiveBuilder<'a> {
     let handler = &sess.diagnostic().handler;
     let config = ArchiveConfig {
         handler: handler,
@@ -1028,17 +1028,30 @@ fn link_rlib<'a>(sess: &'a Session,
         os: sess.targ_cfg.os,
         maybe_ar_prog: sess.opts.cg.ar.clone()
     };
-    let mut a = Archive::create(config, obj_filename);
+    let mut ab = ArchiveBuilder::create(config);
+    ab.add_file(obj_filename).unwrap();
 
     for &(ref l, kind) in sess.cstore.get_used_libraries().borrow().iter() {
         match kind {
             cstore::NativeStatic => {
-                a.add_native_library(l.as_slice()).unwrap();
+                ab.add_native_library(l.as_slice()).unwrap();
             }
             cstore::NativeFramework | cstore::NativeUnknown => {}
         }
     }
 
+    // After adding all files to the archive, we need to update the
+    // symbol table of the archive.
+    ab.update_symbols();
+
+    let mut ab = match sess.targ_cfg.os {
+        // For OSX/iOS, we must be careful to update symbols only when adding
+        // object files.  We're about to start adding non-object files, so run
+        // `ar` now to process the object files.
+        abi::OsMacos | abi::OsiOS => ab.build().extend(),
+        _ => ab,
+    };
+
     // Note that it is important that we add all of our non-object "magical
     // files" *after* all of the object files in the archive. The reason for
     // this is as follows:
@@ -1078,7 +1091,7 @@ fn link_rlib<'a>(sess: &'a Session,
                     sess.abort_if_errors();
                 }
             }
-            a.add_file(&metadata, false);
+            ab.add_file(&metadata).unwrap();
             remove(sess, &metadata);
 
             // For LTO purposes, the bytecode of this library is also inserted
@@ -1105,25 +1118,18 @@ fn link_rlib<'a>(sess: &'a Session,
                     sess.abort_if_errors()
                 }
             }
-            a.add_file(&bc_deflated, false);
+            ab.add_file(&bc_deflated).unwrap();
             remove(sess, &bc_deflated);
             if !sess.opts.cg.save_temps &&
                !sess.opts.output_types.contains(&OutputTypeBitcode) {
                 remove(sess, &bc);
             }
-
-            // After adding all files to the archive, we need to update the
-            // symbol table of the archive. This currently dies on OSX (see
-            // #11162), and isn't necessary there anyway
-            match sess.targ_cfg.os {
-                abi::OsMacos | abi::OsiOS => {}
-                _ => { a.update_symbols(); }
-            }
         }
 
         None => {}
     }
-    return a;
+
+    ab
 }
 
 // Create a static archive
@@ -1139,9 +1145,13 @@ fn link_rlib<'a>(sess: &'a Session,
 // link in the metadata object file (and also don't prepare the archive with a
 // metadata file).
 fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
-    let mut a = link_rlib(sess, None, obj_filename, out_filename);
-    a.add_native_library("morestack").unwrap();
-    a.add_native_library("compiler-rt").unwrap();
+    let ab = link_rlib(sess, None, obj_filename, out_filename);
+    let mut ab = match sess.targ_cfg.os {
+        abi::OsMacos | abi::OsiOS => ab.build().extend(),
+        _ => ab,
+    };
+    ab.add_native_library("morestack").unwrap();
+    ab.add_native_library("compiler-rt").unwrap();
 
     let crates = sess.cstore.get_used_crates(cstore::RequireStatic);
     let mut all_native_libs = vec![];
@@ -1155,12 +1165,15 @@ fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
                 continue
             }
         };
-        a.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();
+        ab.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();
 
         let native_libs = csearch::get_native_libraries(&sess.cstore, cnum);
         all_native_libs.extend(native_libs.move_iter());
     }
 
+    ab.update_symbols();
+    let _ = ab.build();
+
     if !all_native_libs.is_empty() {
         sess.warn("link against the following native artifacts when linking against \
                   this static library");

src/librustc_back/archive.rs

@@ -36,6 +36,17 @@ pub struct Archive<'a> {
     maybe_ar_prog: Option<String>
 }
 
+/// Helper for adding many files to an archive with a single invocation of
+/// `ar`.
+#[must_use = "must call build() to finish building the archive"]
+pub struct ArchiveBuilder<'a> {
+    archive: Archive<'a>,
+    work_dir: TempDir,
+    /// Filename of each member that should be added to the archive.
+    members: Vec<Path>,
+    should_update_symbols: bool,
+}
+
 fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
           args: &str, cwd: Option<&Path>,
           paths: &[&Path]) -> ProcessOutput {
@@ -85,10 +96,8 @@ fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
 }
 
 impl<'a> Archive<'a> {
-    /// Initializes a new static archive with the given object file
-    pub fn create<'b>(config: ArchiveConfig<'a>, initial_object: &'b Path) -> Archive<'a> {
+    fn new(config: ArchiveConfig<'a>) -> Archive<'a> {
         let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
-        run_ar(handler, &maybe_ar_prog, "crus", None, [&dst, initial_object]);
         Archive {
             handler: handler,
             dst: dst,
@@ -100,17 +109,47 @@ impl<'a> Archive<'a> {
 
     /// Opens an existing static archive
     pub fn open(config: ArchiveConfig<'a>) -> Archive<'a> {
-        let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
-        assert!(dst.exists());
-        Archive {
-            handler: handler,
-            dst: dst,
-            lib_search_paths: lib_search_paths,
-            os: os,
-            maybe_ar_prog: maybe_ar_prog
+        let archive = Archive::new(config);
+        assert!(archive.dst.exists());
+        archive
+    }
+
+    /// Removes a file from this archive
+    pub fn remove_file(&mut self, file: &str) {
+        run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
+    }
+
+    /// Lists all files in an archive
+    pub fn files(&self) -> Vec<String> {
+        let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
+        let output = str::from_utf8(output.output.as_slice()).unwrap();
+        // use lines_any because windows delimits output with `\r\n` instead of
+        // just `\n`
+        output.lines_any().map(|s| s.to_string()).collect()
+    }
+
+    /// Creates an `ArchiveBuilder` for adding files to this archive.
+    pub fn extend(self) -> ArchiveBuilder<'a> {
+        ArchiveBuilder::new(self)
+    }
+}
+
+impl<'a> ArchiveBuilder<'a> {
+    fn new(archive: Archive<'a>) -> ArchiveBuilder<'a> {
+        ArchiveBuilder {
+            archive: archive,
+            work_dir: TempDir::new("rsar").unwrap(),
+            members: vec![],
+            should_update_symbols: false,
         }
     }
 
+    /// Create a new static archive, ready for adding files.
+    pub fn create(config: ArchiveConfig<'a>) -> ArchiveBuilder<'a> {
+        let archive = Archive::new(config);
+        ArchiveBuilder::new(archive)
+    }
+
     /// Adds all of the contents of a native library to this archive. This will
     /// search in the relevant locations for a library named `name`.
     pub fn add_native_library(&mut self, name: &str) -> io::IoResult<()> {
@@ -135,48 +174,96 @@ impl<'a> Archive<'a> {
     }
 
     /// Adds an arbitrary file to this archive
-    pub fn add_file(&mut self, file: &Path, has_symbols: bool) {
-        let cmd = if has_symbols {"r"} else {"rS"};
-        run_ar(self.handler, &self.maybe_ar_prog, cmd, None, [&self.dst, file]);
-    }
-
-    /// Removes a file from this archive
-    pub fn remove_file(&mut self, file: &str) {
-        run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
+    pub fn add_file(&mut self, file: &Path) -> io::IoResult<()> {
+        let filename = Path::new(file.filename().unwrap());
+        let new_file = self.work_dir.path().join(&filename);
+        try!(fs::copy(file, &new_file));
+        self.members.push(filename);
+        Ok(())
     }
 
-    /// Updates all symbols in the archive (runs 'ar s' over it)
+    /// Indicate that the next call to `build` should updates all symbols in
+    /// the archive (run 'ar s' over it).
     pub fn update_symbols(&mut self) {
-        run_ar(self.handler, &self.maybe_ar_prog, "s", None, [&self.dst]);
+        self.should_update_symbols = true;
     }
 
-    /// Lists all files in an archive
-    pub fn files(&self) -> Vec<String> {
-        let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
-        let output = str::from_utf8(output.output.as_slice()).unwrap();
-        // use lines_any because windows delimits output with `\r\n` instead of
-        // just `\n`
-        output.lines_any().map(|s| s.to_string()).collect()
+    /// Combine the provided files, rlibs, and native libraries into a single
+    /// `Archive`.
+    pub fn build(self) -> Archive<'a> {
+        // Get an absolute path to the destination, so `ar` will work even
+        // though we run it from `self.work_dir`.
+        let abs_dst = os::getcwd().join(&self.archive.dst);
+        assert!(!abs_dst.is_relative());
+        let mut args = vec![&abs_dst];
+        let mut total_len = abs_dst.as_vec().len();
+
+        if self.members.is_empty() {
+            // OSX `ar` does not allow using `r` with no members, but it does
+            // allow running `ar s file.a` to update symbols only.
+            if self.should_update_symbols {
+                run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+                       "s", Some(self.work_dir.path()), args.as_slice());
+            }
+            return self.archive;
+        }
+
+        // Don't allow the total size of `args` to grow beyond 32,000 bytes.
+        // Windows will raise an error if the argument string is longer than
+        // 32,768, and we leave a bit of extra space for the program name.
+        static ARG_LENGTH_LIMIT: uint = 32000;
+
+        for member_name in self.members.iter() {
+            let len = member_name.as_vec().len();
+
+            // `len + 1` to account for the space that's inserted before each
+            // argument.  (Windows passes command-line arguments as a single
+            // string, not an array of strings.)
+            if total_len + len + 1 > ARG_LENGTH_LIMIT {
+                // Add the archive members seen so far, without updating the
+                // symbol table (`S`).
+                run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+                       "cruS", Some(self.work_dir.path()), args.as_slice());
+
+                args.clear();
+                args.push(&abs_dst);
+                total_len = abs_dst.as_vec().len();
+            }
+
+            args.push(member_name);
+            total_len += len + 1;
+        }
+
+        // Add the remaining archive members, and update the symbol table if
+        // necessary.
+        let flags = if self.should_update_symbols { "crus" } else { "cruS" };
+        run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+               flags, Some(self.work_dir.path()), args.as_slice());
+
+        self.archive
     }
 
     fn add_archive(&mut self, archive: &Path, name: &str,
                    skip: &[&str]) -> io::IoResult<()> {
         let loc = TempDir::new("rsar").unwrap();
 
-        // First, extract the contents of the archive to a temporary directory
+        // First, extract the contents of the archive to a temporary directory.
+        // We don't unpack directly into `self.work_dir` due to the possibility
+        // of filename collisions.
         let archive = os::make_absolute(archive);
-        run_ar(self.handler, &self.maybe_ar_prog, "x", Some(loc.path()), [&archive]);
+        run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+               "x", Some(loc.path()), [&archive]);
 
         // Next, we must rename all of the inputs to "guaranteed unique names".
-        // The reason for this is that archives are keyed off the name of the
-        // files, so if two files have the same name they will override one
-        // another in the archive (bad).
+        // We move each file into `self.work_dir` under its new unique name.
+        // The reason for this renaming is that archives are keyed off the name
+        // of the files, so if two files have the same name they will override
+        // one another in the archive (bad).
         //
         // We skip any files explicitly desired for skipping, and we also skip
         // all SYMDEF files as these are just magical placeholders which get
         // re-created when we make a new archive anyway.
         let files = try!(fs::readdir(loc.path()));
-        let mut inputs = Vec::new();
         for file in files.iter() {
             let filename = file.filename_str().unwrap();
             if skip.iter().any(|s| *s == filename) { continue }
@@ -192,29 +279,23 @@ impl<'a> Archive<'a> {
             } else {
                 filename
             };
-            let new_filename = file.with_filename(filename);
+            let new_filename = self.work_dir.path().join(filename.as_slice());
             try!(fs::rename(file, &new_filename));
-            inputs.push(new_filename);
+            self.members.push(Path::new(filename));
         }
-        if inputs.len() == 0 { return Ok(()) }
-
-        // Finally, add all the renamed files to this archive
-        let mut args = vec!(&self.dst);
-        args.extend(inputs.iter());
-        run_ar(self.handler, &self.maybe_ar_prog, "r", None, args.as_slice());
         Ok(())
     }
 
     fn find_library(&self, name: &str) -> Path {
-        let (osprefix, osext) = match self.os {
+        let (osprefix, osext) = match self.archive.os {
             abi::OsWin32 => ("", "lib"), _ => ("lib", "a"),
         };
         // On Windows, static libraries sometimes show up as libfoo.a and other
         // times show up as foo.lib
         let oslibname = format!("{}{}.{}", osprefix, name, osext);
         let unixlibname = format!("lib{}.a", name);
 
-        for path in self.lib_search_paths.iter() {
+        for path in self.archive.lib_search_paths.iter() {
             debug!("looking for {} inside {}", name, path.display());
             let test = path.join(oslibname.as_slice());
             if test.exists() { return test }
@@ -223,9 +304,9 @@ impl<'a> Archive<'a> {
                 if test.exists() { return test }
             }
         }
-        self.handler.fatal(format!("could not find native static library `{}`, \
-                                 perhaps an -L flag is missing?",
-                                name).as_slice());
+        self.archive.handler.fatal(format!("could not find native static library `{}`, \
+                                            perhaps an -L flag is missing?",
+                                           name).as_slice());
     }
 }