Skip to content

speed up static linking by combining ar invocations #15670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 30, 2014
Merged

speed up static linking by combining ar invocations #15670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 33 additions & 20 deletions src/librustc/back/link.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use super::archive::{Archive, ArchiveConfig, METADATA_FILENAME};
use super::archive::{Archive, ArchiveBuilder, ArchiveConfig, METADATA_FILENAME};
use super::rpath;
use super::rpath::RPathConfig;
use super::svh::Svh;
Expand Down Expand Up @@ -983,7 +983,7 @@ fn link_binary_output(sess: &Session,

match crate_type {
config::CrateTypeRlib => {
link_rlib(sess, Some(trans), &obj_filename, &out_filename);
link_rlib(sess, Some(trans), &obj_filename, &out_filename).build();
}
config::CrateTypeStaticlib => {
link_staticlib(sess, &obj_filename, &out_filename);
Expand Down Expand Up @@ -1019,7 +1019,7 @@ fn archive_search_paths(sess: &Session) -> Vec<Path> {
fn link_rlib<'a>(sess: &'a Session,
trans: Option<&CrateTranslation>, // None == no metadata/bytecode
obj_filename: &Path,
out_filename: &Path) -> Archive<'a> {
out_filename: &Path) -> ArchiveBuilder<'a> {
let handler = &sess.diagnostic().handler;
let config = ArchiveConfig {
handler: handler,
Expand All @@ -1028,17 +1028,30 @@ fn link_rlib<'a>(sess: &'a Session,
os: sess.targ_cfg.os,
maybe_ar_prog: sess.opts.cg.ar.clone()
};
let mut a = Archive::create(config, obj_filename);
let mut ab = ArchiveBuilder::create(config);
ab.add_file(obj_filename).unwrap();

for &(ref l, kind) in sess.cstore.get_used_libraries().borrow().iter() {
match kind {
cstore::NativeStatic => {
a.add_native_library(l.as_slice()).unwrap();
ab.add_native_library(l.as_slice()).unwrap();
}
cstore::NativeFramework | cstore::NativeUnknown => {}
}
}

// After adding all files to the archive, we need to update the
// symbol table of the archive.
ab.update_symbols();

let mut ab = match sess.targ_cfg.os {
// For OSX/iOS, we must be careful to update symbols only when adding
// object files. We're about to start adding non-object files, so run
// `ar` now to process the object files.
abi::OsMacos | abi::OsiOS => ab.build().extend(),
_ => ab,
};

// Note that it is important that we add all of our non-object "magical
// files" *after* all of the object files in the archive. The reason for
// this is as follows:
Expand Down Expand Up @@ -1078,7 +1091,7 @@ fn link_rlib<'a>(sess: &'a Session,
sess.abort_if_errors();
}
}
a.add_file(&metadata, false);
ab.add_file(&metadata).unwrap();
remove(sess, &metadata);

// For LTO purposes, the bytecode of this library is also inserted
Expand All @@ -1105,25 +1118,18 @@ fn link_rlib<'a>(sess: &'a Session,
sess.abort_if_errors()
}
}
a.add_file(&bc_deflated, false);
ab.add_file(&bc_deflated).unwrap();
remove(sess, &bc_deflated);
if !sess.opts.cg.save_temps &&
!sess.opts.output_types.contains(&OutputTypeBitcode) {
remove(sess, &bc);
}

// After adding all files to the archive, we need to update the
// symbol table of the archive. This currently dies on OSX (see
// #11162), and isn't necessary there anyway
match sess.targ_cfg.os {
abi::OsMacos | abi::OsiOS => {}
_ => { a.update_symbols(); }
}
}

None => {}
}
return a;

ab
}

// Create a static archive
Expand All @@ -1139,9 +1145,13 @@ fn link_rlib<'a>(sess: &'a Session,
// link in the metadata object file (and also don't prepare the archive with a
// metadata file).
fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
let mut a = link_rlib(sess, None, obj_filename, out_filename);
a.add_native_library("morestack").unwrap();
a.add_native_library("compiler-rt").unwrap();
let ab = link_rlib(sess, None, obj_filename, out_filename);
let mut ab = match sess.targ_cfg.os {
abi::OsMacos | abi::OsiOS => ab.build().extend(),
_ => ab,
};
ab.add_native_library("morestack").unwrap();
ab.add_native_library("compiler-rt").unwrap();

let crates = sess.cstore.get_used_crates(cstore::RequireStatic);
let mut all_native_libs = vec![];
Expand All @@ -1155,12 +1165,15 @@ fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
continue
}
};
a.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();
ab.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();

let native_libs = csearch::get_native_libraries(&sess.cstore, cnum);
all_native_libs.extend(native_libs.move_iter());
}

ab.update_symbols();
let _ = ab.build();

if !all_native_libs.is_empty() {
sess.warn("link against the following native artifacts when linking against \
this static library");
Expand Down
175 changes: 128 additions & 47 deletions src/librustc_back/archive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,17 @@ pub struct Archive<'a> {
maybe_ar_prog: Option<String>
}

/// Helper for adding many files to an archive with a single invocation of
/// `ar`.
#[must_use = "must call build() to finish building the archive"]
pub struct ArchiveBuilder<'a> {
archive: Archive<'a>,
work_dir: TempDir,
/// Filename of each member that should be added to the archive.
members: Vec<Path>,
should_update_symbols: bool,
}

fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
args: &str, cwd: Option<&Path>,
paths: &[&Path]) -> ProcessOutput {
Expand Down Expand Up @@ -85,10 +96,8 @@ fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
}

impl<'a> Archive<'a> {
/// Initializes a new static archive with the given object file
pub fn create<'b>(config: ArchiveConfig<'a>, initial_object: &'b Path) -> Archive<'a> {
fn new(config: ArchiveConfig<'a>) -> Archive<'a> {
let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
run_ar(handler, &maybe_ar_prog, "crus", None, [&dst, initial_object]);
Archive {
handler: handler,
dst: dst,
Expand All @@ -100,17 +109,47 @@ impl<'a> Archive<'a> {

/// Opens an existing static archive
pub fn open(config: ArchiveConfig<'a>) -> Archive<'a> {
let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
assert!(dst.exists());
Archive {
handler: handler,
dst: dst,
lib_search_paths: lib_search_paths,
os: os,
maybe_ar_prog: maybe_ar_prog
let archive = Archive::new(config);
assert!(archive.dst.exists());
archive
}

/// Removes a file from this archive
pub fn remove_file(&mut self, file: &str) {
run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
}

/// Lists all files in an archive
pub fn files(&self) -> Vec<String> {
let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
let output = str::from_utf8(output.output.as_slice()).unwrap();
// use lines_any because windows delimits output with `\r\n` instead of
// just `\n`
output.lines_any().map(|s| s.to_string()).collect()
}

/// Creates an `ArchiveBuilder` for adding files to this archive.
pub fn extend(self) -> ArchiveBuilder<'a> {
ArchiveBuilder::new(self)
}
}

impl<'a> ArchiveBuilder<'a> {
fn new(archive: Archive<'a>) -> ArchiveBuilder<'a> {
ArchiveBuilder {
archive: archive,
work_dir: TempDir::new("rsar").unwrap(),
members: vec![],
should_update_symbols: false,
}
}

/// Create a new static archive, ready for adding files.
pub fn create(config: ArchiveConfig<'a>) -> ArchiveBuilder<'a> {
let archive = Archive::new(config);
ArchiveBuilder::new(archive)
}

/// Adds all of the contents of a native library to this archive. This will
/// search in the relevant locations for a library named `name`.
pub fn add_native_library(&mut self, name: &str) -> io::IoResult<()> {
Expand All @@ -135,48 +174,96 @@ impl<'a> Archive<'a> {
}

/// Adds an arbitrary file to this archive
pub fn add_file(&mut self, file: &Path, has_symbols: bool) {
let cmd = if has_symbols {"r"} else {"rS"};
run_ar(self.handler, &self.maybe_ar_prog, cmd, None, [&self.dst, file]);
}

/// Removes a file from this archive
pub fn remove_file(&mut self, file: &str) {
run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
pub fn add_file(&mut self, file: &Path) -> io::IoResult<()> {
let filename = Path::new(file.filename().unwrap());
let new_file = self.work_dir.path().join(&filename);
try!(fs::copy(file, &new_file));
self.members.push(filename);
Ok(())
}

/// Updates all symbols in the archive (runs 'ar s' over it)
/// Indicate that the next call to `build` should updates all symbols in
/// the archive (run 'ar s' over it).
pub fn update_symbols(&mut self) {
run_ar(self.handler, &self.maybe_ar_prog, "s", None, [&self.dst]);
self.should_update_symbols = true;
}

/// Lists all files in an archive
pub fn files(&self) -> Vec<String> {
let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
let output = str::from_utf8(output.output.as_slice()).unwrap();
// use lines_any because windows delimits output with `\r\n` instead of
// just `\n`
output.lines_any().map(|s| s.to_string()).collect()
/// Combine the provided files, rlibs, and native libraries into a single
/// `Archive`.
pub fn build(self) -> Archive<'a> {
// Get an absolute path to the destination, so `ar` will work even
// though we run it from `self.work_dir`.
let abs_dst = os::getcwd().join(&self.archive.dst);
assert!(!abs_dst.is_relative());
let mut args = vec![&abs_dst];
let mut total_len = abs_dst.as_vec().len();

if self.members.is_empty() {
// OSX `ar` does not allow using `r` with no members, but it does
// allow running `ar s file.a` to update symbols only.
if self.should_update_symbols {
run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
"s", Some(self.work_dir.path()), args.as_slice());
}
return self.archive;
}

// Don't allow the total size of `args` to grow beyond 32,000 bytes.
// Windows will raise an error if the argument string is longer than
// 32,768, and we leave a bit of extra space for the program name.
static ARG_LENGTH_LIMIT: uint = 32000;

for member_name in self.members.iter() {
let len = member_name.as_vec().len();

// `len + 1` to account for the space that's inserted before each
// argument. (Windows passes command-line arguments as a single
// string, not an array of strings.)
if total_len + len + 1 > ARG_LENGTH_LIMIT {
// Add the archive members seen so far, without updating the
// symbol table (`S`).
run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
"cruS", Some(self.work_dir.path()), args.as_slice());

args.clear();
args.push(&abs_dst);
total_len = abs_dst.as_vec().len();
}

args.push(member_name);
total_len += len + 1;
}

// Add the remaining archive members, and update the symbol table if
// necessary.
let flags = if self.should_update_symbols { "crus" } else { "cruS" };
run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
flags, Some(self.work_dir.path()), args.as_slice());

self.archive
}

fn add_archive(&mut self, archive: &Path, name: &str,
skip: &[&str]) -> io::IoResult<()> {
let loc = TempDir::new("rsar").unwrap();

// First, extract the contents of the archive to a temporary directory
// First, extract the contents of the archive to a temporary directory.
// We don't unpack directly into `self.work_dir` due to the possibility
// of filename collisions.
let archive = os::make_absolute(archive);
run_ar(self.handler, &self.maybe_ar_prog, "x", Some(loc.path()), [&archive]);
run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
"x", Some(loc.path()), [&archive]);

// Next, we must rename all of the inputs to "guaranteed unique names".
// The reason for this is that archives are keyed off the name of the
// files, so if two files have the same name they will override one
// another in the archive (bad).
// We move each file into `self.work_dir` under its new unique name.
// The reason for this renaming is that archives are keyed off the name
// of the files, so if two files have the same name they will override
// one another in the archive (bad).
//
// We skip any files explicitly desired for skipping, and we also skip
// all SYMDEF files as these are just magical placeholders which get
// re-created when we make a new archive anyway.
let files = try!(fs::readdir(loc.path()));
let mut inputs = Vec::new();
for file in files.iter() {
let filename = file.filename_str().unwrap();
if skip.iter().any(|s| *s == filename) { continue }
Expand All @@ -192,29 +279,23 @@ impl<'a> Archive<'a> {
} else {
filename
};
let new_filename = file.with_filename(filename);
let new_filename = self.work_dir.path().join(filename.as_slice());
try!(fs::rename(file, &new_filename));
inputs.push(new_filename);
self.members.push(Path::new(filename));
}
if inputs.len() == 0 { return Ok(()) }

// Finally, add all the renamed files to this archive
let mut args = vec!(&self.dst);
args.extend(inputs.iter());
run_ar(self.handler, &self.maybe_ar_prog, "r", None, args.as_slice());
Ok(())
}

fn find_library(&self, name: &str) -> Path {
let (osprefix, osext) = match self.os {
let (osprefix, osext) = match self.archive.os {
abi::OsWin32 => ("", "lib"), _ => ("lib", "a"),
};
// On Windows, static libraries sometimes show up as libfoo.a and other
// times show up as foo.lib
let oslibname = format!("{}{}.{}", osprefix, name, osext);
let unixlibname = format!("lib{}.a", name);

for path in self.lib_search_paths.iter() {
for path in self.archive.lib_search_paths.iter() {
debug!("looking for {} inside {}", name, path.display());
let test = path.join(oslibname.as_slice());
if test.exists() { return test }
Expand All @@ -223,9 +304,9 @@ impl<'a> Archive<'a> {
if test.exists() { return test }
}
}
self.handler.fatal(format!("could not find native static library `{}`, \
perhaps an -L flag is missing?",
name).as_slice());
self.archive.handler.fatal(format!("could not find native static library `{}`, \
perhaps an -L flag is missing?",
name).as_slice());
}
}