Skip to content

Commit

Permalink
Auto merge of #97485 - bjorn3:new_archive_writer, r=wesleywiser
Browse files Browse the repository at this point in the history 
Rewrite LLVM's archive writer in Rust

This allows it to be used by other codegen backends.

Fixes https://github.com/bjorn3/rustc_codegen_cranelift/issues/1155
  • Loading branch information
@bors
bors committed Dec 3, 2022
2 parents 4bb1575 + a99838a commit cab4fd6
Show file tree
Hide file tree
Showing 24 changed files with 491 additions and 527 deletions.
10 changes: 10 additions & 0 deletions Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,15 @@ version = "1.0.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"

[[package]]
name = "ar_archive_writer"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "276881980556fdadeb88aa1ffc667e4d2e8fe72531dfabcb7a82bb3c9ea9ba31"
dependencies = [
"object",
]

[[package]]
name = "array_tool"
version = "1.0.3"
Expand Down Expand Up @@ -3495,6 +3504,7 @@ dependencies = [
name = "rustc_codegen_ssa"
version = "0.0.0"
dependencies = [
"ar_archive_writer",
"bitflags",
"cc",
"itertools",
Expand Down
15 changes: 15 additions & 0 deletions LICENSES/LLVM-exception.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---- LLVM Exceptions to the Apache 2.0 License ----

As an exception, if, as a result of your compiling your source code, portions
of this Software are embedded into an Object form of such source code, you
may redistribute such embedded portions in such Object form without complying
with the conditions of Sections 4(a), 4(b) and 4(d) of the License.

In addition, if you combine or link compiled forms of this Software with
software that is licensed under the GPLv2 ("Combined Software") and if a
court of competent jurisdiction determines that the patent provision (Section
3), the indemnity provision (Section 9) or other Section of the License
conflicts with the conditions of the GPLv2, you may retroactively and
prospectively choose to deem waived or otherwise exclude such Section(s) of
the License, but only in their entirety and only with respect to the Combined
Software.
6 changes: 0 additions & 6 deletions compiler/rustc_codegen_cranelift/Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@ version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c794e162a5eff65c72ef524dfe393eb923c354e350bb78b9c7383df13f3bc142"

[[package]]
name = "ar"
version = "0.8.0"
source = "git+https://github.com/bjorn3/rust-ar.git?branch=do_not_remove_cg_clif_ranlib#de9ab0e56bf3a208381d342aa5b60f9ff2891648"

[[package]]
name = "arrayvec"
version = "0.7.2"
Expand Down Expand Up @@ -324,7 +319,6 @@ dependencies = [
name = "rustc_codegen_cranelift"
version = "0.1.0"
dependencies = [
"ar",
"cranelift-codegen",
"cranelift-frontend",
"cranelift-jit",
Expand Down
1 change: 0 additions & 1 deletion compiler/rustc_codegen_cranelift/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ target-lexicon = "0.12.0"
gimli = { version = "0.26.0", default-features = false, features = ["write"]}
object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
indexmap = "1.9.1"
libloading = { version = "0.7.3", optional = true }
once_cell = "1.10.0"
Expand Down
224 changes: 5 additions & 219 deletions compiler/rustc_codegen_cranelift/src/archive.rs
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,15 @@
//! Creation of ar archives like for the lib and staticlib crate type

use std::collections::BTreeMap;
use std::fs::File;
use std::io::{self, Read, Seek};
use std::path::{Path, PathBuf};

use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
use rustc_codegen_ssa::back::archive::{
get_native_object_symbols, ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder,
};
use rustc_session::Session;

use object::read::archive::ArchiveFile;
use object::{Object, ObjectSymbol, ReadCache};

#[derive(Debug)]
enum ArchiveEntry {
FromArchive { archive_index: usize, file_range: (u64, u64) },
File(PathBuf),
}

pub(crate) struct ArArchiveBuilderBuilder;

impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
Box::new(ArArchiveBuilder {
sess,
use_gnu_style_archive: sess.target.archive_format == "gnu",
// FIXME fix builtin ranlib on macOS
no_builtin_ranlib: sess.target.is_like_osx,

src_archives: vec![],
entries: vec![],
})
Box::new(ArArchiveBuilder::new(sess, get_native_object_symbols))
}

fn create_dll_import_lib(
Expand All @@ -40,200 +20,6 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
_tmpdir: &Path,
_is_direct_dependency: bool,
) -> PathBuf {
bug!("creating dll imports is not supported");
}
}

pub(crate) struct ArArchiveBuilder<'a> {
sess: &'a Session,
use_gnu_style_archive: bool,
no_builtin_ranlib: bool,

src_archives: Vec<File>,
// Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
// the end of an archive for linkers to not get confused.
entries: Vec<(Vec<u8>, ArchiveEntry)>,
}

impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
fn add_file(&mut self, file: &Path) {
self.entries.push((
file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(),
ArchiveEntry::File(file.to_owned()),
));
}

fn add_archive(
&mut self,
archive_path: &Path,
mut skip: Box<dyn FnMut(&str) -> bool + 'static>,
) -> std::io::Result<()> {
let read_cache = ReadCache::new(std::fs::File::open(&archive_path)?);
let archive = ArchiveFile::parse(&read_cache).unwrap();
let archive_index = self.src_archives.len();

for entry in archive.members() {
let entry = entry.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
let file_name = String::from_utf8(entry.name().to_vec())
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
if !skip(&file_name) {
self.entries.push((
file_name.into_bytes(),
ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
));
}
}

self.src_archives.push(read_cache.into_inner());
Ok(())
}

fn build(mut self: Box<Self>, output: &Path) -> bool {
enum BuilderKind {
Bsd(ar::Builder<File>),
Gnu(ar::GnuBuilder<File>),
}

let sess = self.sess;

let mut symbol_table = BTreeMap::new();

let mut entries = Vec::new();

for (mut entry_name, entry) in self.entries {
// FIXME only read the symbol table of the object files to avoid having to keep all
// object files in memory at once, or read them twice.
let data = match entry {
ArchiveEntry::FromArchive { archive_index, file_range } => {
// FIXME read symbols from symtab
let src_read_cache = &mut self.src_archives[archive_index];

src_read_cache.seek(io::SeekFrom::Start(file_range.0)).unwrap();
let mut data = std::vec::from_elem(0, usize::try_from(file_range.1).unwrap());
src_read_cache.read_exact(&mut data).unwrap();

data
}
ArchiveEntry::File(file) => std::fs::read(file).unwrap_or_else(|err| {
sess.fatal(&format!(
"error while reading object file during archive building: {}",
err
));
}),
};

if !self.no_builtin_ranlib {
if symbol_table.contains_key(&entry_name) {
// The ar crate can't handle creating a symbol table in case of multiple archive
// members with the same name. Work around this by prepending a number until we
// get a unique name.
for i in 1.. {
let new_name = format!("{}_", i)
.into_bytes()
.into_iter()
.chain(entry_name.iter().copied())
.collect::<Vec<_>>();
if !symbol_table.contains_key(&new_name) {
entry_name = new_name;
break;
}
}
}

match object::File::parse(&*data) {
Ok(object) => {
symbol_table.insert(
entry_name.to_vec(),
object
.symbols()
.filter_map(|symbol| {
if symbol.is_undefined() || symbol.is_local() {
None
} else {
symbol.name().map(|name| name.as_bytes().to_vec()).ok()
}
})
.collect::<Vec<_>>(),
);
}
Err(err) => {
let err = err.to_string();
if err == "Unknown file magic" {
// Not an object file; skip it.
} else if object::read::archive::ArchiveFile::parse(&*data).is_ok() {
// Nested archive file; skip it.
} else {
sess.fatal(&format!(
"error parsing `{}` during archive creation: {}",
String::from_utf8_lossy(&entry_name),
err
));
}
}
}
}

entries.push((entry_name, data));
}

let mut builder = if self.use_gnu_style_archive {
BuilderKind::Gnu(
ar::GnuBuilder::new(
File::create(output).unwrap_or_else(|err| {
sess.fatal(&format!(
"error opening destination during archive building: {}",
err
));
}),
entries.iter().map(|(name, _)| name.clone()).collect(),
ar::GnuSymbolTableFormat::Size32,
symbol_table,
)
.unwrap(),
)
} else {
BuilderKind::Bsd(
ar::Builder::new(
File::create(output).unwrap_or_else(|err| {
sess.fatal(&format!(
"error opening destination during archive building: {}",
err
));
}),
symbol_table,
)
.unwrap(),
)
};

let any_members = !entries.is_empty();

// Add all files
for (entry_name, data) in entries.into_iter() {
let header = ar::Header::new(entry_name, data.len() as u64);
match builder {
BuilderKind::Bsd(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
BuilderKind::Gnu(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
}
}

// Finalize archive
std::mem::drop(builder);

if self.no_builtin_ranlib {
let ranlib = crate::toolchain::get_toolchain_binary(self.sess, "ranlib");

// Run ranlib to be able to link the archive
let status = std::process::Command::new(ranlib)
.arg(output)
.status()
.expect("Couldn't run ranlib");

if !status.success() {
self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
}
}

any_members
unimplemented!("creating dll imports is not yet supported");
}
}
14 changes: 0 additions & 14 deletions compiler/rustc_codegen_gcc/Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,6 @@ dependencies = [
"memchr",
]

[[package]]
name = "ar"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "450575f58f7bee32816abbff470cbc47797397c2a81e0eaced4b98436daf52e1"

[[package]]
name = "bitflags"
version = "1.3.2"
Expand Down Expand Up @@ -212,10 +206,8 @@ dependencies = [
name = "rustc_codegen_gcc"
version = "0.1.0"
dependencies = [
"ar",
"gccjit",
"lang_tester",
"target-lexicon",
"tempfile",
]

Expand All @@ -228,12 +220,6 @@ dependencies = [
"winapi-util",
]

[[package]]
name = "target-lexicon"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d"

[[package]]
name = "tempfile"
version = "3.2.0"
Expand Down
4 changes: 0 additions & 4 deletions compiler/rustc_codegen_gcc/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,6 @@ gccjit = { git = "https://github.com/antoyo/gccjit.rs" }
# Local copy.
#gccjit = { path = "../gccjit.rs" }

target-lexicon = "0.10.0"

ar = "0.8.0"

[dev-dependencies]
lang_tester = "0.3.9"
tempfile = "3.1.0"
Expand Down
Loading

0 comments on commit cab4fd6

Please sign in to comment.