Auto merge of #97485 - bjorn3:new_archive_writer, r=wesleywiser

Rewrite LLVM's archive writer in Rust

This allows it to be used by other codegen backends.

Fixes https://github.com/bjorn3/rustc_codegen_cranelift/issues/1155

Cargo.lock

@@ -92,6 +92,15 @@ version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"
 
+[[package]]
+name = "ar_archive_writer"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "276881980556fdadeb88aa1ffc667e4d2e8fe72531dfabcb7a82bb3c9ea9ba31"
+dependencies = [
+ "object",
+]
+
 [[package]]
 name = "array_tool"
 version = "1.0.3"
@@ -3495,6 +3504,7 @@ dependencies = [
 name = "rustc_codegen_ssa"
 version = "0.0.0"
 dependencies = [
+ "ar_archive_writer",
  "bitflags",
  "cc",
  "itertools",

LICENSES/LLVM-exception.txt

@@ -0,0 +1,15 @@
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+ As an exception, if, as a result of your compiling your source code, portions
+ of this Software are embedded into an Object form of such source code, you
+ may redistribute such embedded portions in such Object form without complying
+ with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+ In addition, if you combine or link compiled forms of this Software with
+ software that is licensed under the GPLv2 ("Combined Software") and if a
+ court of competent jurisdiction determines that the patent provision (Section
+ 3), the indemnity provision (Section 9) or other Section of the License
+ conflicts with the conditions of the GPLv2, you may retroactively and
+ prospectively choose to deem waived or otherwise exclude such Section(s) of
+ the License, but only in their entirety and only with respect to the Combined
+ Software.

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -19,11 +19,6 @@ version = "1.0.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c794e162a5eff65c72ef524dfe393eb923c354e350bb78b9c7383df13f3bc142"
 
-[[package]]
-name = "ar"
-version = "0.8.0"
-source = "git+https://github.com/bjorn3/rust-ar.git?branch=do_not_remove_cg_clif_ranlib#de9ab0e56bf3a208381d342aa5b60f9ff2891648"
-
 [[package]]
 name = "arrayvec"
 version = "0.7.2"
@@ -324,7 +319,6 @@ dependencies = [
 name = "rustc_codegen_cranelift"
 version = "0.1.0"
 dependencies = [
- "ar",
  "cranelift-codegen",
  "cranelift-frontend",
  "cranelift-jit",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -18,7 +18,6 @@ target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
-ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.9.1"
 libloading = { version = "0.7.3", optional = true }
 once_cell = "1.10.0"

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -1,35 +1,15 @@
-//! Creation of ar archives like for the lib and staticlib crate type
-
-use std::collections::BTreeMap;
-use std::fs::File;
-use std::io::{self, Read, Seek};
 use std::path::{Path, PathBuf};
 
-use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
+use rustc_codegen_ssa::back::archive::{
+ get_native_object_symbols, ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder,
+};
 use rustc_session::Session;
 
-use object::read::archive::ArchiveFile;
-use object::{Object, ObjectSymbol, ReadCache};
-
-#[derive(Debug)]
-enum ArchiveEntry {
- FromArchive { archive_index: usize, file_range: (u64, u64) },
- File(PathBuf),
-}
-
 pub(crate) struct ArArchiveBuilderBuilder;
 
 impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
  fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
- Box::new(ArArchiveBuilder {
- sess,
- use_gnu_style_archive: sess.target.archive_format == "gnu",
- // FIXME fix builtin ranlib on macOS
- no_builtin_ranlib: sess.target.is_like_osx,
-
- src_archives: vec![],
- entries: vec![],
- })
+ Box::new(ArArchiveBuilder::new(sess, get_native_object_symbols))
  }
 
  fn create_dll_import_lib(
@@ -40,200 +20,6 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
  _tmpdir: &Path,
  _is_direct_dependency: bool,
  ) -> PathBuf {
- bug!("creating dll imports is not supported");
- }
-}
-
-pub(crate) struct ArArchiveBuilder<'a> {
- sess: &'a Session,
- use_gnu_style_archive: bool,
- no_builtin_ranlib: bool,
-
- src_archives: Vec<File>,
- // Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
- // the end of an archive for linkers to not get confused.
- entries: Vec<(Vec<u8>, ArchiveEntry)>,
-}
-
-impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
- fn add_file(&mut self, file: &Path) {
- self.entries.push((
- file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(),
- ArchiveEntry::File(file.to_owned()),
- ));
- }
-
- fn add_archive(
- &mut self,
- archive_path: &Path,
- mut skip: Box<dyn FnMut(&str) -> bool + 'static>,
- ) -> std::io::Result<()> {
- let read_cache = ReadCache::new(std::fs::File::open(&archive_path)?);
- let archive = ArchiveFile::parse(&read_cache).unwrap();
- let archive_index = self.src_archives.len();
-
- for entry in archive.members() {
- let entry = entry.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
- let file_name = String::from_utf8(entry.name().to_vec())
- .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
- if !skip(&file_name) {
- self.entries.push((
- file_name.into_bytes(),
- ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
- ));
- }
- }
-
- self.src_archives.push(read_cache.into_inner());
- Ok(())
- }
-
- fn build(mut self: Box<Self>, output: &Path) -> bool {
- enum BuilderKind {
- Bsd(ar::Builder<File>),
- Gnu(ar::GnuBuilder<File>),
- }
-
- let sess = self.sess;
-
- let mut symbol_table = BTreeMap::new();
-
- let mut entries = Vec::new();
-
- for (mut entry_name, entry) in self.entries {
- // FIXME only read the symbol table of the object files to avoid having to keep all
- // object files in memory at once, or read them twice.
- let data = match entry {
- ArchiveEntry::FromArchive { archive_index, file_range } => {
- // FIXME read symbols from symtab
- let src_read_cache = &mut self.src_archives[archive_index];
-
- src_read_cache.seek(io::SeekFrom::Start(file_range.0)).unwrap();
- let mut data = std::vec::from_elem(0, usize::try_from(file_range.1).unwrap());
- src_read_cache.read_exact(&mut data).unwrap();
-
- data
- }
- ArchiveEntry::File(file) => std::fs::read(file).unwrap_or_else(|err| {
- sess.fatal(&format!(
- "error while reading object file during archive building: {}",
- err
- ));
- }),
- };
-
- if !self.no_builtin_ranlib {
- if symbol_table.contains_key(&entry_name) {
- // The ar crate can't handle creating a symbol table in case of multiple archive
- // members with the same name. Work around this by prepending a number until we
- // get a unique name.
- for i in 1.. {
- let new_name = format!("{}_", i)
- .into_bytes()
- .into_iter()
- .chain(entry_name.iter().copied())
- .collect::<Vec<_>>();
- if !symbol_table.contains_key(&new_name) {
- entry_name = new_name;
- break;
- }
- }
- }
-
- match object::File::parse(&*data) {
- Ok(object) => {
- symbol_table.insert(
- entry_name.to_vec(),
- object
- .symbols()
- .filter_map(|symbol| {
- if symbol.is_undefined() || symbol.is_local() {
- None
- } else {
- symbol.name().map(|name| name.as_bytes().to_vec()).ok()
- }
- })
- .collect::<Vec<_>>(),
- );
- }
- Err(err) => {
- let err = err.to_string();
- if err == "Unknown file magic" {
- // Not an object file; skip it.
- } else if object::read::archive::ArchiveFile::parse(&*data).is_ok() {
- // Nested archive file; skip it.
- } else {
- sess.fatal(&format!(
- "error parsing `{}` during archive creation: {}",
- String::from_utf8_lossy(&entry_name),
- err
- ));
- }
- }
- }
- }
-
- entries.push((entry_name, data));
- }
-
- let mut builder = if self.use_gnu_style_archive {
- BuilderKind::Gnu(
- ar::GnuBuilder::new(
- File::create(output).unwrap_or_else(|err| {
- sess.fatal(&format!(
- "error opening destination during archive building: {}",
- err
- ));
- }),
- entries.iter().map(|(name, _)| name.clone()).collect(),
- ar::GnuSymbolTableFormat::Size32,
- symbol_table,
- )
- .unwrap(),
- )
- } else {
- BuilderKind::Bsd(
- ar::Builder::new(
- File::create(output).unwrap_or_else(|err| {
- sess.fatal(&format!(
- "error opening destination during archive building: {}",
- err
- ));
- }),
- symbol_table,
- )
- .unwrap(),
- )
- };
-
- let any_members = !entries.is_empty();
-
- // Add all files
- for (entry_name, data) in entries.into_iter() {
- let header = ar::Header::new(entry_name, data.len() as u64);
- match builder {
- BuilderKind::Bsd(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
- BuilderKind::Gnu(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
- }
- }
-
- // Finalize archive
- std::mem::drop(builder);
-
- if self.no_builtin_ranlib {
- let ranlib = crate::toolchain::get_toolchain_binary(self.sess, "ranlib");
-
- // Run ranlib to be able to link the archive
- let status = std::process::Command::new(ranlib)
- .arg(output)
- .status()
- .expect("Couldn't run ranlib");
-
- if !status.success() {
- self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
- }
- }
-
- any_members
+ unimplemented!("creating dll imports is not yet supported");
  }
 }

compiler/rustc_codegen_gcc/Cargo.lock

@@ -11,12 +11,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "ar"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "450575f58f7bee32816abbff470cbc47797397c2a81e0eaced4b98436daf52e1"
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -212,10 +206,8 @@ dependencies = [
 name = "rustc_codegen_gcc"
 version = "0.1.0"
 dependencies = [
- "ar",
  "gccjit",
  "lang_tester",
- "target-lexicon",
  "tempfile",
 ]
 
@@ -228,12 +220,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "target-lexicon"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d"
-
 [[package]]
 name = "tempfile"
 version = "3.2.0"

compiler/rustc_codegen_gcc/Cargo.toml

@@ -27,10 +27,6 @@ gccjit = { git = "https://github.com/antoyo/gccjit.rs" }
 # Local copy.
 #gccjit = { path = "../gccjit.rs" }
 
-target-lexicon = "0.10.0"
-
-ar = "0.8.0"
-
 [dev-dependencies]
 lang_tester = "0.3.9"
 tempfile = "3.1.0"