Skip to content

Commit

Permalink
Auto merge of #63352 - jgalenson:reproducible-lto, r=alexcrichton
Browse files Browse the repository at this point in the history
Sort the fat LTO modules to produce deterministic output.

Some projects that use LTO for their release builds are not reproducible.  We can fix this by sorting the fat LTO modules before using them.

It might also be useful to do this for thin LTO, but I couldn't get that to work to test it so I didn't do it.
  • Loading branch information
bors committed Aug 10, 2019
2 parents d19a359 + b6767b3 commit 6f70adc
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 2 deletions.
7 changes: 5 additions & 2 deletions src/librustc_codegen_llvm/back/lto.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
// and we want to move everything to the same LLVM context. Currently the
// way we know of to do that is to serialize them to a string and them parse
// them later. Not great but hey, that's why it's "fat" LTO, right?
serialized_modules.extend(modules.into_iter().map(|module| {
let mut new_modules = modules.into_iter().map(|module| {
match module {
FatLTOInput::InMemory(module) => {
let buffer = ModuleBuffer::new(module.module_llvm.llmod());
Expand All @@ -277,7 +277,10 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
(SerializedModule::Local(buffer), llmod_id)
}
}
}));
}).collect::<Vec<_>>();
// Sort the modules to ensure we produce deterministic results.
new_modules.sort_by(|module1, module2| module1.1.partial_cmp(&module2.1).unwrap());
serialized_modules.extend(new_modules);
serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
(buffer, CString::new(wp.cgu_name).unwrap())
}));
Expand Down
16 changes: 16 additions & 0 deletions src/test/run-make-fulldeps/reproducible-build-2/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-include ../tools.mk

# ignore-musl
# ignore-windows
# Objects are reproducible but their path is not.

all: \
fat_lto

fat_lto:
rm -rf $(TMPDIR) && mkdir $(TMPDIR)
$(RUSTC) reproducible-build-aux.rs
$(RUSTC) reproducible-build.rs -C lto=fat
cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a
$(RUSTC) reproducible-build.rs -C lto=fat
cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1
44 changes: 44 additions & 0 deletions src/test/run-make-fulldeps/reproducible-build-2/linker.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
use std::env;
use std::path::Path;
use std::fs::File;
use std::io::{Read, Write};

fn main() {
let mut dst = env::current_exe().unwrap();
dst.pop();
dst.push("linker-arguments1");
if dst.exists() {
dst.pop();
dst.push("linker-arguments2");
assert!(!dst.exists());
}

let mut out = String::new();
for arg in env::args().skip(1) {
let path = Path::new(&arg);
if !path.is_file() {
out.push_str(&arg);
out.push_str("\n");
continue
}

let mut contents = Vec::new();
File::open(path).unwrap().read_to_end(&mut contents).unwrap();

out.push_str(&format!("{}: {}\n", arg, hash(&contents)));
}

File::create(dst).unwrap().write_all(out.as_bytes()).unwrap();
}

// fnv hash for now
fn hash(contents: &[u8]) -> u64 {
let mut hash = 0xcbf29ce484222325;

for byte in contents {
hash = hash ^ (*byte as u64);
hash = hash.wrapping_mul(0x100000001b3);
}

hash
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#![crate_type="lib"]

pub static STATIC: i32 = 1234;

pub struct Struct<T1, T2> {
_t1: std::marker::PhantomData<T1>,
_t2: std::marker::PhantomData<T2>,
}

pub fn regular_fn(_: i32) {}

pub fn generic_fn<T1, T2>() {}

impl<T1, T2> Drop for Struct<T1, T2> {
fn drop(&mut self) {}
}

pub enum Enum {
Variant1,
Variant2(u32),
Variant3 { x: u32 }
}

pub struct TupleStruct(pub i8, pub i16, pub i32, pub i64);

pub trait Trait<T1, T2> {
fn foo(&self);
}
116 changes: 116 additions & 0 deletions src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// This test case makes sure that two identical invocations of the compiler
// (i.e., same code base, same compile-flags, same compiler-versions, etc.)
// produce the same output. In the past, symbol names of monomorphized functions
// were not deterministic (which we want to avoid).
//
// The test tries to exercise as many different paths into symbol name
// generation as possible:
//
// - regular functions
// - generic functions
// - methods
// - statics
// - closures
// - enum variant constructors
// - tuple struct constructors
// - drop glue
// - FnOnce adapters
// - Trait object shims
// - Fn Pointer shims

#![allow(dead_code, warnings)]

extern crate reproducible_build_aux;

static STATIC: i32 = 1234;

pub struct Struct<T1, T2> {
x: T1,
y: T2,
}

fn regular_fn(_: i32) {}

fn generic_fn<T1, T2>() {}

impl<T1, T2> Drop for Struct<T1, T2> {
fn drop(&mut self) {}
}

pub enum Enum {
Variant1,
Variant2(u32),
Variant3 { x: u32 }
}

struct TupleStruct(i8, i16, i32, i64);

impl TupleStruct {
pub fn bar(&self) {}
}

trait Trait<T1, T2> {
fn foo(&self);
}

impl Trait<i32, u64> for u64 {
fn foo(&self) {}
}

impl reproducible_build_aux::Trait<char, String> for TupleStruct {
fn foo(&self) {}
}

fn main() {
regular_fn(STATIC);
generic_fn::<u32, char>();
generic_fn::<char, Struct<u32, u64>>();
generic_fn::<Struct<u64, u32>, reproducible_build_aux::Struct<u32, u64>>();

let dropped = Struct {
x: "",
y: 'a',
};

let _ = Enum::Variant1;
let _ = Enum::Variant2(0);
let _ = Enum::Variant3 { x: 0 };
let _ = TupleStruct(1, 2, 3, 4);

let closure = |x| {
x + 1i32
};

fn inner<F: Fn(i32) -> i32>(f: F) -> i32 {
f(STATIC)
}

println!("{}", inner(closure));

let object_shim: &Trait<i32, u64> = &0u64;
object_shim.foo();

fn with_fn_once_adapter<F: FnOnce(i32)>(f: F) {
f(0);
}

with_fn_once_adapter(|_:i32| { });

reproducible_build_aux::regular_fn(STATIC);
reproducible_build_aux::generic_fn::<u32, char>();
reproducible_build_aux::generic_fn::<char, Struct<u32, u64>>();
reproducible_build_aux::generic_fn::<Struct<u64, u32>,
reproducible_build_aux::Struct<u32, u64>>();

let _ = reproducible_build_aux::Enum::Variant1;
let _ = reproducible_build_aux::Enum::Variant2(0);
let _ = reproducible_build_aux::Enum::Variant3 { x: 0 };
let _ = reproducible_build_aux::TupleStruct(1, 2, 3, 4);

let object_shim: &reproducible_build_aux::Trait<char, String> = &TupleStruct(0, 1, 2, 3);
object_shim.foo();

let pointer_shim: &Fn(i32) = &regular_fn;

TupleStruct(1, 2, 3, 4).bar();
}

0 comments on commit 6f70adc

Please sign in to comment.