Auto merge of rust-lang#63352 - jgalenson:reproducible-lto, r=alexcrichton

Sort the fat LTO modules to produce deterministic output.

Some projects that use LTO for their release builds are not reproducible.  We can fix this by sorting the fat LTO modules before using them.

It might also be useful to do this for thin LTO, but I couldn't get that to work to test it so I didn't do it.

Author: bors

src/librustc_codegen_llvm/back/lto.rs

@@ -265,7 +265,7 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
         // and we want to move everything to the same LLVM context. Currently the
         // way we know of to do that is to serialize them to a string and them parse
         // them later. Not great but hey, that's why it's "fat" LTO, right?
-        serialized_modules.extend(modules.into_iter().map(|module| {
+        let mut new_modules = modules.into_iter().map(|module| {
             match module {
                 FatLTOInput::InMemory(module) => {
                     let buffer = ModuleBuffer::new(module.module_llvm.llmod());
@@ -277,7 +277,10 @@ fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
                     (SerializedModule::Local(buffer), llmod_id)
                 }
             }
-        }));
+        }).collect::<Vec<_>>();
+        // Sort the modules to ensure we produce deterministic results.
+        new_modules.sort_by(|module1, module2| module1.1.partial_cmp(&module2.1).unwrap());
+        serialized_modules.extend(new_modules);
         serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
             (buffer, CString::new(wp.cgu_name).unwrap())
         }));

src/test/run-make-fulldeps/reproducible-build-2/Makefile

@@ -0,0 +1,16 @@
+-include ../tools.mk
+
+# ignore-musl
+# ignore-windows
+# Objects are reproducible but their path is not.
+
+all:  \
+	fat_lto
+
+fat_lto:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
+	$(RUSTC) reproducible-build-aux.rs
+	$(RUSTC) reproducible-build.rs -C lto=fat
+	cp $(TMPDIR)/reproducible-build $(TMPDIR)/reproducible-build-a
+	$(RUSTC) reproducible-build.rs -C lto=fat
+	cmp "$(TMPDIR)/reproducible-build-a" "$(TMPDIR)/reproducible-build" || exit 1

src/test/run-make-fulldeps/reproducible-build-2/linker.rs

@@ -0,0 +1,44 @@
+use std::env;
+use std::path::Path;
+use std::fs::File;
+use std::io::{Read, Write};
+
+fn main() {
+    let mut dst = env::current_exe().unwrap();
+    dst.pop();
+    dst.push("linker-arguments1");
+    if dst.exists() {
+        dst.pop();
+        dst.push("linker-arguments2");
+        assert!(!dst.exists());
+    }
+
+    let mut out = String::new();
+    for arg in env::args().skip(1) {
+        let path = Path::new(&arg);
+        if !path.is_file() {
+            out.push_str(&arg);
+            out.push_str("\n");
+            continue
+        }
+
+        let mut contents = Vec::new();
+        File::open(path).unwrap().read_to_end(&mut contents).unwrap();
+
+        out.push_str(&format!("{}: {}\n", arg, hash(&contents)));
+    }
+
+    File::create(dst).unwrap().write_all(out.as_bytes()).unwrap();
+}
+
+// fnv hash for now
+fn hash(contents: &[u8]) -> u64 {
+    let mut hash = 0xcbf29ce484222325;
+
+    for byte in contents {
+        hash = hash ^ (*byte as u64);
+        hash = hash.wrapping_mul(0x100000001b3);
+    }
+
+    hash
+}

src/test/run-make-fulldeps/reproducible-build-2/reproducible-build-aux.rs

@@ -0,0 +1,28 @@
+#![crate_type="lib"]
+
+pub static STATIC: i32 = 1234;
+
+pub struct Struct<T1, T2> {
+    _t1: std::marker::PhantomData<T1>,
+    _t2: std::marker::PhantomData<T2>,
+}
+
+pub fn regular_fn(_: i32) {}
+
+pub fn generic_fn<T1, T2>() {}
+
+impl<T1, T2> Drop for Struct<T1, T2> {
+    fn drop(&mut self) {}
+}
+
+pub enum Enum {
+    Variant1,
+    Variant2(u32),
+    Variant3 { x: u32 }
+}
+
+pub struct TupleStruct(pub i8, pub i16, pub i32, pub i64);
+
+pub trait Trait<T1, T2> {
+    fn foo(&self);
+}

src/test/run-make-fulldeps/reproducible-build-2/reproducible-build.rs

@@ -0,0 +1,116 @@
+// This test case makes sure that two identical invocations of the compiler
+// (i.e., same code base, same compile-flags, same compiler-versions, etc.)
+// produce the same output. In the past, symbol names of monomorphized functions
+// were not deterministic (which we want to avoid).
+//
+// The test tries to exercise as many different paths into symbol name
+// generation as possible:
+//
+// - regular functions
+// - generic functions
+// - methods
+// - statics
+// - closures
+// - enum variant constructors
+// - tuple struct constructors
+// - drop glue
+// - FnOnce adapters
+// - Trait object shims
+// - Fn Pointer shims
+
+#![allow(dead_code, warnings)]
+
+extern crate reproducible_build_aux;
+
+static STATIC: i32 = 1234;
+
+pub struct Struct<T1, T2> {
+    x: T1,
+    y: T2,
+}
+
+fn regular_fn(_: i32) {}
+
+fn generic_fn<T1, T2>() {}
+
+impl<T1, T2> Drop for Struct<T1, T2> {
+    fn drop(&mut self) {}
+}
+
+pub enum Enum {
+    Variant1,
+    Variant2(u32),
+    Variant3 { x: u32 }
+}
+
+struct TupleStruct(i8, i16, i32, i64);
+
+impl TupleStruct {
+    pub fn bar(&self) {}
+}
+
+trait Trait<T1, T2> {
+    fn foo(&self);
+}
+
+impl Trait<i32, u64> for u64 {
+    fn foo(&self) {}
+}
+
+impl reproducible_build_aux::Trait<char, String> for TupleStruct {
+    fn foo(&self) {}
+}
+
+fn main() {
+    regular_fn(STATIC);
+    generic_fn::<u32, char>();
+    generic_fn::<char, Struct<u32, u64>>();
+    generic_fn::<Struct<u64, u32>, reproducible_build_aux::Struct<u32, u64>>();
+
+    let dropped = Struct {
+        x: "",
+        y: 'a',
+    };
+
+    let _ = Enum::Variant1;
+    let _ = Enum::Variant2(0);
+    let _ = Enum::Variant3 { x: 0 };
+    let _ = TupleStruct(1, 2, 3, 4);
+
+    let closure  = |x| {
+        x + 1i32
+    };
+
+    fn inner<F: Fn(i32) -> i32>(f: F) -> i32 {
+        f(STATIC)
+    }
+
+    println!("{}", inner(closure));
+
+    let object_shim: &Trait<i32, u64> = &0u64;
+    object_shim.foo();
+
+    fn with_fn_once_adapter<F: FnOnce(i32)>(f: F) {
+        f(0);
+    }
+
+    with_fn_once_adapter(|_:i32| { });
+
+    reproducible_build_aux::regular_fn(STATIC);
+    reproducible_build_aux::generic_fn::<u32, char>();
+    reproducible_build_aux::generic_fn::<char, Struct<u32, u64>>();
+    reproducible_build_aux::generic_fn::<Struct<u64, u32>,
+                                         reproducible_build_aux::Struct<u32, u64>>();
+
+    let _ = reproducible_build_aux::Enum::Variant1;
+    let _ = reproducible_build_aux::Enum::Variant2(0);
+    let _ = reproducible_build_aux::Enum::Variant3 { x: 0 };
+    let _ = reproducible_build_aux::TupleStruct(1, 2, 3, 4);
+
+    let object_shim: &reproducible_build_aux::Trait<char, String> = &TupleStruct(0, 1, 2, 3);
+    object_shim.foo();
+
+    let pointer_shim: &Fn(i32) = &regular_fn;
+
+    TupleStruct(1, 2, 3, 4).bar();
+}