Auto merge of rust-lang#84449 - alexcrichton:metadata-in-object, r=nagisa

rustc: Store metadata-in-rlibs in object files

This commit updates how rustc compiler metadata is stored in rlibs.
Previously metadata was stored as a raw file that has the same format as
`--emit metadata`. After this commit, however, the metadata is encoded
into a small object file which has one section which is the contents of
the metadata.

The motivation for this commit is to fix a common case where rust-lang#83730
arises. The problem is that when rustc crates a `dylib` crate type it
needs to include entire rlib files into the dylib, so it passes
`--whole-archive` (or the equivalent) to the linker. The problem with
this, though, is that the linker will attempt to read all files in the
archive. If the metadata file were left as-is (today) then the linker
would generate an error saying it can't read the file. The previous
solution was to alter the rlib just before linking, creating a new
archive in a temporary directory which has the metadata file removed.

This problem from before this commit is now removed if the metadata file
is stored in an object file that the linker can read. The only caveat we
have to take care of is to ensure that the linker never actually
includes the contents of the object file into the final output. We apply
similar tricks as the `.llvmbc` bytecode sections to do this.

This involved changing the metadata loading code a bit, namely updating
some of the LLVM C APIs used to use non-deprecated ones and fiddling
with the lifetimes a bit to get everything to work out. Otherwise though
this isn't intended to be a functional change really, only that metadata
is stored differently in archives now.

This should end up fixing rust-lang#83730 because by default dylibs will no
longer have their rlib dependencies "altered" meaning that
split-debuginfo will continue to have valid paths pointing at the
original rlibs. (note that we still "alter" rlibs if LTO is enabled to
remove Rust object files and we also "alter" for the #[link(cfg)]
feature, but that's rarely used).

Closes rust-lang#83730

Author: bors

Cargo.lock

@@ -2154,9 +2154,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
+checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
 [[package]]
 name = "memmap2"
@@ -2359,6 +2359,17 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "object"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8bc1d42047cf336f0f939c99e97183cf31551bf0f2865a2ec9c8d91fd4ffb5e"
+dependencies = [
+ "crc32fast",
+ "indexmap",
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.7.2"
@@ -3701,7 +3712,7 @@ dependencies = [
  "itertools 0.9.0",
  "jobserver",
  "libc",
- "object",
+ "object 0.25.2",
  "pathdiff",
  "rustc_apfloat",
  "rustc_ast",
@@ -4912,7 +4923,7 @@ dependencies = [
  "hermit-abi",
  "libc",
  "miniz_oxide",
- "object",
+ "object 0.22.0",
  "panic_abort",
  "panic_unwind",
  "profiler_builtins",

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -582,11 +582,6 @@ pub struct PassManager<'a>(InvariantOpaque<'a>);
 extern "C" {
     pub type PassManagerBuilder;
 }
-extern "C" {
-    pub type ObjectFile;
-}
-#[repr(C)]
-pub struct SectionIterator<'a>(InvariantOpaque<'a>);
 extern "C" {
     pub type Pass;
 }
@@ -1703,35 +1698,6 @@ extern "C" {
 
     pub fn LLVMDisposeMessage(message: *mut c_char);
 
-    // Stuff that's in llvm-wrapper/ because it's not upstream yet.
-
-    /// Opens an object file.
-    pub fn LLVMCreateObjectFile(
-        MemBuf: &'static mut MemoryBuffer,
-    ) -> Option<&'static mut ObjectFile>;
-    /// Closes an object file.
-    pub fn LLVMDisposeObjectFile(ObjFile: &'static mut ObjectFile);
-
-    /// Enumerates the sections in an object file.
-    pub fn LLVMGetSections(ObjFile: &'a ObjectFile) -> &'a mut SectionIterator<'a>;
-    /// Destroys a section iterator.
-    pub fn LLVMDisposeSectionIterator(SI: &'a mut SectionIterator<'a>);
-    /// Returns `true` if the section iterator is at the end of the section
-    /// list:
-    pub fn LLVMIsSectionIteratorAtEnd(ObjFile: &'a ObjectFile, SI: &SectionIterator<'a>) -> Bool;
-    /// Moves the section iterator to point to the next section.
-    pub fn LLVMMoveToNextSection(SI: &SectionIterator<'_>);
-    /// Returns the current section size.
-    pub fn LLVMGetSectionSize(SI: &SectionIterator<'_>) -> c_ulonglong;
-    /// Returns the current section contents as a string buffer.
-    pub fn LLVMGetSectionContents(SI: &SectionIterator<'_>) -> *const c_char;
-
-    /// Reads the given file and returns it as a memory buffer. Use
-    /// LLVMDisposeMemoryBuffer() to get rid of it.
-    pub fn LLVMRustCreateMemoryBufferWithContentsOfFile(
-        Path: *const c_char,
-    ) -> Option<&'static mut MemoryBuffer>;
-
     pub fn LLVMStartMultithreaded() -> Bool;
 
     /// Returns a string describing the last error caused by an LLVMRust* call.
@@ -2236,12 +2202,6 @@ extern "C" {
     pub fn LLVMRustArchiveIteratorFree(AIR: &'a mut ArchiveIterator<'a>);
     pub fn LLVMRustDestroyArchive(AR: &'static mut Archive);
 
-    #[allow(improper_ctypes)]
-    pub fn LLVMRustGetSectionName(
-        SI: &SectionIterator<'_>,
-        data: &mut Option<std::ptr::NonNull<c_char>>,
-    ) -> size_t;
-
     #[allow(improper_ctypes)]
     pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString);
 

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -150,50 +150,6 @@ impl Attribute {
     }
 }
 
-// Memory-managed interface to object files.
-
-pub struct ObjectFile {
-    pub llof: &'static mut ffi::ObjectFile,
-}
-
-unsafe impl Send for ObjectFile {}
-
-impl ObjectFile {
-    // This will take ownership of llmb
-    pub fn new(llmb: &'static mut MemoryBuffer) -> Option<ObjectFile> {
-        unsafe {
-            let llof = LLVMCreateObjectFile(llmb)?;
-            Some(ObjectFile { llof })
-        }
-    }
-}
-
-impl Drop for ObjectFile {
-    fn drop(&mut self) {
-        unsafe {
-            LLVMDisposeObjectFile(&mut *(self.llof as *mut _));
-        }
-    }
-}
-
-// Memory-managed interface to section iterators.
-
-pub struct SectionIter<'a> {
-    pub llsi: &'a mut SectionIterator<'a>,
-}
-
-impl Drop for SectionIter<'a> {
-    fn drop(&mut self) {
-        unsafe {
-            LLVMDisposeSectionIterator(&mut *(self.llsi as *mut _));
-        }
-    }
-}
-
-pub fn mk_section_iter(llof: &ffi::ObjectFile) -> SectionIter<'_> {
-    unsafe { SectionIter { llsi: LLVMGetSections(llof) } }
-}
-
 pub fn set_section(llglobal: &Value, section_name: &str) {
     let section_name_cstr = CString::new(section_name).expect("unexpected CString error");
     unsafe {

compiler/rustc_codegen_ssa/Cargo.toml

@@ -35,6 +35,6 @@ rustc_target = { path = "../rustc_target" }
 rustc_session = { path = "../rustc_session" }
 
 [dependencies.object]
-version = "0.22.0"
+version = "0.25.2"
 default-features = false
-features = ["read_core", "elf", "macho", "pe", "unaligned", "archive"]
+features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"]