Merge pull request #678 from rust-lang/remove-embed-lto-bitcode

Remove EMBED_LTO_BITCODE

Author: antoyo

.github/workflows/release.yml

@@ -62,7 +62,7 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        EMBED_LTO_BITCODE=1 ./y.sh build --sysroot --release --release-sysroot
+        ./y.sh build --sysroot --release --release-sysroot
         ./y.sh test --cargo-tests
         ./y.sh clean all
 
@@ -80,11 +80,11 @@ jobs:
         # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
         # FIXME(antoyo): this should probably not be needed since we embed the LTO bitcode.
         printf '[profile.release]\nlto = "fat"\n' >> build/build_sysroot/sysroot_src/library/Cargo.toml
-        EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot --keep-lto-tests ${{ matrix.commands }}
+        ./y.sh test --release --clean --release-sysroot --build-sysroot --keep-lto-tests ${{ matrix.commands }}
 
     - name: LTO test
       run: |
-        EMBED_LTO_BITCODE=1 CHANNEL="release" ./y.sh cargo build --release --manifest-path tests/hello-world/Cargo.toml
+        CHANNEL="release" ./y.sh cargo build --release --manifest-path tests/hello-world/Cargo.toml
         call_found=$(objdump -dj .text tests/hello-world/target/release/hello_world | grep -c "call .*mylib.*my_func" ) ||:
         if [ $call_found -gt 0 ]; then
           echo "ERROR: call my_func found in asm"

Cargo.lock

@@ -56,18 +56,18 @@ dependencies = [
 
 [[package]]
 name = "gccjit"
-version = "2.9.0"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a0e310ef75f396cd11b2443b353d55376656ca92c13cba36f92b7aff346ac1a"
+checksum = "60362e038e71e4bdc1a5b23fb45e1aba587b5947fe0db58f4871d95608f89eca"
 dependencies = [
  "gccjit_sys",
 ]
 
 [[package]]
 name = "gccjit_sys"
-version = "0.8.2"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95ed7572b30cd32430294dde6fb70822d58e67c6846a548647e8739776a0125b"
+checksum = "ddd542c8414e122217551c6af6b7d33acf51a227aee85276f218c087525e01bb"
 dependencies = [
  "libc",
 ]

Cargo.toml

@@ -24,7 +24,7 @@ default = ["master"]
 [dependencies]
 object = { version = "0.37.0", default-features = false, features = ["std", "read"] }
 tempfile = "3.20"
-gccjit = "2.8"
+gccjit = "2.10"
 #gccjit = { git = "https://github.com/rust-lang/gccjit.rs" }
 
 # Local copy.

Readme.md

@@ -141,16 +141,6 @@ $ CHANNEL="release" $CG_GCCJIT_DIR/y.sh cargo run
 
 If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./y.sh test`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely.
 
-### LTO
-
-To use LTO, you need to set the variable `EMBED_LTO_BITCODE=1` in addition to setting `lto = "fat"` in the `Cargo.toml`.
-
-Failing to set `EMBED_LTO_BITCODE` will give you the following error:
-
-```
-error: failed to copy bitcode to object file: No such file or directory (os error 2)
-```
-
 ### Rustc
 
 If you want to run `rustc` directly, you can do so with:

libgccjit.version

@@ -1 +1 @@
-0256a7539a83aa26aade8cca138d9c69634a51b1
+28b84db392ac0a572f1a2a2a1317aa5f2bc742cb

src/back/lto.rs

@@ -21,6 +21,7 @@ use std::ffi::{CStr, CString};
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::sync::atomic::Ordering;
 
 use gccjit::{Context, OutputKind};
 use object::read::archive::ArchiveFile;
@@ -38,7 +39,7 @@ use tempfile::{TempDir, tempdir};
 
 use crate::back::write::save_temp_bitcode;
 use crate::errors::LtoBitcodeFromRlib;
-use crate::{GccCodegenBackend, GccContext, SyncContext, to_gcc_opt_level};
+use crate::{GccCodegenBackend, GccContext, LTO_SUPPORTED, LtoMode, SyncContext, to_gcc_opt_level};
 
 struct LtoData {
     // TODO(antoyo): use symbols_below_threshold.
@@ -228,7 +229,7 @@ fn fat_lto(
             info!("linking {:?}", name);
             match bc_decoded {
                 SerializedModule::Local(ref module_buffer) => {
-                    module.module_llvm.should_combine_object_files = true;
+                    module.module_llvm.lto_mode = LtoMode::Fat;
                     module
                         .module_llvm
                         .context
@@ -533,7 +534,7 @@ pub fn optimize_thin_module(
     // that LLVM Context and Module.
     //let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
     //let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &dcx)? as *const _;
-    let mut should_combine_object_files = false;
+    let mut lto_mode = LtoMode::None;
     let context = match thin_module.shared.thin_buffers.get(thin_module.idx) {
         Some(thin_buffer) => Arc::clone(&thin_buffer.context),
         None => {
@@ -544,7 +545,7 @@ pub fn optimize_thin_module(
                 SerializedModule::Local(ref module_buffer) => {
                     let path = module_buffer.0.to_str().expect("path");
                     context.add_driver_option(path);
-                    should_combine_object_files = true;
+                    lto_mode = LtoMode::Thin;
                     /*module.module_llvm.should_combine_object_files = true;
                     module
                         .module_llvm
@@ -559,11 +560,13 @@ pub fn optimize_thin_module(
             Arc::new(SyncContext::new(context))
         }
     };
+    let lto_supported = LTO_SUPPORTED.load(Ordering::SeqCst);
     let module = ModuleCodegen::new_regular(
         thin_module.name().to_string(),
         GccContext {
             context,
-            should_combine_object_files,
+            lto_mode,
+            lto_supported,
             // TODO(antoyo): use the correct relocation model here.
             relocation_model: RelocModel::Pic,
             temp_dir: None,

src/back/write.rs

@@ -10,7 +10,7 @@ use rustc_target::spec::SplitDebuginfo;
 
 use crate::base::add_pic_option;
 use crate::errors::CopyBitcode;
-use crate::{GccCodegenBackend, GccContext};
+use crate::{GccCodegenBackend, GccContext, LtoMode};
 
 pub(crate) fn codegen(
     cgcx: &CodegenContext<GccCodegenBackend>,
@@ -24,12 +24,8 @@ pub(crate) fn codegen(
     {
         let context = &module.module_llvm.context;
 
-        let should_combine_object_files = module.module_llvm.should_combine_object_files;
-
-        // NOTE: Only generate object files with GIMPLE when this environment variable is set for
-        // now because this requires a particular setup (same gcc/lto1/lto-wrapper commit as libgccjit).
-        // TODO(antoyo): remove this environment variable.
-        let fat_lto = env::var("EMBED_LTO_BITCODE").as_deref() == Ok("1");
+        let lto_mode = module.module_llvm.lto_mode;
+        let lto_supported = module.module_llvm.lto_supported;
 
         let bc_out = cgcx.output_filenames.temp_path_for_cgu(
             OutputType::Bitcode,
@@ -43,80 +39,48 @@ pub(crate) fn codegen(
         );
 
         if config.bitcode_needed() {
-            if fat_lto {
+            let _timer = cgcx
+                .prof
+                .generic_activity_with_arg("GCC_module_codegen_make_bitcode", &*module.name);
+
+            // TODO(antoyo)
+            /*if let Some(bitcode_filename) = bc_out.file_name() {
+                cgcx.prof.artifact_size(
+                    "llvm_bitcode",
+                    bitcode_filename.to_string_lossy(),
+                    data.len() as u64,
+                );
+            }*/
+
+            if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
                 let _timer = cgcx
                     .prof
-                    .generic_activity_with_arg("GCC_module_codegen_make_bitcode", &*module.name);
-
-                // TODO(antoyo)
-                /*if let Some(bitcode_filename) = bc_out.file_name() {
-                    cgcx.prof.artifact_size(
-                        "llvm_bitcode",
-                        bitcode_filename.to_string_lossy(),
-                        data.len() as u64,
-                    );
-                }*/
-
-                if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
-                    let _timer = cgcx.prof.generic_activity_with_arg(
-                        "GCC_module_codegen_emit_bitcode",
-                        &*module.name,
-                    );
+                    .generic_activity_with_arg("GCC_module_codegen_emit_bitcode", &*module.name);
+                if lto_supported {
                     context.add_command_line_option("-flto=auto");
                     context.add_command_line_option("-flto-partition=one");
                     // TODO(antoyo): remove since we don't want fat objects when it is for Bitcode only.
                     context.add_command_line_option("-ffat-lto-objects");
-                    context.compile_to_file(
-                        OutputKind::ObjectFile,
-                        bc_out.to_str().expect("path to str"),
-                    );
                 }
+                context
+                    .compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
+            }
 
-                if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
-                    let _timer = cgcx.prof.generic_activity_with_arg(
-                        "GCC_module_codegen_embed_bitcode",
-                        &*module.name,
-                    );
+            if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
+                let _timer = cgcx
+                    .prof
+                    .generic_activity_with_arg("GCC_module_codegen_embed_bitcode", &*module.name);
+                if lto_supported {
                     // TODO(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes?
                     //embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);
 
                     context.add_command_line_option("-flto=auto");
                     context.add_command_line_option("-flto-partition=one");
                     context.add_command_line_option("-ffat-lto-objects");
-                    // TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
-                    context.compile_to_file(
-                        OutputKind::ObjectFile,
-                        bc_out.to_str().expect("path to str"),
-                    );
-                }
-            } else {
-                if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
-                    let _timer = cgcx.prof.generic_activity_with_arg(
-                        "GCC_module_codegen_emit_bitcode",
-                        &*module.name,
-                    );
-                    context.compile_to_file(
-                        OutputKind::ObjectFile,
-                        bc_out.to_str().expect("path to str"),
-                    );
-                }
-
-                if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
-                    // TODO(antoyo): we might want to emit to emit an error here, saying to set the
-                    // environment variable EMBED_LTO_BITCODE.
-                    let _timer = cgcx.prof.generic_activity_with_arg(
-                        "GCC_module_codegen_embed_bitcode",
-                        &*module.name,
-                    );
-                    // TODO(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes?
-                    //embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);
-
-                    // TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
-                    context.compile_to_file(
-                        OutputKind::ObjectFile,
-                        bc_out.to_str().expect("path to str"),
-                    );
                 }
+                // TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
+                context
+                    .compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
             }
         }
 
@@ -165,7 +129,10 @@ pub(crate) fn codegen(
                     context.set_debug_info(true);
                     context.dump_to_file(path, true);
                 }
-                if should_combine_object_files {
+                if lto_mode != LtoMode::None {
+                    let fat_lto = lto_mode == LtoMode::Fat;
+                    // We need to check if we're doing LTO since this code is also used for the
+                    // dummy ThinLTO implementation to combine the object files.
                     if fat_lto {
                         context.add_command_line_option("-flto=auto");
                         context.add_command_line_option("-flto-partition=one");

src/base.rs

@@ -21,7 +21,7 @@ use rustc_target::spec::SymbolVisibility;
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;
-use crate::{GccContext, LockedTargetInfo, SyncContext, gcc_util, new_context};
+use crate::{GccContext, LockedTargetInfo, LtoMode, SyncContext, gcc_util, new_context};
 
 #[cfg(feature = "master")]
 pub fn visibility_to_gcc(visibility: Visibility) -> gccjit::Visibility {
@@ -74,6 +74,7 @@ pub fn compile_codegen_unit(
     tcx: TyCtxt<'_>,
     cgu_name: Symbol,
     target_info: LockedTargetInfo,
+    lto_supported: bool,
 ) -> (ModuleCodegen<GccContext>, u64) {
     let prof_timer = tcx.prof.generic_activity("codegen_module");
     let start_time = Instant::now();
@@ -82,7 +83,7 @@ pub fn compile_codegen_unit(
     let (module, _) = tcx.dep_graph.with_task(
         dep_node,
         tcx,
-        (cgu_name, target_info),
+        (cgu_name, target_info, lto_supported),
         module_codegen,
         Some(dep_graph::hash_result),
     );
@@ -95,7 +96,7 @@ pub fn compile_codegen_unit(
 
     fn module_codegen(
         tcx: TyCtxt<'_>,
-        (cgu_name, target_info): (Symbol, LockedTargetInfo),
+        (cgu_name, target_info, lto_supported): (Symbol, LockedTargetInfo, bool),
     ) -> ModuleCodegen<GccContext> {
         let cgu = tcx.codegen_unit(cgu_name);
         // Instantiate monomorphizations without filling out definitions yet...
@@ -247,7 +248,8 @@ pub fn compile_codegen_unit(
             GccContext {
                 context: Arc::new(SyncContext::new(context)),
                 relocation_model: tcx.sess.relocation_model(),
-                should_combine_object_files: false,
+                lto_supported,
+                lto_mode: LtoMode::None,
                 temp_dir: None,
             },
         )