Add codegen for global_asm! sym operands

Amanieu · Amanieu · commit c1fa773c8810 · 2022-04-04T02:44:57.000+01:00
diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs
@@ -258,9 +258,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 }
 
                 InlineAsmOperandRef::SymFn { instance } => {
+                    // TODO(@Amanieu): Additional mangling is needed on
+                    // some targets to add a leading underscore (Mach-O)
+                    // or byte count suffixes (x86 Windows).
                     constants_len += self.tcx.symbol_name(instance).name.len();
                 }
                 InlineAsmOperandRef::SymStatic { def_id } => {
+                    // TODO(@Amanieu): Additional mangling is needed on
+                    // some targets to add a leading underscore (Mach-O).
                     constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
             }
@@ -412,13 +417,16 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         }
 
                         InlineAsmOperandRef::SymFn { instance } => {
+                            // TODO(@Amanieu): Additional mangling is needed on
+                            // some targets to add a leading underscore (Mach-O)
+                            // or byte count suffixes (x86 Windows).
                             let name = self.tcx.symbol_name(instance).name;
                             template_str.push_str(name);
                         }
 
                         InlineAsmOperandRef::SymStatic { def_id } => {
-                            // TODO(@Commeownist): This may not be sufficient for all kinds of statics.
-                            // Some statics may need the `@plt` suffix, like thread-local vars.
+                            // TODO(@Amanieu): Additional mangling is needed on
+                            // some targets to add a leading underscore (Mach-O).
                             let instance = Instance::mono(self.tcx, def_id);
                             let name = self.tcx.symbol_name(instance).name;
                             template_str.push_str(name);
@@ -656,8 +664,8 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
     }
 }
 
-impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> {
-    fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef], options: InlineAsmOptions, _line_spans: &[Span]) {
+impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
+    fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
         // Default to Intel syntax on x86
@@ -690,6 +698,22 @@ impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> {
                             // here unlike normal inline assembly.
                             template_str.push_str(string);
                         }
+
+                        GlobalAsmOperandRef::SymFn { instance } => {
+                            // TODO(@Amanieu): Additional mangling is needed on
+                            // some targets to add a leading underscore (Mach-O)
+                            // or byte count suffixes (x86 Windows).
+                            let name = self.tcx.symbol_name(instance).name;
+                            template_str.push_str(name);
+                        }
+
+                        GlobalAsmOperandRef::SymStatic { def_id } => {
+                            // TODO(@Amanieu): Additional mangling is needed on
+                            // some targets to add a leading underscore (Mach-O).
+                            let instance = Instance::mono(self.tcx, def_id);
+                            let name = self.tcx.symbol_name(instance).name;
+                            template_str.push_str(name);
+                        }
                     }
                 }
             }
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -307,11 +307,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 }
 
-impl AsmMethods for CodegenCx<'_, '_> {
+impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn codegen_global_asm(
         &self,
         template: &[InlineAsmTemplatePiece],
-        operands: &[GlobalAsmOperandRef],
+        operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
     ) {
@@ -337,6 +337,29 @@ impl AsmMethods for CodegenCx<'_, '_> {
                             // here unlike normal inline assembly.
                             template_str.push_str(string);
                         }
+                        GlobalAsmOperandRef::SymFn { instance } => {
+                            let llval = self.get_fn(instance);
+                            self.add_compiler_used_global(llval);
+                            let symbol = llvm::build_string(|s| unsafe {
+                                llvm::LLVMRustGetMangledName(llval, s);
+                            })
+                            .expect("symbol is not valid UTF-8");
+                            template_str.push_str(&symbol);
+                        }
+                        GlobalAsmOperandRef::SymStatic { def_id } => {
+                            let llval = self
+                                .renamed_statics
+                                .borrow()
+                                .get(&def_id)
+                                .copied()
+                                .unwrap_or_else(|| self.get_static(def_id));
+                            self.add_compiler_used_global(llval);
+                            let symbol = llvm::build_string(|s| unsafe {
+                                llvm::LLVMRustGetMangledName(llval, s);
+                            })
+                            .expect("symbol is not valid UTF-8");
+                            template_str.push_str(&symbol);
+                        }
                     }
                 }
             }
diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs
@@ -99,7 +99,16 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
                 attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
             }
 
-            // Run replace-all-uses-with for statics that need it
+            // Create the llvm.used and llvm.compiler.used variables.
+            if !cx.used_statics().borrow().is_empty() {
+                cx.create_used_variable()
+            }
+            if !cx.compiler_used_statics().borrow().is_empty() {
+                cx.create_compiler_used_variable()
+            }
+
+            // Run replace-all-uses-with for statics that need it. This must
+            // happen after the llvm.used variables are created.
             for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() {
                 unsafe {
                     let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g));
@@ -114,14 +123,6 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
                 cx.coverageinfo_finalize();
             }
 
-            // Create the llvm.used and llvm.compiler.used variables.
-            if !cx.used_statics().borrow().is_empty() {
-                cx.create_used_variable()
-            }
-            if !cx.compiler_used_statics().borrow().is_empty() {
-                cx.create_compiler_used_variable()
-            }
-
             // Finalize debuginfo
             if cx.sess().opts.debuginfo != DebugInfo::None {
                 cx.debuginfo_finalize();
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -409,6 +409,13 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                 llvm::LLVMRustSetLinkage(new_g, linkage);
                 llvm::LLVMRustSetVisibility(new_g, visibility);
 
+                // The old global has had its name removed but is returned by
+                // get_static since it is in the instance cache. Provide an
+                // alternative lookup that points to the new global so that
+                // global_asm! can compute the correct mangled symbol name
+                // for the global.
+                self.renamed_statics.borrow_mut().insert(def_id, new_g);
+
                 // To avoid breaking any invariants, we leave around the old
                 // global for the moment; we'll replace all references to it
                 // with the new global later. (See base::codegen_backend.)
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
@@ -14,6 +14,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::base_n;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::small_c_str::SmallCStr;
+use rustc_hir::def_id::DefId;
 use rustc_middle::mir::mono::CodegenUnit;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers,
@@ -105,6 +106,12 @@ pub struct CodegenCx<'ll, 'tcx> {
 
     /// A counter that is used for generating local symbol names
     local_gen_sym_counter: Cell<usize>,
+
+    /// `codegen_static` will sometimes create a second global variable with a
+    /// different type and clear the symbol name of the original global.
+    /// `global_asm!` needs to be able to find this new global so that it can
+    /// compute the correct mangled symbol name to insert into the asm.
+    pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>,
 }
 
 pub struct TypeLowering<'ll> {
@@ -436,6 +443,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             rust_try_fn: Cell::new(None),
             intrinsics: Default::default(),
             local_gen_sym_counter: Cell::new(0),
+            renamed_statics: Default::default(),
         }
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2537,4 +2537,6 @@ extern "C" {
         remark_passes_len: usize,
     );
 
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustGetMangledName(V: &Value, out: &RustString);
 }
diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs
@@ -4,7 +4,9 @@ use crate::traits::*;
 use rustc_hir as hir;
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::mir::mono::{Linkage, Visibility};
+use rustc_middle::ty;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::Instance;
 
 pub trait MonoItemExt<'a, 'tcx> {
     fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx);
@@ -56,7 +58,27 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                                 );
                                 GlobalAsmOperandRef::Const { string }
                             }
-                            _ => span_bug!(*op_sp, "invalid operand type for global_asm!"),
+                            hir::InlineAsmOperand::SymFn { ref anon_const } => {
+                                let ty = cx
+                                    .tcx()
+                                    .typeck_body(anon_const.body)
+                                    .node_type(anon_const.hir_id);
+                                let instance = match ty.kind() {
+                                    &ty::FnDef(def_id, substs) => Instance::new(def_id, substs),
+                                    _ => span_bug!(*op_sp, "asm sym is not a function"),
+                                };
+
+                                GlobalAsmOperandRef::SymFn { instance }
+                            }
+                            hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
+                                GlobalAsmOperandRef::SymStatic { def_id }
+                            }
+                            hir::InlineAsmOperand::In { .. }
+                            | hir::InlineAsmOperand::Out { .. }
+                            | hir::InlineAsmOperand::InOut { .. }
+                            | hir::InlineAsmOperand::SplitInOut { .. } => {
+                                span_bug!(*op_sp, "invalid operand type for global_asm!")
+                            }
                         })
                         .collect();
 
diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs
@@ -36,8 +36,10 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
 }
 
 #[derive(Debug)]
-pub enum GlobalAsmOperandRef {
+pub enum GlobalAsmOperandRef<'tcx> {
     Const { string: String },
+    SymFn { instance: Instance<'tcx> },
+    SymStatic { def_id: DefId },
 }
 
 pub trait AsmBuilderMethods<'tcx>: BackendTypes {
@@ -53,11 +55,11 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
     );
 }
 
-pub trait AsmMethods {
+pub trait AsmMethods<'tcx> {
     fn codegen_global_asm(
         &self,
         template: &[InlineAsmTemplatePiece],
-        operands: &[GlobalAsmOperandRef],
+        operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
     );
diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs
@@ -60,7 +60,7 @@ pub trait CodegenMethods<'tcx>:
     + StaticMethods
     + CoverageInfoMethods<'tcx>
     + DebugInfoMethods<'tcx>
-    + AsmMethods
+    + AsmMethods<'tcx>
     + PreDefineMethods<'tcx>
     + HasParamEnv<'tcx>
     + HasTyCtxt<'tcx>
@@ -76,7 +76,7 @@ impl<'tcx, T> CodegenMethods<'tcx> for T where
         + StaticMethods
         + CoverageInfoMethods<'tcx>
         + DebugInfoMethods<'tcx>
-        + AsmMethods
+        + AsmMethods<'tcx>
         + PreDefineMethods<'tcx>
         + HasParamEnv<'tcx>
         + HasTyCtxt<'tcx>
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -1835,3 +1835,9 @@ extern "C" void LLVMRustContextConfigureDiagnosticHandler(
   unwrap(C)->setDiagnosticHandler(std::make_unique<RustDiagnosticHandler>(
       DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes));
 }
+
+extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) {
+  RawRustStringOstream OS(Str);
+  GlobalValue *GV = unwrap<GlobalValue>(V);
+  Mangler().getNameWithPrefix(OS, GV, true);
+}
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
@@ -435,7 +435,27 @@ fn collect_items_rec<'tcx>(
                             // are supported. Therefore the value should not
                             // depend on any other items.
                         }
-                        _ => span_bug!(*op_sp, "invalid operand type for global_asm!"),
+                        hir::InlineAsmOperand::SymFn { anon_const } => {
+                            let def_id = tcx.hir().body_owner_def_id(anon_const.body).to_def_id();
+                            if let Ok(val) = tcx.const_eval_poly(def_id) {
+                                rustc_data_structures::stack::ensure_sufficient_stack(|| {
+                                    collect_const_value(tcx, val, &mut neighbors);
+                                });
+                            }
+                        }
+                        hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
+                            let instance = Instance::mono(tcx, *def_id);
+                            if should_codegen_locally(tcx, &instance) {
+                                trace!("collecting static {:?}", def_id);
+                                neighbors.push(dummy_spanned(MonoItem::Static(*def_id)));
+                            }
+                        }
+                        hir::InlineAsmOperand::In { .. }
+                        | hir::InlineAsmOperand::Out { .. }
+                        | hir::InlineAsmOperand::InOut { .. }
+                        | hir::InlineAsmOperand::SplitInOut { .. } => {
+                            span_bug!(*op_sp, "invalid operand type for global_asm!")
+                        }
                     }
                 }
             } else {

Original file line number	Diff line number	Diff line change
`@@ -258,9 +258,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {`
`258`	`258`	`}`
`259`	`259`
`260`	`260`	`InlineAsmOperandRef::SymFn { instance } => {`
	`261`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`262`	`+ // some targets to add a leading underscore (Mach-O)`
	`263`	`+ // or byte count suffixes (x86 Windows).`
`261`	`264`	`constants_len += self.tcx.symbol_name(instance).name.len();`
`262`	`265`	`}`
`263`	`266`	`InlineAsmOperandRef::SymStatic { def_id } => {`
	`267`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`268`	`+ // some targets to add a leading underscore (Mach-O).`
`264`	`269`	`constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();`
`265`	`270`	`}`
`266`	`271`	`}`
`@@ -412,13 +417,16 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {`
`412`	`417`	`}`
`413`	`418`
`414`	`419`	`InlineAsmOperandRef::SymFn { instance } => {`
	`420`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`421`	`+ // some targets to add a leading underscore (Mach-O)`
	`422`	`+ // or byte count suffixes (x86 Windows).`
`415`	`423`	`let name = self.tcx.symbol_name(instance).name;`
`416`	`424`	`template_str.push_str(name);`
`417`	`425`	`}`
`418`	`426`
`419`	`427`	`InlineAsmOperandRef::SymStatic { def_id } => {`
`420`		`- // TODO(@Commeownist): This may not be sufficient for all kinds of statics.`
`421`		- // Some statics may need the `@plt` suffix, like thread-local vars.
	`428`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`429`	`+ // some targets to add a leading underscore (Mach-O).`
`422`	`430`	`let instance = Instance::mono(self.tcx, def_id);`
`423`	`431`	`let name = self.tcx.symbol_name(instance).name;`
`424`	`432`	`template_str.push_str(name);`
`@@ -656,8 +664,8 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl`
`656`	`664`	`}`
`657`	`665`	`}`
`658`	`666`
`659`		`-impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> {`
`660`		`- fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef], options: InlineAsmOptions, _line_spans: &[Span]) {`
	`667`	`+impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {`
	`668`	`+ fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) {`
`661`	`669`	`let asm_arch = self.tcx.sess.asm_arch.unwrap();`
`662`	`670`
`663`	`671`	`// Default to Intel syntax on x86`
`@@ -690,6 +698,22 @@ impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> {`
`690`	`698`	`// here unlike normal inline assembly.`
`691`	`699`	`template_str.push_str(string);`
`692`	`700`	`}`
	`701`	`+`
	`702`	`+ GlobalAsmOperandRef::SymFn { instance } => {`
	`703`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`704`	`+ // some targets to add a leading underscore (Mach-O)`
	`705`	`+ // or byte count suffixes (x86 Windows).`
	`706`	`+ let name = self.tcx.symbol_name(instance).name;`
	`707`	`+ template_str.push_str(name);`
	`708`	`+ }`
	`709`	`+`
	`710`	`+ GlobalAsmOperandRef::SymStatic { def_id } => {`
	`711`	`+ // TODO(@Amanieu): Additional mangling is needed on`
	`712`	`+ // some targets to add a leading underscore (Mach-O).`
	`713`	`+ let instance = Instance::mono(self.tcx, def_id);`
	`714`	`+ let name = self.tcx.symbol_name(instance).name;`
	`715`	`+ template_str.push_str(name);`
	`716`	`+ }`
`693`	`717`	`}`
`694`	`718`	`}`
`695`	`719`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2537,4 +2537,6 @@ extern "C" {`
`2537`	`2537`	`remark_passes_len: usize,`
`2538`	`2538`	`);`
`2539`	`2539`
	`2540`	`+ #[allow(improper_ctypes)]`
	`2541`	`+ pub fn LLVMRustGetMangledName(V: &Value, out: &RustString);`
`2540`	`2542`	`}`