deduplicate tail call terminator codegen into call terminator codegen

Author: xacrimon

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -160,6 +160,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         mut unwind: mir::UnwindAction,
         lifetime_ends_after_call: &[(Bx::Value, Size)],
         instance: Option<Instance<'tcx>>,
+        tail: bool,
         mergeable_succ: bool,
     ) -> MergingSucc {
         let tcx = bx.tcx();
@@ -252,8 +253,16 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             }
             MergingSucc::False
         } else {
-            let llret =
-                bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, llargs, self.funclet(fx), instance);
+            let llret = if !tail {
+                bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, llargs, self.funclet(fx), instance)
+            } else {
+                bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, llargs, None, instance)
+            };
+
+            if tail {
+                bx.set_tail_call(llret);
+            }
+
             if fx.mir[self.bb].is_cleanup {
                 bx.apply_attrs_to_cleanup_callsite(llret);
             }
@@ -264,6 +273,12 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 }
                 fx.store_return(bx, ret_dest, &fn_abi.ret, llret);
                 self.funclet_br(fx, bx, target, mergeable_succ)
+            } else if tail {
+                for &(tmp, size) in lifetime_ends_after_call {
+                    bx.lifetime_end(tmp, size);
+                }
+                bx.ret(llret);
+                MergingSucc::False
             } else {
                 bx.unreachable();
                 MergingSucc::False
@@ -659,6 +674,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             unwind,
             &[],
             Some(drop_instance),
+            false,
             !maybe_null && mergeable_succ,
         )
     }
@@ -747,8 +763,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let (fn_abi, llfn, instance) = common::build_langcall(bx, span, lang_item);
 
         // Codegen the actual panic invoke/call.
-        let merging_succ =
-            helper.do_call(self, bx, fn_abi, llfn, &args, None, unwind, &[], Some(instance), false);
+        let merging_succ = helper.do_call(
+            self,
+            bx,
+            fn_abi,
+            llfn,
+            &args,
+            None,
+            unwind,
+            &[],
+            Some(instance),
+            false,
+            false,
+        );
         assert_eq!(merging_succ, MergingSucc::False);
         MergingSucc::False
     }
@@ -778,6 +805,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             &[],
             Some(instance),
             false,
+            false,
         );
         assert_eq!(merging_succ, MergingSucc::False);
     }
@@ -845,6 +873,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             unwind,
             &[],
             Some(instance),
+            false,
             mergeable_succ,
         ))
     }
@@ -860,6 +889,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
         fn_span: Span,
+        tail: bool,
         mergeable_succ: bool,
     ) -> MergingSucc {
         let source_info = mir::SourceInfo { span: fn_span, ..terminator.source_info };
@@ -1003,8 +1033,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // We still need to call `make_return_dest` even if there's no `target`, since
         // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited,
         // and `make_return_dest` adds the return-place indirect pointer to `llargs`.
-        let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs);
-        let destination = target.map(|target| (return_dest, target));
+        let destination = if !tail {
+            let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs);
+            target.map(|target| (return_dest, target))
+        } else {
+            None
+        };
 
         // Split the rust-call tupled arguments off.
         let (first_args, untuple) = if sig.abi() == ExternAbi::RustCall
@@ -1147,106 +1181,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             unwind,
             &lifetime_ends_after_call,
             instance,
+            tail,
             mergeable_succ,
         )
     }
 
-    fn codegen_tail_call_terminator(
-        &mut self,
-        bx: &mut Bx,
-        func: &mir::Operand<'tcx>,
-        args: &[Spanned<mir::Operand<'tcx>>],
-        fn_span: Span,
-        mergeable_succ: bool,
-    ) -> MergingSucc {
-        // We don't need source_info as we already have fn_span for diagnostics
-        let func = self.codegen_operand(bx, func);
-        let fn_ty = func.layout.ty;
-
-        // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
-        let (fn_ptr, fn_abi, instance) = match *fn_ty.kind() {
-            ty::FnDef(def_id, substs) => {
-                let instance = ty::Instance::expect_resolve(
-                    bx.tcx(),
-                    bx.typing_env(),
-                    def_id,
-                    substs,
-                    fn_span,
-                );
-                let fn_ptr = bx.get_fn_addr(instance);
-                let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty());
-                (fn_ptr, fn_abi, Some(instance))
-            }
-            ty::FnPtr(..) => {
-                let sig = fn_ty.fn_sig(bx.tcx());
-                let extra_args = bx.tcx().mk_type_list(&[]);
-                let fn_ptr = func.immediate();
-                let fn_abi = bx.fn_abi_of_fn_ptr(sig, extra_args);
-                (fn_ptr, fn_abi, None)
-            }
-            _ => bug!("{} is not callable", func.layout.ty),
-        };
-
-        let mut llargs = Vec::with_capacity(args.len());
-        let mut lifetime_ends_after_call = Vec::new();
-
-        // Process arguments
-        for arg in args {
-            let op = self.codegen_operand(bx, &arg.node);
-            let arg_idx = llargs.len();
-
-            if arg_idx < fn_abi.args.len() {
-                self.codegen_argument(
-                    bx,
-                    op,
-                    &mut llargs,
-                    &fn_abi.args[arg_idx],
-                    &mut lifetime_ends_after_call,
-                );
-            } else {
-                // This can happen in case of C-variadic functions
-                let is_immediate = match op.val {
-                    Immediate(_) => true,
-                    _ => false,
-                };
-
-                if is_immediate {
-                    llargs.push(op.immediate());
-                } else {
-                    let temp = PlaceRef::alloca(bx, op.layout);
-                    op.val.store(bx, temp);
-                    llargs.push(bx.load(
-                        bx.backend_type(op.layout),
-                        temp.val.llval,
-                        temp.val.align,
-                    ));
-                }
-            }
-        }
-
-        // Call the function
-        let fn_ty = bx.fn_decl_backend_type(fn_abi);
-        let fn_attrs = if let Some(instance) = instance
-            && bx.tcx().def_kind(instance.def_id()).has_codegen_attrs()
-        {
-            Some(bx.tcx().codegen_fn_attrs(instance.def_id()))
-        } else {
-            None
-        };
-
-        // Perform the actual function call
-        let llret = bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, &llargs, None, instance);
-
-        // Mark as tail call - this is the critical part
-        bx.set_tail_call(llret);
-
-        // Return the result - musttail requires ret immediately after the call
-        bx.ret(llret);
-
-        assert_eq!(mergeable_succ, false);
-        MergingSucc::False
-    }
-
     fn codegen_asm_terminator(
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
@@ -1484,10 +1423,27 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 target,
                 unwind,
                 fn_span,
+                false,
                 mergeable_succ(),
             ),
             mir::TerminatorKind::TailCall { ref func, ref args, fn_span } => {
-                self.codegen_tail_call_terminator(bx, func, args, fn_span, mergeable_succ())
+                let destination = mir::Place::from(mir::RETURN_PLACE);
+                let target = None;
+                let unwind = mir::UnwindAction::Unreachable;
+
+                self.codegen_call_terminator(
+                    helper,
+                    bx,
+                    terminator,
+                    func,
+                    args,
+                    destination,
+                    target,
+                    unwind,
+                    fn_span,
+                    true,
+                    mergeable_succ(),
+                )
             }
             mir::TerminatorKind::CoroutineDrop | mir::TerminatorKind::Yield { .. } => {
                 bug!("coroutine ops in codegen")