fix likely and unlikely intrinsics

Author: x17jiri

compiler/rustc_borrowck/src/lib.rs

@@ -623,7 +623,10 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 );
             }
             StatementKind::Intrinsic(box kind) => match kind {
-                NonDivergingIntrinsic::Assume(op) => self.consume_operand(location, (op, span), flow_state),
+                NonDivergingIntrinsic::Assume(op)
+                | NonDivergingIntrinsic::Expect(op, ..)
+                => self.consume_operand(location, (op, span), flow_state),
+
                 NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
                     span,
                     "Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -52,7 +52,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
             StatementKind::FakeRead(box (_, _)) => {
                 // Only relevant for initialized/liveness/safety checks.
             }
-            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op))
+            | StatementKind::Intrinsic(box NonDivergingIntrinsic::Expect(op, ..)) => {
                 self.consume_operand(location, op);
             }
             StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1367,7 +1367,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
             }
             StatementKind::Intrinsic(box kind) => match kind {
-                NonDivergingIntrinsic::Assume(op) => self.check_operand(op, location),
+                NonDivergingIntrinsic::Assume(op) | NonDivergingIntrinsic::Expect(op, ..) => {
+                    self.check_operand(op, location)
+                }
                 NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
                     stmt.source_info.span,
                     "Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -438,11 +438,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
             fx.bcx.ins().trap(TrapCode::User(0));
             return;
         }
-        sym::likely | sym::unlikely => {
-            intrinsic_args!(fx, args => (a); intrinsic);
-
-            ret.write_cvalue(fx, a);
-        }
         sym::is_val_statically_known => {
             intrinsic_args!(fx, args => (_a); intrinsic);
 

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -117,12 +117,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     let func = unsafe { std::mem::transmute(simple.expect("simple")) };
                     self.call(self.type_void(), None, None, func, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None)
                 },
-                sym::likely => {
-                    self.expect(args[0].immediate(), true)
-                }
-                sym::unlikely => {
-                    self.expect(args[0].immediate(), false)
-                }
                 sym::is_val_statically_known => {
                     let a = args[0].immediate();
                     let builtin = self.context.get_builtin_function("__builtin_constant_p");

compiler/rustc_codegen_llvm/src/builder.rs

@@ -197,6 +197,49 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn cond_br_with_expect(
+        &mut self,
+        cond: &'ll Value,
+        then_llbb: &'ll BasicBlock,
+        else_llbb: &'ll BasicBlock,
+        expect: ExpectKind,
+    ) {
+        // emit the branch instruction
+        let n = unsafe { llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb) };
+
+        // emit expectation metadata
+        match expect {
+            ExpectKind::None => {}
+            ExpectKind::True | ExpectKind::False => unsafe {
+                let s = "branch_weights";
+                let v = [
+                    llvm::LLVMMDStringInContext(
+                        self.cx.llcx,
+                        s.as_ptr() as *const c_char,
+                        s.len() as c_uint,
+                    ),
+                    // 'then' branch weight
+                    self.cx.const_u32(if expect == ExpectKind::True { 2000 } else { 1 }),
+                    // 'else' branch weight
+                    self.cx.const_u32(if expect == ExpectKind::True { 1 } else { 2000 }),
+                ];
+                llvm::LLVMSetMetadata(
+                    n,
+                    llvm::MD_prof as c_uint,
+                    llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
+                );
+            },
+            ExpectKind::Unpredictable => unsafe {
+                let v: [&Value; 0] = [];
+                llvm::LLVMSetMetadata(
+                    n,
+                    llvm::MD_unpredictable as c_uint,
+                    llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
+                );
+            },
+        }
+    }
+
     fn switch(
         &mut self,
         v: &'ll Value,

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -116,9 +116,6 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     None,
                 )
             }
-            sym::likely => {
-                self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)])
-            }
             sym::is_val_statically_known => {
                 let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
                 match self.type_kind(intrinsic_type) {
@@ -131,8 +128,6 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     _ => self.const_bool(false),
                 }
             }
-            sym::unlikely => self
-                .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]),
             kw::Try => {
                 try_intrinsic(
                     self,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -427,6 +427,7 @@ pub enum MetadataType {
     MD_nontemporal = 9,
     MD_mem_parallel_loop_access = 10,
     MD_nonnull = 11,
+    MD_unpredictable = 15,
     MD_align = 17,
     MD_type = 19,
     MD_vcall_visibility = 28,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -315,6 +315,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
         bx: &mut Bx,
+        bb: mir::BasicBlock,
         discr: &mir::Operand<'tcx>,
         targets: &SwitchTargets,
     ) {
@@ -328,10 +329,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let lltrue = helper.llbb_with_cleanup(self, target);
             let llfalse = helper.llbb_with_cleanup(self, targets.otherwise());
             if switch_ty == bx.tcx().types.bool {
+                let expect = if let Some(x) = self.mir[bb].statements.last()
+                    && let mir::StatementKind::Intrinsic(box mir::NonDivergingIntrinsic::Expect(
+                        op,
+                        kind,
+                    )) = &x.kind
+                    && self.codegen_operand(bx, op).immediate() == discr.immediate()
+                {
+                    match kind {
+                        mir::ExpectKind::True => ExpectKind::True,
+                        mir::ExpectKind::False => ExpectKind::False,
+                        mir::ExpectKind::Unpredictable => ExpectKind::Unpredictable,
+                    }
+                } else {
+                    ExpectKind::None
+                };
+
                 // Don't generate trivial icmps when switching on bool.
                 match test_value {
-                    0 => bx.cond_br(discr.immediate(), llfalse, lltrue),
-                    1 => bx.cond_br(discr.immediate(), lltrue, llfalse),
+                    0 => bx.cond_br_with_expect(discr.immediate(), llfalse, lltrue, expect),
+                    1 => bx.cond_br_with_expect(discr.immediate(), lltrue, llfalse, expect.not()),
                     _ => bug!(),
                 }
             } else {
@@ -1218,7 +1235,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             mir::TerminatorKind::SwitchInt { ref discr, ref targets } => {
-                self.codegen_switchint_terminator(helper, bx, discr, targets);
+                self.codegen_switchint_terminator(helper, bx, bb, discr, targets);
                 MergingSucc::False
             }
 

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -70,6 +70,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let op_val = self.codegen_operand(bx, op);
                 bx.assume(op_val.immediate());
             }
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Expect(..)) => {}
             mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
                 mir::CopyNonOverlapping { ref count, ref src, ref dst },
             )) => {