Use ValTree in all type level constants

Author: oli-obk

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -120,17 +120,19 @@ pub(crate) fn codegen_constant<'tcx>(
         ConstantKind::Ty(ct) => ct,
         ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
     };
-    let const_val = match const_.val {
-        ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => {
+    match const_.val {
+        ConstKind::Value(valtree) => codegen_valtree(fx, valtree, const_.ty, constant.span),
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
-            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
+            codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx)
         }
         ConstKind::Unevaluated(unevaluated) => {
             match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
-                Ok(const_val) => const_val,
+                Ok(const_val) => codegen_const_value(fx, const_val, const_.ty),
                 Err(_) => {
                     span_bug!(constant.span, "erroneous constant not captured by required_consts");
                 }
@@ -141,9 +143,53 @@ pub(crate) fn codegen_constant<'tcx>(
         | ConstKind::Bound(_, _)
         | ConstKind::Placeholder(_)
         | ConstKind::Error(_) => unreachable!("{:?}", const_),
+    }
+}
+
+pub(crate) fn codegen_valtree<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    valtree: ty::ValTree<'tcx>,
+    ty: Ty<'tcx>,
+    span: Span,
+) -> CValue<'tcx> {
+    let layout = fx.layout_of(ty);
+    let tcx = fx.tcx;
+    let mut encode_slice = |valtree: ty::ValTree<'_>| {
+        let s: Vec<u8> = valtree
+            .unwrap_branch()
+            .iter()
+            .map(|b| u8::try_from(b.unwrap_leaf()).unwrap())
+            .collect();
+        let alloc_id = fx.tcx.allocate_bytes(&s);
+
+        let ptr = pointer_for_alloc_id(fx, alloc_id, Mutability::Not).get_addr(fx);
+        let len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(s.len()).unwrap());
+        CValue::by_val_pair(ptr, len, layout)
     };
 
-    codegen_const_value(fx, const_val, const_.ty)
+    match *ty.kind() {
+        ty::Ref(_, pointee, _) => match *pointee.kind() {
+            ty::Str => encode_slice(valtree),
+            ty::Slice(elem_ty) if elem_ty == tcx.types.u8 => encode_slice(valtree),
+            ty::Array(elem_ty, _) if elem_ty == tcx.types.u8 => {
+                let s: Vec<u8> = valtree
+                    .unwrap_branch()
+                    .iter()
+                    .map(|b| u8::try_from(b.unwrap_leaf()).unwrap())
+                    .collect();
+                let alloc_id = fx.tcx.allocate_bytes(&s);
+                fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
+                let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, Mutability::Not);
+                let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                #[cfg(debug_assertions)]
+                fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+                let ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
+                CValue::by_val(ptr, layout)
+            }
+            _ => span_bug!(span, "{}: {:?}", ty, valtree),
+        },
+        _ => codegen_const_value(fx, ConstValue::Scalar(valtree.unwrap_leaf().into()), ty),
+    }
 }
 
 pub(crate) fn codegen_const_value<'tcx>(
@@ -237,8 +283,16 @@ fn pointer_for_allocation<'tcx>(
     alloc: &'tcx Allocation,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
+    pointer_for_alloc_id(fx, alloc_id, alloc.mutability)
+}
+
+fn pointer_for_alloc_id<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    alloc_id: AllocId,
+    mutability: Mutability,
+) -> crate::pointer::Pointer {
     fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
-    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
+    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     #[cfg(debug_assertions)]
@@ -420,17 +474,19 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
     assert!(cx.todo.is_empty(), "{:?}", cx.todo);
 }
 
-pub(crate) fn mir_operand_get_const_val<'tcx>(
+pub(crate) fn mir_operand_get_const_int<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
-) -> Option<ConstValue<'tcx>> {
+) -> Option<ty::ScalarInt> {
+    mir_operand_get_const(fx, operand)?.try_to_scalar_int()
+}
+
+pub(crate) fn mir_operand_get_const<'tcx>(
+    _fx: &FunctionCx<'_, '_, 'tcx>,
+    operand: &Operand<'tcx>,
+) -> Option<mir::ConstantKind<'tcx>> {
     match operand {
         Operand::Copy(_) | Operand::Move(_) => None,
-        Operand::Constant(const_) => match const_.literal {
-            ConstantKind::Ty(const_) => {
-                fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value()
-            }
-            ConstantKind::Val(val, _) => Some(val),
-        },
+        Operand::Constant(const_) => Some(const_.literal),
     }
 }

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -52,8 +52,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, res);
         };
         llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
-            let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
-            let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
+            let kind_const = crate::constant::mir_operand_get_const_int(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
+            let flt_cc = match kind_const.to_bits(Size::from_bytes(1)).unwrap_or_else(|_| panic!("kind not scalar: {:?}", kind_const)) {
                 0 => FloatCC::Equal,
                 1 => FloatCC::LessThan,
                 2 => FloatCC::LessThanOrEqual,
@@ -82,19 +82,19 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             });
         };
         llvm.x86.sse2.psrli.d, (c a, o imm8) {
-            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
+            let imm8 = crate::constant::mir_operand_get_const_int(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.to_bits(Size::from_bytes(4)).unwrap_or_else(|_| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };
                 CValue::by_val(res_lane, res_lane_layout)
             });
         };
         llvm.x86.sse2.pslli.d, (c a, o imm8) {
-            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
+            let imm8 = crate::constant::mir_operand_get_const_int(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.to_bits(Size::from_bytes(4)).unwrap_or_else(|_| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -82,24 +82,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             let total_len = lane_count * 2;
 
             let indexes = {
-                use rustc_middle::mir::interpret::*;
-                let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
-
-                let idx_bytes = match idx_const {
-                    ConstValue::ByRef { alloc, offset } => {
-                        let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
-                        alloc.get_bytes(fx, ptr, size).unwrap()
-                    }
+                let idx_const = crate::constant::mir_operand_get_const(fx, idx).expect("simd_shuffle* idx not const").const_for_ty().expect("simd shuffle arg must be structural-eq");
+
+                let idxs = match idx_const.val.try_to_value() {
+                    Some(ty::ValTree::Branch(branches)) => branches,
                     _ => unreachable!("{:?}", idx_const),
                 };
 
-                (0..ret_lane_count).map(|i| {
-                    let i = usize::try_from(i).unwrap();
-                    let idx = rustc_middle::mir::interpret::read_target_uint(
-                        fx.tcx.data_layout.endian,
-                        &idx_bytes[4*i.. 4*i + 4],
-                    ).expect("read_target_uint");
+                assert_eq!(u64::try_from(idxs.len()), Ok(ret_lane_count));
+                idxs.iter().map(|idx|{
+                    let idx = idx.unwrap_leaf();
+                    let idx = u32::try_from(idx).expect("simd_shuffle indices are u32");
                     u16::try_from(idx).expect("try_from u32")
                 }).collect::<Vec<u16>>()
             };
@@ -121,7 +114,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_insert, (c base, o idx, c val) {
             // FIXME validate
-            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
+            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_int(fx, idx) {
                 idx_const
             } else {
                 fx.tcx.sess.span_fatal(
@@ -130,7 +123,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 );
             };
 
-            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|_| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
@@ -143,7 +136,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         simd_extract, (c v, o idx) {
             validate_simd_type!(fx, intrinsic, span, v.layout().ty);
-            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
+            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_int(fx, idx) {
                 idx_const
             } else {
                 fx.tcx.sess.span_warn(
@@ -159,7 +152,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 return;
             };
 
-            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|_| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -12,11 +12,12 @@ use crate::MemFlags;
 use rustc_ast as ast;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
-use rustc_middle::mir::interpret::ConstValue;
+use rustc_middle::mir::interpret::{ConstValue, Scalar};
 use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::{self, SwitchTargets};
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
 use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::ValTree;
 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
@@ -634,7 +635,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         // promotes any complex rvalues to constants.
                         if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
                             if let mir::Operand::Constant(constant) = arg {
-                                let c = self.eval_mir_constant(constant);
+                                let c = self.eval_mir_constant(constant).map(Result::unwrap);
                                 let (llval, ty) =
                                     self.simd_shuffle_indices(&bx, constant.span, constant.ty(), c);
                                 return OperandRef {
@@ -829,7 +830,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         let ty = constant.ty();
                         let size = bx.layout_of(ty).size;
                         let scalar = match const_value {
-                            ConstValue::Scalar(s) => s,
+                            Ok(ConstValue::Scalar(Scalar::Int(s))) | Err(ValTree::Leaf(s)) => s,
                             _ => span_bug!(
                                 span,
                                 "expected Scalar for promoted asm const, but got {:#?}",
@@ -1152,7 +1153,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 caller.line as u32,
                 caller.col_display as u32 + 1,
             ));
-            OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
+            OperandRef::from_const(bx, Ok(const_loc), bx.tcx().caller_location_ty(), span)
         };
 
         // Walk up the `SourceScope`s, in case some of them are from MIR inlining.

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -6,6 +6,7 @@ use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::{self, Ty};
 use rustc_span::source_map::Span;
 use rustc_target::abi::Abi;
+use rustc_errors::ErrorReported;
 
 use super::FunctionCx;
 
@@ -17,17 +18,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> Result<OperandRef<'tcx, Bx::Value>, ErrorHandled> {
         let val = self.eval_mir_constant(constant)?;
         let ty = self.monomorphize(constant.ty());
-        Ok(OperandRef::from_const(bx, val, ty))
+        Ok(OperandRef::from_const(bx, val, ty, constant.span))
     }
 
     pub fn eval_mir_constant(
         &self,
         constant: &mir::Constant<'tcx>,
-    ) -> Result<ConstValue<'tcx>, ErrorHandled> {
+    ) -> Result<Result<ConstValue<'tcx>, ty::ValTree<'tcx>>, ErrorHandled> {
         let ct = self.monomorphize(constant.literal);
         let ct = match ct {
             mir::ConstantKind::Ty(ct) => ct,
-            mir::ConstantKind::Val(val, _) => return Ok(val),
+            mir::ConstantKind::Val(val, _) => return Ok(Ok(val)),
         };
         match ct.val {
             ty::ConstKind::Unevaluated(ct) => self
@@ -37,8 +38,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 .map_err(|err| {
                     self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
                     err
-                }),
-            ty::ConstKind::Value(value) => Ok(value),
+                })
+                .map(Ok),
+            ty::ConstKind::Value(value) => Ok(Err(value)),
+            // Already reported an error, nothing to do.
+            ty::ConstKind::Error(_) => Err(ErrorHandled::Reported(ErrorReported)),
             err => span_bug!(
                 constant.span,
                 "encountered bad ConstKind after monomorphizing: {:?}",

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -113,7 +113,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     .tcx()
                     .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
                     .unwrap();
-                OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx)
+                OperandRef::from_const(bx, Ok(value), ret_ty, span).immediate_or_packed_pair(bx)
             }
             sym::offset => {
                 let ptr = args[0].immediate();