Require CodegenCx::type_ptr_to callers to specify an address space at all times

Most of the time, the callers will be able to simply pass through the
address space of the pointer being operated on.

Author: dylanmckay

src/librustc_codegen_llvm/abi.rs

@@ -14,7 +14,7 @@ pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
 use rustc_middle::ty::Ty;
 use rustc_target::abi::call::ArgAbi;
 pub use rustc_target::abi::call::*;
-use rustc_target::abi::{self, HasDataLayout, Int, LayoutOf};
+use rustc_target::abi::{self, AddressSpace, HasDataLayout, Int, LayoutOf};
 pub use rustc_target::spec::abi::Abi;
 
 use libc::c_uint;
@@ -207,7 +207,8 @@ impl ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
             let can_store_through_cast_ptr = false;
             if can_store_through_cast_ptr {
-                let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx));
+                let cast_ptr_llty =
+                    bx.type_ptr_to(cast.llvm_type(bx), bx.cx().address_space_of_value(dst.llval));
                 let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty);
                 bx.store(val, cast_dst, self.layout.align.abi);
             } else {
@@ -323,7 +324,8 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
             PassMode::Cast(cast) => cast.llvm_type(cx),
             PassMode::Indirect(..) => {
-                llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
+                llargument_tys
+                    .push(cx.type_ptr_to(self.ret.memory_ty(cx), AddressSpace::default()));
                 cx.type_void()
             }
         };
@@ -350,7 +352,9 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     continue;
                 }
                 PassMode::Cast(cast) => cast.llvm_type(cx),
-                PassMode::Indirect(_, None) => cx.type_ptr_to(arg.memory_ty(cx)),
+                PassMode::Indirect(_, None) => {
+                    cx.type_ptr_to(arg.memory_ty(cx), AddressSpace::default())
+                }
             };
             llargument_tys.push(llarg_ty);
         }

src/librustc_codegen_llvm/builder.rs

@@ -719,7 +719,10 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         if flags.contains(MemFlags::NONTEMPORAL) {
             // HACK(nox): This is inefficient but there is no nontemporal memcpy.
             let val = self.load(src, src_align);
-            let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
+            let ptr = self.pointercast(
+                dst,
+                self.type_ptr_to(self.val_ty(val), self.cx.address_space_of_value(dst)),
+            );
             self.store_with_flags(val, ptr, dst_align, flags);
             return;
         }
@@ -752,7 +755,10 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         if flags.contains(MemFlags::NONTEMPORAL) {
             // HACK(nox): This is inefficient but there is no nontemporal memmove.
             let val = self.load(src, src_align);
-            let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
+            let ptr = self.pointercast(
+                dst,
+                self.type_ptr_to(self.val_ty(val), self.cx.address_space_of_value(dst)),
+            );
             self.store_with_flags(val, ptr, dst_align, flags);
             return;
         }
@@ -1191,7 +1197,7 @@ impl Builder<'a, 'll, 'tcx> {
     fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
         let dest_ptr_ty = self.cx.val_ty(ptr);
         let stored_ty = self.cx.val_ty(val);
-        let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
+        let stored_ptr_ty = self.cx.type_ptr_to(stored_ty, self.cx.address_space_of_value(ptr));
 
         assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);
 

src/librustc_codegen_llvm/common.rs

@@ -206,7 +206,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let len = s.as_str().len();
         let cs = consts::ptrcast(
             self.const_cstr(s, false),
-            self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)),
+            self.type_ptr_to(
+                self.layout_of(self.tcx.types.str_).llvm_type(self),
+                AddressSpace::default(),
+            ),
         );
         (cs, self.const_usize(len as u64))
     }
@@ -246,7 +249,8 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             Scalar::Ptr(ptr) => {
                 let base_addr = match self.tcx.global_alloc(ptr.alloc_id) {
                     GlobalAlloc::Memory(alloc) => {
-                        let init = const_alloc_to_llvm(self, alloc);
+                        let init =
+                            const_alloc_to_llvm(self, alloc, self.address_space_of_type(llty));
                         let value = match alloc.mutability {
                             Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
                             _ => self.static_addr_of(init, alloc.align, None),
@@ -265,7 +269,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 };
                 let llval = unsafe {
                     llvm::LLVMConstInBoundsGEP(
-                        self.const_bitcast(base_addr, self.type_i8p(AddressSpace::default())),
+                        self.const_bitcast(
+                            base_addr,
+                            self.type_i8p(self.address_space_of_value(base_addr)),
+                        ),
                         &self.const_usize(ptr.offset.bytes()),
                         1,
                     )
@@ -286,17 +293,22 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         offset: Size,
     ) -> PlaceRef<'tcx, &'ll Value> {
         assert_eq!(alloc.align, layout.align.abi);
-        let llty = self.type_ptr_to(layout.llvm_type(self));
+        let backend_type = layout.llvm_type(self);
+        let address_space = self.default_address_space_of_type(layout.ty);
+        let llty = self.type_ptr_to(backend_type, address_space);
         let llval = if layout.size == Size::ZERO {
             let llval = self.const_usize(alloc.align.bytes());
             unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
         } else {
-            let init = const_alloc_to_llvm(self, alloc);
+            let init = const_alloc_to_llvm(self, alloc, address_space);
             let base_addr = self.static_addr_of(init, alloc.align, None);
 
             let llval = unsafe {
                 llvm::LLVMConstInBoundsGEP(
-                    self.const_bitcast(base_addr, self.type_i8p(AddressSpace::default())),
+                    self.const_bitcast(
+                        base_addr,
+                        self.type_i8p(self.address_space_of_value(base_addr)),
+                    ),
                     &self.const_usize(offset.bytes()),
                     1,
                 )

src/librustc_codegen_llvm/consts.rs

@@ -24,7 +24,11 @@ use rustc_target::abi::{AddressSpace, Align, HasDataLayout, LayoutOf, Primitive,
 
 use std::ffi::CStr;
 
-pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
+pub fn const_alloc_to_llvm(
+    cx: &CodegenCx<'ll, '_>,
+    alloc: &Allocation,
+    address_space: AddressSpace,
+) -> &'ll Value {
     let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
@@ -56,7 +60,7 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
         llvals.push(cx.scalar_to_backend(
             Pointer::new(alloc_id, Size::from_bytes(ptr_offset)).into(),
             &Scalar { value: Primitive::Pointer, valid_range: 0..=!0 },
-            cx.type_i8p(AddressSpace::default()),
+            cx.type_i8p(address_space),
         ));
         next_offset = offset + pointer_size;
     }
@@ -79,11 +83,17 @@ pub fn codegen_static_initializer(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
 ) -> Result<(&'ll Value, &'tcx Allocation), ErrorHandled> {
+    let address_space = if cx.tcx.type_of(def_id).is_fn() {
+        cx.data_layout().instruction_address_space
+    } else {
+        AddressSpace::default()
+    };
+
     let alloc = match cx.tcx.const_eval_poly(def_id)? {
         ConstValue::ByRef { alloc, offset } if offset.bytes() == 0 => alloc,
         val => bug!("static const eval returned {:#?}", val),
     };
-    Ok((const_alloc_to_llvm(cx, alloc), alloc))
+    Ok((const_alloc_to_llvm(cx, alloc, address_space), alloc))
 }
 
 fn set_global_alignment(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) {
@@ -212,12 +222,15 @@ impl CodegenCx<'ll, 'tcx> {
         let g = if let Some(def_id) = def_id.as_local() {
             let id = self.tcx.hir().as_local_hir_id(def_id);
             let llty = self.layout_of(ty).llvm_type(self);
+
             // FIXME: refactor this to work without accessing the HIR
             let (g, attrs) = match self.tcx.hir().get(id) {
                 Node::Item(&hir::Item { attrs, span, kind: hir::ItemKind::Static(..), .. }) => {
                     let sym_str = sym.as_str();
                     if let Some(g) = self.get_declared_value(&sym_str) {
-                        if self.val_ty(g) != self.type_ptr_to(llty) {
+                        if self.val_ty(g)
+                            != self.type_ptr_to(llty, self.address_space_of_type(llty))
+                        {
                             span_bug!(span, "Conflicting types for static");
                         }
                     }
@@ -496,7 +509,8 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> {
 
             if attrs.flags.contains(CodegenFnAttrFlags::USED) {
                 // This static will be stored in the llvm.used variable which is an array of i8*
-                let cast = llvm::LLVMConstPointerCast(g, self.type_i8p(AddressSpace::default()));
+                let cast =
+                    llvm::LLVMConstPointerCast(g, self.type_i8p(self.address_space_of_value(g)));
                 self.used_statics.borrow_mut().push(cast);
             }
         }

src/librustc_codegen_llvm/context.rs

@@ -394,8 +394,10 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn create_used_variable(&self) {
         let name = const_cstr!("llvm.used");
         let section = const_cstr!("llvm.metadata");
-        let array =
-            self.const_array(&self.type_ptr_to(self.type_i8()), &*self.used_statics.borrow());
+        let array = self.const_array(
+            &self.type_ptr_to(self.type_i8(), AddressSpace::default()),
+            &*self.used_statics.borrow(),
+        );
 
         unsafe {
             let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());

src/librustc_codegen_llvm/intrinsic.rs

@@ -310,7 +310,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 let tp_ty = substs.type_at(0);
                 let mut ptr = args[0].immediate();
                 if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                    ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self)));
+                    ptr = self.pointercast(
+                        ptr,
+                        self.type_ptr_to(ty.llvm_type(self), self.cx().address_space_of_value(ptr)),
+                    );
                 }
                 let load = self.volatile_load(ptr);
                 let align = if name == "unaligned_volatile_load" {
@@ -753,7 +756,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
         if !fn_abi.ret.is_ignore() {
             if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                let ptr_llty = self.type_ptr_to(ty.llvm_type(self));
+                let ptr_llty = self.type_ptr_to(
+                    ty.llvm_type(self),
+                    self.cx().address_space_of_value(result.llval),
+                );
                 let ptr = self.pointercast(result.llval, ptr_llty);
                 self.store(llval, ptr, result.align);
             } else {
@@ -1040,7 +1046,7 @@ fn codegen_gnu_try(
         let tydesc = match bx.tcx().lang_items().eh_catch_typeinfo() {
             Some(tydesc) => {
                 let tydesc = bx.get_static(tydesc);
-                bx.bitcast(tydesc, bx.type_i8p(AddressSpace::default()))
+                bx.bitcast(tydesc, bx.type_i8p(bx.cx().address_space_of_value(tydesc)))
             }
             None => bx.const_null(bx.type_i8p(AddressSpace::default())),
         };
@@ -1534,7 +1540,7 @@ fn generic_simd_intrinsic(
             _ => unreachable!(),
         };
         while no_pointers > 0 {
-            elem_ty = cx.type_ptr_to(elem_ty);
+            elem_ty = cx.type_ptr_to(elem_ty, AddressSpace::default());
             no_pointers -= 1;
         }
         cx.type_vector(elem_ty, vec_len)

src/librustc_codegen_llvm/llvm/ffi.rs

@@ -1920,7 +1920,10 @@ extern "C" {
     pub fn LLVMRustDIBuilderCreateOpDeref() -> i64;
     pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64;
 
-    pub fn LLVMRustGetPointerTypeAddressSpace(ty: &Type) -> u32;
+    /// Returns `-1` if the value is not a pointer, otherwise returns the actual address space.
+    pub fn LLVMRustGetPointerTypeAddressSpace(ty: &Type) -> i64;
+    /// Returns `-1` if the value is not a pointer, otherwise returns the actual address space.
+    pub fn LLVMRustGetPointerAddressSpace(val: &Value) -> i64;
 
     #[allow(improper_ctypes)]
     pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString);

src/librustc_codegen_llvm/type_.rs

@@ -15,7 +15,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::Ty;
 use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
-use rustc_target::abi::{AddressSpace, Align, Integer, Size};
+use rustc_target::abi::{AddressSpace, Align, HasDataLayout, Integer, Size};
 
 use std::fmt;
 use std::ptr;
@@ -194,21 +194,36 @@ impl BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() }
     }
 
-    fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type {
-        assert_ne!(
-            self.type_kind(ty),
-            TypeKind::Function,
-            "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense"
-        );
-        ty.ptr_to(AddressSpace::default())
-    }
-
-    fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
+    fn type_ptr_to(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
+        if self.type_kind(ty) == TypeKind::Function {
+            assert!(
+                address_space == self.data_layout().instruction_address_space,
+                "attempted to create a function pointer to the wrong address space: {:?} ({:?})",
+                ty,
+                self.type_kind(ty)
+            );
+        }
         ty.ptr_to(address_space)
     }
 
     fn address_space_of_type(&self, ty: &'ll Type) -> AddressSpace {
-        ty.address_space()
+        let space_number = unsafe { llvm::LLVMRustGetPointerTypeAddressSpace(ty) };
+        assert_ne!(
+            space_number, -1,
+            "attempted to take the address space of a non-pointer type: {:?}",
+            ty
+        );
+        AddressSpace(space_number as u32)
+    }
+
+    fn address_space_of_value(&self, value: &'ll Value) -> AddressSpace {
+        let space_number = unsafe { llvm::LLVMRustGetPointerAddressSpace(value) };
+        assert_ne!(
+            space_number, -1,
+            "attempted to take the address space of a non-pointer value: {:?}",
+            value
+        );
+        AddressSpace(space_number as u32)
     }
 
     fn element_type(&self, ty: &'ll Type) -> &'ll Type {
@@ -252,11 +267,6 @@ impl Type {
         Type::i8_llcx(llcx).ptr_to(AddressSpace::default())
     }
 
-    pub fn address_space(&self) -> AddressSpace {
-        let space_number = unsafe { llvm::LLVMRustGetPointerTypeAddressSpace(self) };
-        AddressSpace(space_number)
-    }
-
     fn ptr_to(&self, address_space: AddressSpace) -> &Type {
         unsafe { llvm::LLVMPointerType(&self, address_space.0) }
     }

src/librustc_codegen_llvm/type_of.rs

@@ -7,7 +7,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::layout::{FnAbiExt, TyAndLayout};
 use rustc_middle::ty::print::obsolete::DefPathBasedNames;
 use rustc_middle::ty::{self, Ty, TypeFoldable};
-use rustc_target::abi::{Abi, Align, FieldsShape};
+use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
 use rustc_target::abi::{Int, Pointer, F32, F64};
 use rustc_target::abi::{LayoutOf, PointeeInfo, Scalar, Size, TyAndLayoutMethods, Variants};
 
@@ -238,11 +238,12 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             }
             let llty = match self.ty.kind {
                 ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => {
-                    cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx))
-                }
-                ty::Adt(def, _) if def.is_box() => {
-                    cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx))
+                    cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx), AddressSpace::default())
                 }
+                ty::Adt(def, _) if def.is_box() => cx.type_ptr_to(
+                    cx.layout_of(self.ty.boxed_ty()).llvm_type(cx),
+                    AddressSpace::default(),
+                ),
                 ty::FnPtr(sig) => cx.fn_ptr_backend_type(&FnAbi::of_fn_ptr(cx, sig, &[])),
                 _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO),
             };
@@ -315,7 +316,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
                 } else {
                     cx.type_i8()
                 };
-                cx.type_ptr_to(pointee)
+                cx.type_ptr_to(pointee, AddressSpace::default())
             }
         }
     }