Rollup of 9 pull requests

src/bootstrap/builder.rs

@@ -1239,7 +1239,7 @@ impl<'a> Builder<'a> {
             && self.config.control_flow_guard
             && compiler.stage >= 1
         {
-            rustflags.arg("-Zcontrol-flow-guard");
+            rustflags.arg("-Ccontrol-flow-guard");
         }
 
         // For `cargo doc` invocations, make rustdoc print the Rust version into the docs

src/doc/rustc/src/codegen-options/index.md

@@ -42,6 +42,18 @@ generated code, but may be slower to compile.
 The default value, if not specified, is 16 for non-incremental builds. For
 incremental builds the default is 256 which allows caching to be more granular.
 
+## control-flow-guard
+
+This flag controls whether LLVM enables the Windows [Control Flow 
+Guard](https://docs.microsoft.com/en-us/windows/win32/secbp/control-flow-guard) 
+platform security feature. This flag is currently ignored for non-Windows targets. 
+It takes one of the following values:
+
+* `y`, `yes`, `on`, `checks`, or no value: enable Control Flow Guard.
+* `nochecks`: emit Control Flow Guard metadata without runtime enforcement checks (this 
+should only be used for testing purposes as it does not provide security enforcement).
+* `n`, `no`, `off`: do not enable Control Flow Guard (the default).
+
 ## debug-assertions
 
 This flag lets you turn `cfg(debug_assertions)` [conditional

src/libcore/ptr/mut_ptr.rs

@@ -47,17 +47,22 @@ impl<T: ?Sized> *mut T {
     /// operation because the returned value could be pointing to invalid
     /// memory.
     ///
-    /// When calling this method, you have to ensure that if the pointer is
-    /// non-NULL, then it is properly aligned, dereferenceable (for the whole
-    /// size of `T`) and points to an initialized instance of `T`. This applies
-    /// even if the result of this method is unused!
+    /// When calling this method, you have to ensure that *either* the pointer is NULL *or*
+    /// all of the following is true:
+    /// - it is properly aligned
+    /// - it must point to an initialized instance of T; in particular, the pointer must be
+    ///   "dereferencable" in the sense defined [here].
+    ///
+    /// This applies even if the result of this method is unused!
     /// (The part about being initialized is not yet fully decided, but until
     /// it is, the only safe approach is to ensure that they are indeed initialized.)
     ///
     /// Additionally, the lifetime `'a` returned is arbitrarily chosen and does
-    /// not necessarily reflect the actual lifetime of the data. It is up to the
-    /// caller to ensure that for the duration of this lifetime, the memory this
-    /// pointer points to does not get written to outside of `UnsafeCell<U>`.
+    /// not necessarily reflect the actual lifetime of the data. *You* must enforce
+    /// Rust's aliasing rules. In particular, for the duration of this lifetime,
+    /// the memory the pointer points to must not get mutated (except inside `UnsafeCell`).
+    ///
+    /// [here]: crate::ptr#safety
     ///
     /// # Examples
     ///

src/librustc_codegen_llvm/abi.rs

@@ -366,7 +366,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         unsafe {
             llvm::LLVMPointerType(
                 self.llvm_type(cx),
-                cx.data_layout().instruction_address_space as c_uint,
+                cx.data_layout().instruction_address_space.0 as c_uint,
             )
         }
     }

src/librustc_codegen_llvm/builder.rs

@@ -1330,7 +1330,12 @@ impl Builder<'a, 'll, 'tcx> {
         self.call(lifetime_intrinsic, &[self.cx.const_u64(size), ptr], None);
     }
 
-    fn phi(&mut self, ty: &'ll Type, vals: &[&'ll Value], bbs: &[&'ll BasicBlock]) -> &'ll Value {
+    pub(crate) fn phi(
+        &mut self,
+        ty: &'ll Type,
+        vals: &[&'ll Value],
+        bbs: &[&'ll BasicBlock],
+    ) -> &'ll Value {
         assert_eq!(vals.len(), bbs.len());
         let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
         unsafe {

src/librustc_codegen_llvm/common.rs

@@ -16,7 +16,7 @@ use rustc_middle::bug;
 use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_span::symbol::Symbol;
-use rustc_target::abi::{self, HasDataLayout, LayoutOf, Pointer, Size};
+use rustc_target::abi::{self, AddressSpace, HasDataLayout, LayoutOf, Pointer, Size};
 
 use libc::{c_char, c_uint};
 use log::debug;
@@ -244,7 +244,7 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 }
             }
             Scalar::Ptr(ptr) => {
-                let base_addr = match self.tcx.global_alloc(ptr.alloc_id) {
+                let (base_addr, base_addr_space) = match self.tcx.global_alloc(ptr.alloc_id) {
                     GlobalAlloc::Memory(alloc) => {
                         let init = const_alloc_to_llvm(self, alloc);
                         let value = match alloc.mutability {
@@ -254,18 +254,21 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                         if !self.sess().fewer_names() {
                             llvm::set_value_name(value, format!("{:?}", ptr.alloc_id).as_bytes());
                         }
-                        value
+                        (value, AddressSpace::DATA)
                     }
-                    GlobalAlloc::Function(fn_instance) => self.get_fn_addr(fn_instance),
+                    GlobalAlloc::Function(fn_instance) => (
+                        self.get_fn_addr(fn_instance),
+                        self.data_layout().instruction_address_space,
+                    ),
                     GlobalAlloc::Static(def_id) => {
                         assert!(self.tcx.is_static(def_id));
                         assert!(!self.tcx.is_thread_local_static(def_id));
-                        self.get_static(def_id)
+                        (self.get_static(def_id), AddressSpace::DATA)
                     }
                 };
                 let llval = unsafe {
                     llvm::LLVMConstInBoundsGEP(
-                        self.const_bitcast(base_addr, self.type_i8p()),
+                        self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)),
                         &self.const_usize(ptr.offset.bytes()),
                         1,
                     )

src/librustc_codegen_llvm/consts.rs

@@ -13,14 +13,14 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::Node;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ConstValue, ErrorHandled, Pointer,
+    read_target_uint, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer,
 };
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
-use rustc_target::abi::{Align, HasDataLayout, LayoutOf, Primitive, Scalar, Size};
+use rustc_target::abi::{AddressSpace, Align, HasDataLayout, LayoutOf, Primitive, Scalar, Size};
 
 use std::ffi::CStr;
 
@@ -53,10 +53,16 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
         )
         .expect("const_alloc_to_llvm: could not read relocation pointer")
             as u64;
+
+        let address_space = match cx.tcx.global_alloc(alloc_id) {
+            GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space,
+            GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) => AddressSpace::DATA,
+        };
+
         llvals.push(cx.scalar_to_backend(
             Pointer::new(alloc_id, Size::from_bytes(ptr_offset)).into(),
             &Scalar { value: Primitive::Pointer, valid_range: 0..=!0 },
-            cx.type_i8p(),
+            cx.type_i8p_ext(address_space),
         ));
         next_offset = offset + pointer_size;
     }

src/librustc_codegen_llvm/context.rs

@@ -190,7 +190,7 @@ pub unsafe fn create_module(
 
     // Control Flow Guard is currently only supported by the MSVC linker on Windows.
     if sess.target.target.options.is_like_msvc {
-        match sess.opts.debugging_opts.control_flow_guard {
+        match sess.opts.cg.control_flow_guard {
             CFGuard::Disabled => {}
             CFGuard::NoChecks => {
                 // Set `cfguard=1` module flag to emit metadata only.

src/librustc_codegen_llvm/type_.rs

@@ -15,7 +15,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::Ty;
 use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
-use rustc_target::abi::{Align, Integer, Size};
+use rustc_target::abi::{AddressSpace, Align, Integer, Size};
 
 use std::fmt;
 use std::ptr;
@@ -198,9 +198,13 @@ impl BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         assert_ne!(
             self.type_kind(ty),
             TypeKind::Function,
-            "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead"
+            "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense"
         );
-        ty.ptr_to()
+        ty.ptr_to(AddressSpace::DATA)
+    }
+
+    fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
+        ty.ptr_to(address_space)
     }
 
     fn element_type(&self, ty: &'ll Type) -> &'ll Type {
@@ -241,11 +245,11 @@ impl Type {
     }
 
     pub fn i8p_llcx(llcx: &llvm::Context) -> &Type {
-        Type::i8_llcx(llcx).ptr_to()
+        Type::i8_llcx(llcx).ptr_to(AddressSpace::DATA)
     }
 
-    fn ptr_to(&self) -> &Type {
-        unsafe { llvm::LLVMPointerType(&self, 0) }
+    fn ptr_to(&self, address_space: AddressSpace) -> &Type {
+        unsafe { llvm::LLVMPointerType(&self, address_space.0) }
     }
 }
 

src/librustc_codegen_llvm/type_of.rs

@@ -7,7 +7,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::layout::{FnAbiExt, TyAndLayout};
 use rustc_middle::ty::print::obsolete::DefPathBasedNames;
 use rustc_middle::ty::{self, Ty, TypeFoldable};
-use rustc_target::abi::{Abi, Align, FieldsShape};
+use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
 use rustc_target::abi::{Int, Pointer, F32, F64};
 use rustc_target::abi::{LayoutOf, PointeeInfo, Scalar, Size, TyAndLayoutMethods, Variants};
 
@@ -310,12 +310,13 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             F64 => cx.type_f64(),
             Pointer => {
                 // If we know the alignment, pick something better than i8.
-                let pointee = if let Some(pointee) = self.pointee_info_at(cx, offset) {
-                    cx.type_pointee_for_align(pointee.align)
-                } else {
-                    cx.type_i8()
-                };
-                cx.type_ptr_to(pointee)
+                let (pointee, address_space) =
+                    if let Some(pointee) = self.pointee_info_at(cx, offset) {
+                        (cx.type_pointee_for_align(pointee.align), pointee.address_space)
+                    } else {
+                        (cx.type_i8(), AddressSpace::DATA)
+                    };
+                cx.type_ptr_to_ext(pointee, address_space)
             }
         }
     }

src/librustc_codegen_llvm/va_arg.rs

@@ -3,8 +3,9 @@ use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 use rustc_codegen_ssa::mir::operand::OperandRef;
-use rustc_codegen_ssa::traits::{
-    BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods,
+use rustc_codegen_ssa::{
+    common::IntPredicate,
+    traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods},
 };
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::Ty;
@@ -89,6 +90,81 @@ fn emit_ptr_va_arg(
     }
 }
 
+fn emit_aapcs_va_arg(
+    bx: &mut Builder<'a, 'll, 'tcx>,
+    list: OperandRef<'tcx, &'ll Value>,
+    target_ty: Ty<'tcx>,
+) -> &'ll Value {
+    // Implementation of the AAPCS64 calling convention for va_args see
+    // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst
+    let va_list_addr = list.immediate();
+    let layout = bx.cx.layout_of(target_ty);
+
+    let mut maybe_reg = bx.build_sibling_block("va_arg.maybe_reg");
+    let mut in_reg = bx.build_sibling_block("va_arg.in_reg");
+    let mut on_stack = bx.build_sibling_block("va_arg.on_stack");
+    let mut end = bx.build_sibling_block("va_arg.end");
+    let zero = bx.const_i32(0);
+    let offset_align = Align::from_bytes(4).unwrap();
+    assert!(&*bx.tcx().sess.target.target.target_endian == "little");
+
+    let gr_type = target_ty.is_any_ptr() || target_ty.is_integral();
+    let (reg_off, reg_top_index, slot_size) = if gr_type {
+        let gr_offs = bx.struct_gep(va_list_addr, 7);
+        let nreg = (layout.size.bytes() + 7) / 8;
+        (gr_offs, 3, nreg * 8)
+    } else {
+        let vr_off = bx.struct_gep(va_list_addr, 9);
+        let nreg = (layout.size.bytes() + 15) / 16;
+        (vr_off, 5, nreg * 16)
+    };
+
+    // if the offset >= 0 then the value will be on the stack
+    let mut reg_off_v = bx.load(reg_off, offset_align);
+    let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero);
+    bx.cond_br(use_stack, &on_stack.llbb(), &maybe_reg.llbb());
+
+    // The value at this point might be in a register, but there is a chance that
+    // it could be on the stack so we have to update the offset and then check
+    // the offset again.
+
+    if gr_type && layout.align.abi.bytes() > 8 {
+        reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(15));
+        reg_off_v = maybe_reg.and(reg_off_v, bx.const_i32(-16));
+    }
+    let new_reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(slot_size as i32));
+
+    maybe_reg.store(new_reg_off_v, reg_off, offset_align);
+
+    // Check to see if we have overflowed the registers as a result of this.
+    // If we have then we need to use the stack for this value
+    let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero);
+    maybe_reg.cond_br(use_stack, &on_stack.llbb(), &in_reg.llbb());
+
+    let top = in_reg.struct_gep(va_list_addr, reg_top_index);
+    let top = in_reg.load(top, bx.tcx().data_layout.pointer_align.abi);
+
+    // reg_value = *(@top + reg_off_v);
+    let top = in_reg.gep(top, &[reg_off_v]);
+    let top = in_reg.bitcast(top, bx.cx.type_ptr_to(layout.llvm_type(bx)));
+    let reg_value = in_reg.load(top, layout.align.abi);
+    in_reg.br(&end.llbb());
+
+    // On Stack block
+    let stack_value =
+        emit_ptr_va_arg(&mut on_stack, list, target_ty, false, Align::from_bytes(8).unwrap(), true);
+    on_stack.br(&end.llbb());
+
+    let val = end.phi(
+        layout.immediate_llvm_type(bx),
+        &[reg_value, stack_value],
+        &[&in_reg.llbb(), &on_stack.llbb()],
+    );
+
+    *bx = end;
+    val
+}
+
 pub(super) fn emit_va_arg(
     bx: &mut Builder<'a, 'll, 'tcx>,
     addr: OperandRef<'tcx, &'ll Value>,
@@ -115,6 +191,7 @@ pub(super) fn emit_va_arg(
         ("aarch64", _) if target.target_os == "ios" => {
             emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true)
         }
+        ("aarch64", _) => emit_aapcs_va_arg(bx, addr, target_ty),
         // Windows x86_64
         ("x86_64", true) => {
             let target_ty_size = bx.cx.size_of(target_ty).bytes();

src/librustc_codegen_ssa/back/link.rs

@@ -1700,7 +1700,7 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
     }
 
     // OBJECT-FILES-NO, AUDIT-ORDER
-    if sess.opts.debugging_opts.control_flow_guard != CFGuard::Disabled {
+    if sess.opts.cg.control_flow_guard != CFGuard::Disabled {
         cmd.control_flow_guard();
     }
 

src/librustc_codegen_ssa/meth.rs

@@ -75,7 +75,7 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
     }
 
     // Not in the cache; build it.
-    let nullptr = cx.const_null(cx.type_i8p());
+    let nullptr = cx.const_null(cx.type_i8p_ext(cx.data_layout().instruction_address_space));
 
     let methods_root;
     let methods = if let Some(trait_ref) = trait_ref {

src/librustc_codegen_ssa/mir/mod.rs

@@ -6,6 +6,7 @@ use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable};
 use rustc_target::abi::call::{FnAbi, PassMode};
+use rustc_target::abi::HasDataLayout;
 
 use std::iter;
 
@@ -323,7 +324,9 @@ fn create_funclets<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     // C++ personality function, but `catch (...)` has no type so
                     // it's null. The 64 here is actually a bitfield which
                     // represents that this is a catch-all block.
-                    let null = bx.const_null(bx.type_i8p());
+                    let null = bx.const_null(
+                        bx.type_i8p_ext(bx.cx().data_layout().instruction_address_space),
+                    );
                     let sixty_four = bx.const_i32(64);
                     funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]);
                     cp_bx.br(llbb);