@@ -37,6 +37,7 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
3737 align : Align ,
3838 slot_size : Align ,
3939 allow_higher_align : bool ,
40+ force_right_adjust : bool ,
4041) -> ( & ' ll Value , Align ) {
4142 let va_list_ty = bx. type_ptr ( ) ;
4243 let va_list_addr = list. immediate ( ) ;
@@ -54,7 +55,10 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
5455 let next = bx. inbounds_ptradd ( addr, full_direct_size) ;
5556 bx. store ( next, va_list_addr, bx. tcx ( ) . data_layout . pointer_align . abi ) ;
5657
57- if size. bytes ( ) < slot_size. bytes ( ) && bx. tcx ( ) . sess . target . endian == Endian :: Big {
58+ if size. bytes ( ) < slot_size. bytes ( )
59+ && bx. tcx ( ) . sess . target . endian == Endian :: Big
60+ && force_right_adjust
61+ {
5862 let adjusted_size = bx. cx ( ) . const_i32 ( ( slot_size. bytes ( ) - size. bytes ( ) ) as i32 ) ;
5963 let adjusted = bx. inbounds_ptradd ( addr, adjusted_size) ;
6064 ( adjusted, addr_align)
@@ -63,14 +67,40 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
6367 }
6468}
6569
70+ enum PassMode {
71+ Direct ,
72+ Indirect ,
73+ }
74+
75+ enum SlotSize {
76+ Bytes8 = 8 ,
77+ Bytes4 = 4 ,
78+ }
79+
80+ enum AllowHigherAlign {
81+ No ,
82+ Yes ,
83+ }
84+
85+ enum ForceRightAdjust {
86+ No ,
87+ Yes ,
88+ }
89+
6690fn emit_ptr_va_arg < ' ll , ' tcx > (
6791 bx : & mut Builder < ' _ , ' ll , ' tcx > ,
6892 list : OperandRef < ' tcx , & ' ll Value > ,
6993 target_ty : Ty < ' tcx > ,
70- indirect : bool ,
71- slot_size : Align ,
72- allow_higher_align : bool ,
94+ pass_mode : PassMode ,
95+ slot_size : SlotSize ,
96+ allow_higher_align : AllowHigherAlign ,
97+ force_right_adjust : ForceRightAdjust ,
7398) -> & ' ll Value {
99+ let indirect = matches ! ( pass_mode, PassMode :: Indirect ) ;
100+ let allow_higher_align = matches ! ( allow_higher_align, AllowHigherAlign :: Yes ) ;
101+ let force_right_adjust = matches ! ( force_right_adjust, ForceRightAdjust :: Yes ) ;
102+ let slot_size = Align :: from_bytes ( slot_size as u64 ) . unwrap ( ) ;
103+
74104 let layout = bx. cx . layout_of ( target_ty) ;
75105 let ( llty, size, align) = if indirect {
76106 (
@@ -81,8 +111,15 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
81111 } else {
82112 ( layout. llvm_type ( bx. cx ) , layout. size , layout. align )
83113 } ;
84- let ( addr, addr_align) =
85- emit_direct_ptr_va_arg ( bx, list, size, align. abi , slot_size, allow_higher_align) ;
114+ let ( addr, addr_align) = emit_direct_ptr_va_arg (
115+ bx,
116+ list,
117+ size,
118+ align. abi ,
119+ slot_size,
120+ allow_higher_align,
121+ force_right_adjust,
122+ ) ;
86123 if indirect {
87124 let tmp_ret = bx. load ( llty, addr, addr_align) ;
88125 bx. load ( bx. cx . layout_of ( target_ty) . llvm_type ( bx. cx ) , tmp_ret, align. abi )
@@ -179,8 +216,15 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
179216
180217 // On Stack block
181218 bx. switch_to_block ( on_stack) ;
182- let stack_value =
183- emit_ptr_va_arg ( bx, list, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , true ) ;
219+ let stack_value = emit_ptr_va_arg (
220+ bx,
221+ list,
222+ target_ty,
223+ PassMode :: Direct ,
224+ SlotSize :: Bytes8 ,
225+ AllowHigherAlign :: Yes ,
226+ ForceRightAdjust :: No ,
227+ ) ;
184228 bx. br ( end) ;
185229
186230 bx. switch_to_block ( end) ;
@@ -386,29 +430,67 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
386430 // Determine the va_arg implementation to use. The LLVM va_arg instruction
387431 // is lacking in some instances, so we should only use it as a fallback.
388432 let target = & bx. cx . tcx . sess . target ;
389- let arch = & bx. cx . tcx . sess . target . arch ;
433+ let arch = & target. arch ;
434+
390435 match & * * arch {
391436 // Windows x86
392- "x86" if target. is_like_windows => {
393- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 4 ) . unwrap ( ) , false )
394- }
437+ "x86" if target. is_like_windows => emit_ptr_va_arg (
438+ bx,
439+ addr,
440+ target_ty,
441+ PassMode :: Direct ,
442+ SlotSize :: Bytes4 ,
443+ AllowHigherAlign :: No ,
444+ ForceRightAdjust :: No ,
445+ ) ,
395446 // Generic x86
396- "x86" => emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 4 ) . unwrap ( ) , true ) ,
447+ "x86" => emit_ptr_va_arg (
448+ bx,
449+ addr,
450+ target_ty,
451+ PassMode :: Direct ,
452+ SlotSize :: Bytes4 ,
453+ AllowHigherAlign :: Yes ,
454+ ForceRightAdjust :: No ,
455+ ) ,
397456 // Windows AArch64
398- "aarch64" | "arm64ec" if target. is_like_windows => {
399- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , false )
400- }
457+ "aarch64" | "arm64ec" if target. is_like_windows => emit_ptr_va_arg (
458+ bx,
459+ addr,
460+ target_ty,
461+ PassMode :: Direct ,
462+ SlotSize :: Bytes8 ,
463+ AllowHigherAlign :: No ,
464+ ForceRightAdjust :: No ,
465+ ) ,
401466 // macOS / iOS AArch64
402- "aarch64" if target. is_like_darwin => {
403- emit_ptr_va_arg ( bx, addr, target_ty, false , Align :: from_bytes ( 8 ) . unwrap ( ) , true )
404- }
467+ "aarch64" if target. is_like_darwin => emit_ptr_va_arg (
468+ bx,
469+ addr,
470+ target_ty,
471+ PassMode :: Direct ,
472+ SlotSize :: Bytes8 ,
473+ AllowHigherAlign :: Yes ,
474+ ForceRightAdjust :: No ,
475+ ) ,
405476 "aarch64" => emit_aapcs_va_arg ( bx, addr, target_ty) ,
406477 "s390x" => emit_s390x_va_arg ( bx, addr, target_ty) ,
407478 // Windows x86_64
408479 "x86_64" if target. is_like_windows => {
409480 let target_ty_size = bx. cx . size_of ( target_ty) . bytes ( ) ;
410- let indirect: bool = target_ty_size > 8 || !target_ty_size. is_power_of_two ( ) ;
411- emit_ptr_va_arg ( bx, addr, target_ty, indirect, Align :: from_bytes ( 8 ) . unwrap ( ) , false )
481+ emit_ptr_va_arg (
482+ bx,
483+ addr,
484+ target_ty,
485+ if target_ty_size > 8 || !target_ty_size. is_power_of_two ( ) {
486+ PassMode :: Indirect
487+ } else {
488+ PassMode :: Direct
489+ } ,
490+ SlotSize :: Bytes8 ,
491+ AllowHigherAlign :: No ,
492+ ForceRightAdjust :: No ,
493+ )
412494 }
413495 "xtensa" => emit_xtensa_va_arg ( bx, addr, target_ty) ,
414496 // For all other architecture/OS combinations fall back to using
0 commit comments