Skip to content

Commit 9ba8053

Browse files
SpriteOvOSpriteOvO
committed
Introduce adjust_for_rust_abi in rustc_target 
1 parent a2a1206 commit 9ba8053

File tree

3 files changed

+161
-138
lines changed

3 files changed

+161
-138
lines changed

‎compiler/rustc_target/src/callconv/mod.rs‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
use std::fmt;
21
use std::str::FromStr;
2+
use std::{fmt, iter};
33

4+
use rustc_abi::AddressSpace;
5+
use rustc_abi::Primitive::Pointer;
46
pub use rustc_abi::{Reg, RegKind};
57
use rustc_macros::HashStable_Generic;
68
use rustc_span::Symbol;
79

810
use crate::abi::{self, Abi, Align, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
11+
use crate::spec::abi::Abi as SpecAbi;
912
use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi};
1013

1114
mod aarch64;
@@ -716,6 +719,116 @@ impl<'a, Ty> FnAbi<'a, Ty> {
716719

717720
Ok(())
718721
}
722+
723+
pub fn adjust_for_rust_abi<C>(&mut self, cx: &C, abi: SpecAbi)
724+
where
725+
Ty: TyAbiInterface<'a, C> + Copy,
726+
C: HasDataLayout + HasTargetSpec,
727+
{
728+
let spec = cx.target_spec();
729+
match &spec.arch[..] {
730+
"x86" => x86::compute_rust_abi_info(cx, self, abi),
731+
_ => {}
732+
};
733+
734+
for (arg_idx, arg) in self
735+
.args
736+
.iter_mut()
737+
.enumerate()
738+
.map(|(idx, arg)| (Some(idx), arg))
739+
.chain(iter::once((None, &mut self.ret)))
740+
{
741+
if arg.is_ignore() {
742+
continue;
743+
}
744+
745+
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
746+
// Return values larger than 2 registers using a return area
747+
// pointer. LLVM and Cranelift disagree about how to return
748+
// values that don't fit in the registers designated for return
749+
// values. LLVM will force the entire return value to be passed
750+
// by return area pointer, while Cranelift will look at each IR level
751+
// return value independently and decide to pass it in a
752+
// register or not, which would result in the return value
753+
// being passed partially in registers and partially through a
754+
// return area pointer.
755+
//
756+
// While Cranelift may need to be fixed as the LLVM behavior is
757+
// generally more correct with respect to the surface language,
758+
// forcing this behavior in rustc itself makes it easier for
759+
// other backends to conform to the Rust ABI and for the C ABI
760+
// rustc already handles this behavior anyway.
761+
//
762+
// In addition LLVM's decision to pass the return value in
763+
// registers or using a return area pointer depends on how
764+
// exactly the return type is lowered to an LLVM IR type. For
765+
// example `Option<u128>` can be lowered as `{ i128, i128 }`
766+
// in which case the x86_64 backend would use a return area
767+
// pointer, or it could be passed as `{ i32, i128 }` in which
768+
// case the x86_64 backend would pass it in registers by taking
769+
// advantage of an LLVM ABI extension that allows using 3
770+
// registers for the x86_64 sysv call conv rather than the
771+
// officially specified 2 registers.
772+
//
773+
// FIXME: Technically we should look at the amount of available
774+
// return registers rather than guessing that there are 2
775+
// registers for return values. In practice only a couple of
776+
// architectures have less than 2 return registers. None of
777+
// which supported by Cranelift.
778+
//
779+
// NOTE: This adjustment is only necessary for the Rust ABI as
780+
// for other ABI's the calling convention implementations in
781+
// rustc_target already ensure any return value which doesn't
782+
// fit in the available amount of return registers is passed in
783+
// the right way for the current target.
784+
arg.make_indirect();
785+
continue;
786+
}
787+
788+
match arg.layout.abi {
789+
Abi::Aggregate { .. } => {}
790+
791+
// This is a fun case! The gist of what this is doing is
792+
// that we want callers and callees to always agree on the
793+
// ABI of how they pass SIMD arguments. If we were to *not*
794+
// make these arguments indirect then they'd be immediates
795+
// in LLVM, which means that they'd used whatever the
796+
// appropriate ABI is for the callee and the caller. That
797+
// means, for example, if the caller doesn't have AVX
798+
// enabled but the callee does, then passing an AVX argument
799+
// across this boundary would cause corrupt data to show up.
800+
//
801+
// This problem is fixed by unconditionally passing SIMD
802+
// arguments through memory between callers and callees
803+
// which should get them all to agree on ABI regardless of
804+
// target feature sets. Some more information about this
805+
// issue can be found in #44367.
806+
//
807+
// Note that the intrinsic ABI is exempt here as
808+
// that's how we connect up to LLVM and it's unstable
809+
// anyway, we control all calls to it in libstd.
810+
Abi::Vector { .. } if abi != SpecAbi::RustIntrinsic && spec.simd_types_indirect => {
811+
arg.make_indirect();
812+
continue;
813+
}
814+
815+
_ => continue,
816+
}
817+
// Compute `Aggregate` ABI.
818+
819+
let is_indirect_not_on_stack =
820+
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
821+
assert!(is_indirect_not_on_stack);
822+
823+
let size = arg.layout.size;
824+
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
825+
// We want to pass small aggregates as immediates, but using
826+
// an LLVM aggregate type for this leads to bad optimizations,
827+
// so we pick an appropriately sized integer type instead.
828+
arg.cast_to(Reg { kind: RegKind::Integer, size });
829+
}
830+
}
831+
}
719832
}
720833

721834
impl FromStr for Conv {

‎compiler/rustc_target/src/callconv/x86.rs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
use rustc_abi::Float::*;
2+
use rustc_abi::Primitive::Float;
3+
14
use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind};
25
use crate::abi::{Abi, Align, HasDataLayout, TyAbiInterface, TyAndLayout};
36
use crate::spec::HasTargetSpec;
7+
use crate::spec::abi::Abi as SpecAbi;
48

59
#[derive(PartialEq)]
610
pub(crate) enum Flavor {
@@ -183,3 +187,38 @@ where
183187
}
184188
}
185189
}
190+
191+
pub(crate) fn compute_rust_abi_info<'a, Ty, C>(_cx: &C, fn_abi: &mut FnAbi<'a, Ty>, abi: SpecAbi)
192+
where
193+
Ty: TyAbiInterface<'a, C> + Copy,
194+
C: HasDataLayout + HasTargetSpec,
195+
{
196+
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
197+
// registers will quiet signalling NaNs.
198+
if !fn_abi.ret.is_ignore()
199+
// Intrinsics themselves are not actual "real" functions, so theres no need to change their ABIs.
200+
&& abi != SpecAbi::RustIntrinsic
201+
{
202+
match fn_abi.ret.layout.abi {
203+
// Handle similar to the way arguments with an `Abi::Aggregate` abi are handled
204+
// in `adjust_for_rust_abi`, by returning arguments up to the size of a pointer (32 bits on x86)
205+
// cast to an appropriately sized integer.
206+
Abi::Scalar(s) if s.primitive() == Float(F32) => {
207+
// Same size as a pointer, return in a register.
208+
fn_abi.ret.cast_to(Reg::i32());
209+
}
210+
Abi::Scalar(s) if s.primitive() == Float(F64) => {
211+
// Larger than a pointer, return indirectly.
212+
fn_abi.ret.make_indirect();
213+
}
214+
Abi::ScalarPair(s1, s2)
215+
if matches!(s1.primitive(), Float(F32 | F64))
216+
|| matches!(s2.primitive(), Float(F32 | F64)) =>
217+
{
218+
// Larger than a pointer, return indirectly.
219+
fn_abi.ret.make_indirect();
220+
}
221+
_ => {}
222+
};
223+
}
224+
}

‎compiler/rustc_ty_utils/src/abi.rs‎

Lines changed: 8 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
use std::iter;
22

3-
use rustc_abi::Float::*;
4-
use rustc_abi::Primitive::{Float, Pointer};
5-
use rustc_abi::{Abi, AddressSpace, PointerKind, Scalar, Size};
3+
use rustc_abi::Primitive::Pointer;
4+
use rustc_abi::{Abi, PointerKind, Scalar, Size};
65
use rustc_hir as hir;
76
use rustc_hir::lang_items::LangItem;
87
use rustc_middle::bug;
@@ -14,8 +13,7 @@ use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt};
1413
use rustc_session::config::OptLevel;
1514
use rustc_span::def_id::DefId;
1615
use rustc_target::abi::call::{
17-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind,
18-
RiscvInterruptKind,
16+
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, RiscvInterruptKind,
1917
};
2018
use rustc_target::spec::abi::Abi as SpecAbi;
2119
use tracing::debug;
@@ -679,6 +677,8 @@ fn fn_abi_adjust_for_abi<'tcx>(
679677
let tcx = cx.tcx();
680678

681679
if abi == SpecAbi::Rust || abi == SpecAbi::RustCall || abi == SpecAbi::RustIntrinsic {
680+
fn_abi.adjust_for_rust_abi(cx, abi);
681+
682682
// Look up the deduced parameter attributes for this function, if we have its def ID and
683683
// we're optimizing in non-incremental mode. We'll tag its parameters with those attributes
684684
// as appropriate.
@@ -689,141 +689,17 @@ fn fn_abi_adjust_for_abi<'tcx>(
689689
&[]
690690
};
691691

692-
let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, arg_idx: Option<usize>| {
692+
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
693693
if arg.is_ignore() {
694-
return;
695-
}
696-
697-
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
698-
// registers will quiet signalling NaNs.
699-
if tcx.sess.target.arch == "x86"
700-
&& arg_idx.is_none()
701-
// Intrinsics themselves are not actual "real" functions, so theres no need to
702-
// change their ABIs.
703-
&& abi != SpecAbi::RustIntrinsic
704-
{
705-
match arg.layout.abi {
706-
// Handle similar to the way arguments with an `Abi::Aggregate` abi are handled
707-
// below, by returning arguments up to the size of a pointer (32 bits on x86)
708-
// cast to an appropriately sized integer.
709-
Abi::Scalar(s) if s.primitive() == Float(F32) => {
710-
// Same size as a pointer, return in a register.
711-
arg.cast_to(Reg::i32());
712-
return;
713-
}
714-
Abi::Scalar(s) if s.primitive() == Float(F64) => {
715-
// Larger than a pointer, return indirectly.
716-
arg.make_indirect();
717-
return;
718-
}
719-
Abi::ScalarPair(s1, s2)
720-
if matches!(s1.primitive(), Float(F32 | F64))
721-
|| matches!(s2.primitive(), Float(F32 | F64)) =>
722-
{
723-
// Larger than a pointer, return indirectly.
724-
arg.make_indirect();
725-
return;
726-
}
727-
_ => {}
728-
};
729-
}
730-
731-
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
732-
// Return values larger than 2 registers using a return area
733-
// pointer. LLVM and Cranelift disagree about how to return
734-
// values that don't fit in the registers designated for return
735-
// values. LLVM will force the entire return value to be passed
736-
// by return area pointer, while Cranelift will look at each IR level
737-
// return value independently and decide to pass it in a
738-
// register or not, which would result in the return value
739-
// being passed partially in registers and partially through a
740-
// return area pointer.
741-
//
742-
// While Cranelift may need to be fixed as the LLVM behavior is
743-
// generally more correct with respect to the surface language,
744-
// forcing this behavior in rustc itself makes it easier for
745-
// other backends to conform to the Rust ABI and for the C ABI
746-
// rustc already handles this behavior anyway.
747-
//
748-
// In addition LLVM's decision to pass the return value in
749-
// registers or using a return area pointer depends on how
750-
// exactly the return type is lowered to an LLVM IR type. For
751-
// example `Option<u128>` can be lowered as `{ i128, i128 }`
752-
// in which case the x86_64 backend would use a return area
753-
// pointer, or it could be passed as `{ i32, i128 }` in which
754-
// case the x86_64 backend would pass it in registers by taking
755-
// advantage of an LLVM ABI extension that allows using 3
756-
// registers for the x86_64 sysv call conv rather than the
757-
// officially specified 2 registers.
758-
//
759-
// FIXME: Technically we should look at the amount of available
760-
// return registers rather than guessing that there are 2
761-
// registers for return values. In practice only a couple of
762-
// architectures have less than 2 return registers. None of
763-
// which supported by Cranelift.
764-
//
765-
// NOTE: This adjustment is only necessary for the Rust ABI as
766-
// for other ABI's the calling convention implementations in
767-
// rustc_target already ensure any return value which doesn't
768-
// fit in the available amount of return registers is passed in
769-
// the right way for the current target.
770-
arg.make_indirect();
771-
return;
772-
}
773-
774-
match arg.layout.abi {
775-
Abi::Aggregate { .. } => {}
776-
777-
// This is a fun case! The gist of what this is doing is
778-
// that we want callers and callees to always agree on the
779-
// ABI of how they pass SIMD arguments. If we were to *not*
780-
// make these arguments indirect then they'd be immediates
781-
// in LLVM, which means that they'd used whatever the
782-
// appropriate ABI is for the callee and the caller. That
783-
// means, for example, if the caller doesn't have AVX
784-
// enabled but the callee does, then passing an AVX argument
785-
// across this boundary would cause corrupt data to show up.
786-
//
787-
// This problem is fixed by unconditionally passing SIMD
788-
// arguments through memory between callers and callees
789-
// which should get them all to agree on ABI regardless of
790-
// target feature sets. Some more information about this
791-
// issue can be found in #44367.
792-
//
793-
// Note that the intrinsic ABI is exempt here as
794-
// that's how we connect up to LLVM and it's unstable
795-
// anyway, we control all calls to it in libstd.
796-
Abi::Vector { .. }
797-
if abi != SpecAbi::RustIntrinsic && tcx.sess.target.simd_types_indirect =>
798-
{
799-
arg.make_indirect();
800-
return;
801-
}
802-
803-
_ => return,
804-
}
805-
// Compute `Aggregate` ABI.
806-
807-
let is_indirect_not_on_stack =
808-
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
809-
assert!(is_indirect_not_on_stack, "{:?}", arg);
810-
811-
let size = arg.layout.size;
812-
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
813-
// We want to pass small aggregates as immediates, but using
814-
// an LLVM aggregate type for this leads to bad optimizations,
815-
// so we pick an appropriately sized integer type instead.
816-
arg.cast_to(Reg { kind: RegKind::Integer, size });
694+
continue;
817695
}
818696

819697
// If we deduced that this parameter was read-only, add that to the attribute list now.
820698
//
821699
// The `readonly` parameter only applies to pointers, so we can only do this if the
822700
// argument was passed indirectly. (If the argument is passed directly, it's an SSA
823701
// value, so it's implicitly immutable.)
824-
if let (Some(arg_idx), &mut PassMode::Indirect { ref mut attrs, .. }) =
825-
(arg_idx, &mut arg.mode)
826-
{
702+
if let &mut PassMode::Indirect { ref mut attrs, .. } = &mut arg.mode {
827703
// The `deduced_param_attrs` list could be empty if this is a type of function
828704
// we can't deduce any parameters for, so make sure the argument index is in
829705
// bounds.
@@ -834,11 +710,6 @@ fn fn_abi_adjust_for_abi<'tcx>(
834710
}
835711
}
836712
}
837-
};
838-
839-
fixup(&mut fn_abi.ret, None);
840-
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
841-
fixup(arg, Some(arg_idx));
842713
}
843714
} else {
844715
fn_abi

0 commit comments

Comments
 (0)