Return f32 and f64 in XMM0 instead of FP0 on i686 Rust calling convention

i686 already uses SSE2 to do calculations with f32 and f64, but the C
calling convention uses the x87 stack to return values. The Rust calling
convention does not need to do this, and LLVM makes it easy to use XMM0
instead, which saves move instructions and fixes problems with NaN values.

Author: Günther Brammer

compiler/rustc_ty_utils/src/abi.rs

@@ -371,6 +371,8 @@ fn fn_abi_new_uncached<'tcx>(
     let target = &cx.tcx.sess.target;
     let target_env_gnu_like = matches!(&target.env[..], "gnu" | "musl" | "uclibc");
     let win_x64_gnu = target.os == "windows" && target.arch == "x86_64" && target.env == "gnu";
+    let x86_sse2 =
+        target.arch == "x86" && (target.features.contains("sse2") || target.cpu == "pentium4");
     let linux_s390x_gnu_like =
         target.os == "linux" && target.arch == "s390x" && target_env_gnu_like;
     let linux_sparc64_gnu_like =
@@ -415,6 +417,10 @@ fn fn_abi_new_uncached<'tcx>(
                 is_return,
                 drop_target_pointee,
             );
+            // Use XMM0 instead of FP0 to preserve NaN payloads
+            if x86_sse2 && rust_abi && is_return && matches!(scalar.primitive(), F32 | F64) {
+                attrs.set(ArgAttribute::InReg);
+            }
             attrs
         });
 

library/std/src/f32/tests.rs

@@ -328,7 +328,7 @@ macro_rules! assert_f32_biteq {
 
 // Ignore test on x87 floating point, these platforms do not guarantee NaN
 // payloads are preserved and flush denormals to zero, failing the tests.
-#[cfg(not(target_arch = "x86"))]
+#[cfg(not(all(target_arch = "x86", not(target_feature = "sse2"))))]
 #[test]
 fn test_next_up() {
     let tiny = f32::from_bits(1);
@@ -361,7 +361,7 @@ fn test_next_up() {
 
 // Ignore test on x87 floating point, these platforms do not guarantee NaN
 // payloads are preserved and flush denormals to zero, failing the tests.
-#[cfg(not(target_arch = "x86"))]
+#[cfg(not(all(target_arch = "x86", not(target_feature = "sse2"))))]
 #[test]
 fn test_next_down() {
     let tiny = f32::from_bits(1);

library/std/src/f64/tests.rs

@@ -318,7 +318,7 @@ macro_rules! assert_f64_biteq {
 
 // Ignore test on x87 floating point, these platforms do not guarantee NaN
 // payloads are preserved and flush denormals to zero, failing the tests.
-#[cfg(not(target_arch = "x86"))]
+#[cfg(not(all(target_arch = "x86", not(target_feature = "sse2"))))]
 #[test]
 fn test_next_up() {
     let tiny = f64::from_bits(1);
@@ -350,7 +350,7 @@ fn test_next_up() {
 
 // Ignore test on x87 floating point, these platforms do not guarantee NaN
 // payloads are preserved and flush denormals to zero, failing the tests.
-#[cfg(not(target_arch = "x86"))]
+#[cfg(not(all(target_arch = "x86", not(target_feature = "sse2"))))]
 #[test]
 fn test_next_down() {
     let tiny = f64::from_bits(1);