Unrolled build for #146949

Rollup merge of #146949 - pmur:murp/improve-ppc-inline-asm, r=Amanieu

Add vsx register support for ppc inline asm, and implement preserves_flag option

This should address the last(?) missing pieces of inline asm for ppc:

* Explicit VSX register support. ISA 2.06 (POWER7) added a 64x128b register overlay extending the fpr's to 128b, and unifies them with the vmx (altivec) registers. Implementations details within gcc/llvm percolate up, and require using the `x` template modifier. I have updated the inline asm to implicitly include this for vsx arguments which do not specify it. ~~Support for the gcc codegen backend is still a todo.~~

* Implement the `preserves_flags` option. All ABI's, and all ISAs store their flags in `cr`, and the carry bit lives inside `xer`. The other status registers hold sticky bits or control bits which do not affect branch instructions.

There is some interest in the e500 (powerpcspe) port. Architecturally, it has a very different FP ISA, and includes a simd extension called SPR (which is not IBM's cell SPE). Notably, it does not have altivec/fpr/vsx registers. It also has an SPE accumulator register which its ABI marks as volatile, but I am not sure if the compiler uses it.

Author: rust-timer

compiler/rustc_codegen_gcc/src/asm.rs

@@ -546,9 +546,16 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         }
 
         if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
-            // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient
-            // on all architectures. For instance, what about FP stack?
-            extended_asm.add_clobber("cc");
+            match asm_arch {
+                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {
+                    // "cc" is cr0 on powerpc.
+                }
+                // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient
+                // on all architectures. For instance, what about FP stack?
+                _ => {
+                    extended_asm.add_clobber("cc");
+                }
+            }
         }
         if !options.contains(InlineAsmOptions::NOMEM) {
             extended_asm.add_clobber("memory");
@@ -698,6 +705,7 @@ fn reg_class_to_gcc(reg_class: InlineAsmRegClass) -> &'static str {
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
+        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => "wa",
         InlineAsmRegClass::PowerPC(
             PowerPCInlineAsmRegClass::cr
             | PowerPCInlineAsmRegClass::ctr
@@ -778,9 +786,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
-        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => {
-            cx.type_vector(cx.type_i32(), 4)
-        }
+        InlineAsmRegClass::PowerPC(
+            PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg,
+        ) => cx.type_vector(cx.type_i32(), 4),
         InlineAsmRegClass::PowerPC(
             PowerPCInlineAsmRegClass::cr
             | PowerPCInlineAsmRegClass::ctr
@@ -957,6 +965,13 @@ fn modifier_to_gcc(
         InlineAsmRegClass::LoongArch(_) => None,
         InlineAsmRegClass::Mips(_) => None,
         InlineAsmRegClass::Nvptx(_) => None,
+        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
+            if modifier.is_none() {
+                Some('x')
+            } else {
+                modifier
+            }
+        }
         InlineAsmRegClass::PowerPC(_) => None,
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
         | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,

compiler/rustc_codegen_llvm/src/asm.rs

@@ -658,6 +658,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
             PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
             PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
             PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
+            PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa",
             PowerPC(
                 PowerPCInlineAsmRegClass::cr
                 | PowerPCInlineAsmRegClass::ctr
@@ -748,6 +749,12 @@ fn modifier_to_llvm(
         LoongArch(_) => None,
         Mips(_) => None,
         Nvptx(_) => None,
+        PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
+            // The documentation for the 'x' modifier is missing for llvm, and the gcc
+            // documentation is simply "use this for any vsx argument". It is needed
+            // to ensure the correct vsx register number is used.
+            if modifier.is_none() { Some('x') } else { modifier }
+        }
         PowerPC(_) => None,
         RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
         RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
@@ -831,6 +838,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
         PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
         PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
         PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
+        PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4),
         PowerPC(
             PowerPCInlineAsmRegClass::cr
             | PowerPCInlineAsmRegClass::ctr
@@ -1061,19 +1069,21 @@ fn llvm_fixup_input<'ll, 'tcx>(
             let value = bx.or(value, bx.const_u32(0xFFFF_0000));
             bx.bitcast(value, bx.type_f32())
         }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F32) =>
-        {
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F32) => {
             let value = bx.insert_element(
                 bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
                 value,
                 bx.const_usize(0),
             );
             bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
         }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F64) =>
-        {
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F64) => {
             let value = bx.insert_element(
                 bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
                 value,
@@ -1224,15 +1234,17 @@ fn llvm_fixup_output<'ll, 'tcx>(
             let value = bx.trunc(value, bx.type_i16());
             bx.bitcast(value, bx.type_f16())
         }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F32) =>
-        {
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F32) => {
             let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
             bx.extract_element(value, bx.const_usize(0))
         }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F64) =>
-        {
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F64) => {
             let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
             bx.extract_element(value, bx.const_usize(0))
         }
@@ -1366,16 +1378,14 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
         {
             cx.type_f32()
         }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F32) =>
-        {
-            cx.type_vector(cx.type_f32(), 4)
-        }
-        (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s))
-            if s.primitive() == Primitive::Float(Float::F64) =>
-        {
-            cx.type_vector(cx.type_f64(), 2)
-        }
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F32) => cx.type_vector(cx.type_f32(), 4),
+        (
+            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
+            BackendRepr::Scalar(s),
+        ) if s.primitive() == Primitive::Float(Float::F64) => cx.type_vector(cx.type_f64(), 2),
         _ => layout.llvm_type(cx),
     }
 }

compiler/rustc_span/src/symbol.rs

@@ -2410,6 +2410,7 @@ symbols! {
         volatile_store,
         vreg,
         vreg_low16,
+        vsreg,
         vsx,
         vtable_align,
         vtable_size,

compiler/rustc_target/src/asm/mod.rs

@@ -1251,9 +1251,17 @@ impl InlineAsmClobberAbi {
                     r3, r4, r5, r6, r7,
                     r8, r9, r10, r11, r12,
 
-                    // f0-f13
+                    // f0-f13 and their vsx overlays.
                     f0, f1, f2, f3, f4, f5, f6, f7,
                     f8, f9, f10, f11, f12, f13,
+                    vs0, vs1, vs2, vs3, vs4, vs5, vs6, vs7,
+                    vs8, vs9, vs10, vs11, vs12, vs13,
+
+                    // vs14-31, the fpr portion is saved, but the rest of the register is volatile.
+                    // We can't express that here, so mark the entire vsx register as volatile.
+                    vs14, vs15, vs16, vs17, vs18, vs19, vs20,
+                    vs21, vs22, vs23, vs24, vs25, vs26, vs27,
+                    vs28, vs29, vs30, vs31,
 
                     // v0-v19
                     v0, v1, v2, v3, v4, v5, v6, v7,

compiler/rustc_target/src/asm/powerpc.rs

@@ -12,6 +12,7 @@ def_reg_class! {
         reg_nonzero,
         freg,
         vreg,
+        vsreg,
         cr,
         ctr,
         lr,
@@ -58,6 +59,10 @@ impl PowerPCInlineAsmRegClass {
                 altivec: VecI8(16), VecI16(8), VecI32(4), VecF32(4);
                 vsx: F32, F64, VecI64(2), VecF64(2);
             },
+            // VSX is a superset of altivec.
+            Self::vsreg => types! {
+                vsx: F32, F64, VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2);
+            },
             Self::cr | Self::ctr | Self::lr | Self::xer => &[],
         }
     }
@@ -86,7 +91,7 @@ fn reserved_v20to31(
 ) -> Result<(), &'static str> {
     if target.is_like_aix {
         match &*target.options.abi {
-            "vec-default" => Err("v20-v31 are reserved on vec-default ABI"),
+            "vec-default" => Err("v20-v31 (vs52-vs63) are reserved on vec-default ABI"),
             "vec-extabi" => Ok(()),
             _ => unreachable!("unrecognized AIX ABI"),
         }
@@ -188,6 +193,71 @@ def_regs! {
         v29: vreg = ["v29"] % reserved_v20to31,
         v30: vreg = ["v30"] % reserved_v20to31,
         v31: vreg = ["v31"] % reserved_v20to31,
+        vs0: vsreg = ["vs0"],
+        vs1: vsreg = ["vs1"],
+        vs2: vsreg = ["vs2"],
+        vs3: vsreg = ["vs3"],
+        vs4: vsreg = ["vs4"],
+        vs5: vsreg = ["vs5"],
+        vs6: vsreg = ["vs6"],
+        vs7: vsreg = ["vs7"],
+        vs8: vsreg = ["vs8"],
+        vs9: vsreg = ["vs9"],
+        vs10: vsreg = ["vs10"],
+        vs11: vsreg = ["vs11"],
+        vs12: vsreg = ["vs12"],
+        vs13: vsreg = ["vs13"],
+        vs14: vsreg = ["vs14"],
+        vs15: vsreg = ["vs15"],
+        vs16: vsreg = ["vs16"],
+        vs17: vsreg = ["vs17"],
+        vs18: vsreg = ["vs18"],
+        vs19: vsreg = ["vs19"],
+        vs20: vsreg = ["vs20"],
+        vs21: vsreg = ["vs21"],
+        vs22: vsreg = ["vs22"],
+        vs23: vsreg = ["vs23"],
+        vs24: vsreg = ["vs24"],
+        vs25: vsreg = ["vs25"],
+        vs26: vsreg = ["vs26"],
+        vs27: vsreg = ["vs27"],
+        vs28: vsreg = ["vs28"],
+        vs29: vsreg = ["vs29"],
+        vs30: vsreg = ["vs30"],
+        vs31: vsreg = ["vs31"],
+        vs32: vsreg = ["vs32"],
+        vs33: vsreg = ["vs33"],
+        vs34: vsreg = ["vs34"],
+        vs35: vsreg = ["vs35"],
+        vs36: vsreg = ["vs36"],
+        vs37: vsreg = ["vs37"],
+        vs38: vsreg = ["vs38"],
+        vs39: vsreg = ["vs39"],
+        vs40: vsreg = ["vs40"],
+        vs41: vsreg = ["vs41"],
+        vs42: vsreg = ["vs42"],
+        vs43: vsreg = ["vs43"],
+        vs44: vsreg = ["vs44"],
+        vs45: vsreg = ["vs45"],
+        vs46: vsreg = ["vs46"],
+        vs47: vsreg = ["vs47"],
+        vs48: vsreg = ["vs48"],
+        vs49: vsreg = ["vs49"],
+        vs50: vsreg = ["vs50"],
+        vs51: vsreg = ["vs51"],
+        // vs52 - vs63 are aliases of v20-v31.
+        vs52: vsreg = ["vs52"] % reserved_v20to31,
+        vs53: vsreg = ["vs53"] % reserved_v20to31,
+        vs54: vsreg = ["vs54"] % reserved_v20to31,
+        vs55: vsreg = ["vs55"] % reserved_v20to31,
+        vs56: vsreg = ["vs56"] % reserved_v20to31,
+        vs57: vsreg = ["vs57"] % reserved_v20to31,
+        vs58: vsreg = ["vs58"] % reserved_v20to31,
+        vs59: vsreg = ["vs59"] % reserved_v20to31,
+        vs60: vsreg = ["vs60"] % reserved_v20to31,
+        vs61: vsreg = ["vs61"] % reserved_v20to31,
+        vs62: vsreg = ["vs62"] % reserved_v20to31,
+        vs63: vsreg = ["vs63"] % reserved_v20to31,
         cr: cr = ["cr"],
         cr0: cr = ["cr0"],
         cr1: cr = ["cr1"],
@@ -245,6 +315,15 @@ impl PowerPCInlineAsmReg {
             (v8, "8"), (v9, "9"), (v10, "10"), (v11, "11"), (v12, "12"), (v13, "13"), (v14, "14"), (v15, "15");
             (v16, "16"), (v17, "17"), (v18, "18"), (v19, "19"), (v20, "20"), (v21, "21"), (v22, "22"), (v23, "23");
             (v24, "24"), (v25, "25"), (v26, "26"), (v27, "27"), (v28, "28"), (v29, "29"), (v30, "30"), (v31, "31");
+            (vs0, "0"), (vs1, "1"), (vs2, "2"), (vs3, "3"), (vs4, "4"), (vs5, "5"), (vs6, "6"), (vs7, "7"),
+            (vs8, "8"), (vs9, "9"), (vs10, "10"), (vs11, "11"), (vs12, "12"), (vs13, "13"), (vs14, "14"),
+            (vs15, "15"), (vs16, "16"), (vs17, "17"), (vs18, "18"), (vs19, "19"), (vs20, "20"), (vs21, "21"),
+            (vs22, "22"), (vs23, "23"), (vs24, "24"), (vs25, "25"), (vs26, "26"), (vs27, "27"), (vs28, "28"),
+            (vs29, "29"), (vs30, "30"), (vs31, "31"), (vs32, "32"), (vs33, "33"), (vs34, "34"), (vs35, "35"),
+            (vs36, "36"), (vs37, "37"), (vs38, "38"), (vs39, "39"), (vs40, "40"), (vs41, "41"), (vs42, "42"),
+            (vs43, "43"), (vs44, "44"), (vs45, "45"), (vs46, "46"), (vs47, "47"), (vs48, "48"), (vs49, "49"),
+            (vs50, "50"), (vs51, "51"), (vs52, "52"), (vs53, "53"), (vs54, "54"), (vs55, "55"), (vs56, "56"),
+            (vs57, "57"), (vs58, "58"), (vs59, "59"), (vs60, "60"), (vs61, "61"), (vs62, "62"), (vs63, "63"),
             (cr, "cr");
             (cr0, "0"), (cr1, "1"), (cr2, "2"), (cr3, "3"), (cr4, "4"), (cr5, "5"), (cr6, "6"), (cr7, "7");
             (ctr, "ctr");
@@ -276,8 +355,77 @@ impl PowerPCInlineAsmReg {
             };
         }
         reg_conflicts! {
-            cr : cr0 cr1 cr2 cr3 cr4 cr5 cr6 cr7;
+            cr : cr0 cr1 cr2 cr3 cr4 cr5 cr6 cr7,
+            // f0-f31 overlap half of each of vs0-vs32.
+            vs0 : f0,
+            vs1 : f1,
+            vs2 : f2,
+            vs3 : f3,
+            vs4 : f4,
+            vs5 : f5,
+            vs6 : f6,
+            vs7 : f7,
+            vs8 : f8,
+            vs9 : f9,
+            vs10 : f10,
+            vs11 : f11,
+            vs12 : f12,
+            vs13 : f13,
+            vs14 : f14,
+            vs15 : f15,
+            vs16 : f16,
+            vs17 : f17,
+            vs18 : f18,
+            vs19 : f19,
+            vs20 : f20,
+            vs21 : f21,
+            vs22 : f22,
+            vs23 : f23,
+            vs24 : f24,
+            vs25 : f25,
+            vs26 : f26,
+            vs27 : f27,
+            vs28 : f28,
+            vs29 : f29,
+            vs30 : f30,
+            vs31 : f31,
+            // vs32-v63 are aliases of v0-v31
+            vs32 : v0,
+            vs33 : v1,
+            vs34 : v2,
+            vs35 : v3,
+            vs36 : v4,
+            vs37 : v5,
+            vs38 : v6,
+            vs39 : v7,
+            vs40 : v8,
+            vs41 : v9,
+            vs42 : v10,
+            vs43 : v11,
+            vs44 : v12,
+            vs45 : v13,
+            vs46 : v14,
+            vs47 : v15,
+            vs48 : v16,
+            vs49 : v17,
+            vs50 : v18,
+            vs51 : v19,
+            vs52 : v20,
+            vs53 : v21,
+            vs54 : v22,
+            vs55 : v23,
+            vs56 : v24,
+            vs57 : v25,
+            vs58 : v26,
+            vs59 : v27,
+            vs60 : v28,
+            vs61 : v29,
+            vs62 : v30,
+            vs63 : v31;
         }
-        // f0-f31 (vsr0-vsr31) and v0-v31 (vsr32-vsr63) do not conflict.
+        // For more detail on how vsx, vmx (altivec), fpr, and mma registers overlap
+        // see OpenPOWER ISA 3.1C, Book I, Section 7.2.1.1 through 7.2.1.3.
+        //
+        // https://files.openpower.foundation/s/9izgC5Rogi5Ywmm
     }
 }