Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 6bba061

Browse files
borsbors
committed
Auto merge of rust-lang#112294 - saethlin:inline-me-maybe, r=oli-obk
Ignore the always part of #[inline(always)] in MIR inlining `#[inline(always)]` is used in two cases: for functions that are so trivial it is always profitable to inline them, but also for functions which LLVM thinks are a bad inlining candidate, but which actually turn out to be profitable to inline. That second justification doesn't apply to the MIR inliner, so ignoring our cost estimation for these functions is not necessarily the right right thing to do. This is basically a wash on non-check runs and a perf benefit in check runs. There are some notable regressions, and I think we might be able to claw those back by turning `#[inline(always)]` into a stronger hint. But I think this PR stands decently on its own as a tidy simplification.
2 parents 6a94e87 + c153f3a commit 6bba061

19 files changed

+349
-934
lines changed

‎compiler/rustc_mir_transform/src/inline.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -479,11 +479,12 @@ impl<'tcx> Inliner<'tcx> {
479479
// Abort if type validation found anything fishy.
480480
checker.validation?;
481481

482+
// N.B. We still apply our cost threshold to #[inline(always)] functions.
483+
// That attribute is often applied to very large functions that exceed LLVM's (very
484+
// generous) inlining threshold. Such functions are very poor MIR inlining candidates.
485+
// Always inlining #[inline(always)] functions in MIR, on net, slows down the compiler.
482486
let cost = checker.cost;
483-
if let InlineAttr::Always = callee_attrs.inline {
484-
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
485-
Ok(())
486-
} else if cost <= threshold {
487+
if cost <= threshold {
487488
debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
488489
Ok(())
489490
} else {

‎src/tools/miri/tests/fail/terminate-terminator.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//@compile-flags: -Zmir-opt-level=3
1+
//@compile-flags: -Zmir-opt-level=3 -Zinline-mir-hint-threshold=1000
22
// Enable MIR inlining to ensure that `TerminatorKind::Terminate` is generated
33
// instead of just `UnwindAction::Terminate`.
44

‎tests/mir-opt/inline/asm_unwind.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
//
33
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
44
// needs-asm-support
5+
// compile-flags: -Zinline-mir-hint-threshold=1000
56
#![feature(asm_unwind)]
67

78
struct D;
@@ -10,7 +11,7 @@ impl Drop for D {
1011
fn drop(&mut self) {}
1112
}
1213

13-
#[inline(always)]
14+
#[inline]
1415
fn foo() {
1516
let _d = D;
1617
unsafe { std::arch::asm!("", options(may_unwind)) };

‎tests/mir-opt/inline/cycle.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
// compile-flags: -Zinline-mir-hint-threshold=1000
23

34
// EMIT_MIR cycle.f.Inline.diff
45
#[inline(always)]

‎tests/mir-opt/inline/inline_diverging.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Tests inlining of diverging calls.
22
//
33
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
4+
// compile-flags: -Zinline-mir-hint-threshold=1000
45
#![crate_type = "lib"]
56

67
// EMIT_MIR inline_diverging.f.Inline.diff

‎tests/mir-opt/inline/inline_generator.main.Inline.panic-abort.diff‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,30 @@
44
fn main() -> () {
55
let mut _0: ();
66
let _1: std::ops::GeneratorState<i32, bool>;
7-
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>;
8-
let mut _3: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
9-
let mut _4: [generator@$DIR/inline_generator.rs:15:5: 15:8];
7+
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>;
8+
let mut _3: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
9+
let mut _4: [generator@$DIR/inline_generator.rs:16:5: 16:8];
1010
+ let mut _5: bool;
1111
scope 1 {
1212
debug _r => _1;
1313
}
1414
+ scope 2 (inlined g) {
1515
+ }
16-
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new) {
16+
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new) {
1717
+ debug pointer => _3;
1818
+ scope 4 {
19-
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new_unchecked) {
19+
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new_unchecked) {
2020
+ debug pointer => _3;
2121
+ }
2222
+ }
2323
+ }
2424
+ scope 6 (inlined g::{closure#0}) {
2525
+ debug a => _5;
26-
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
26+
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
2727
+ let mut _7: u32;
2828
+ let mut _8: i32;
29-
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
30-
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
29+
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
30+
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
3131
+ }
3232

3333
bb0: {
@@ -39,18 +39,18 @@
3939
- }
4040
-
4141
- bb1: {
42-
+ _4 = [generator@$DIR/inline_generator.rs:15:5: 15:8 (#0)];
42+
+ _4 = [generator@$DIR/inline_generator.rs:16:5: 16:8 (#0)];
4343
_3 = &mut _4;
44-
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new(move _3) -> [return: bb2, unwind unreachable];
44+
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new(move _3) -> [return: bb2, unwind unreachable];
4545
- }
4646
-
4747
- bb2: {
48-
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _3 };
48+
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]> { pointer: move _3 };
4949
StorageDead(_3);
50-
- _1 = <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
50+
- _1 = <[generator@$DIR/inline_generator.rs:16:5: 16:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
5151
+ StorageLive(_5);
5252
+ _5 = const false;
53-
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
53+
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
5454
+ _7 = discriminant((*_6));
5555
+ switchInt(move _7) -> [0: bb2, 1: bb6, 3: bb7, otherwise: bb8];
5656
}
@@ -82,7 +82,7 @@
8282
+
8383
+ bb5: {
8484
+ _1 = GeneratorState::<i32, bool>::Yielded(move _8);
85-
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
85+
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
8686
+ discriminant((*_9)) = 3;
8787
+ goto -> bb1;
8888
+ }
@@ -95,7 +95,7 @@
9595
+ StorageLive(_8);
9696
+ StorageDead(_8);
9797
+ _1 = GeneratorState::<i32, bool>::Complete(_5);
98-
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
98+
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
9999
+ discriminant((*_10)) = 1;
100100
+ goto -> bb1;
101101
+ }

‎tests/mir-opt/inline/inline_generator.main.Inline.panic-unwind.diff‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,30 @@
44
fn main() -> () {
55
let mut _0: ();
66
let _1: std::ops::GeneratorState<i32, bool>;
7-
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>;
8-
let mut _3: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
9-
let mut _4: [generator@$DIR/inline_generator.rs:15:5: 15:8];
7+
let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>;
8+
let mut _3: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
9+
let mut _4: [generator@$DIR/inline_generator.rs:16:5: 16:8];
1010
+ let mut _5: bool;
1111
scope 1 {
1212
debug _r => _1;
1313
}
1414
+ scope 2 (inlined g) {
1515
+ }
16-
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new) {
16+
+ scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new) {
1717
+ debug pointer => _3;
1818
+ scope 4 {
19-
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new_unchecked) {
19+
+ scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new_unchecked) {
2020
+ debug pointer => _3;
2121
+ }
2222
+ }
2323
+ }
2424
+ scope 6 (inlined g::{closure#0}) {
2525
+ debug a => _5;
26-
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
26+
+ let mut _6: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
2727
+ let mut _7: u32;
2828
+ let mut _8: i32;
29-
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
30-
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8];
29+
+ let mut _9: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
30+
+ let mut _10: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8];
3131
+ }
3232

3333
bb0: {
@@ -39,18 +39,18 @@
3939
- }
4040
-
4141
- bb1: {
42-
+ _4 = [generator@$DIR/inline_generator.rs:15:5: 15:8 (#0)];
42+
+ _4 = [generator@$DIR/inline_generator.rs:16:5: 16:8 (#0)];
4343
_3 = &mut _4;
44-
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new(move _3) -> [return: bb2, unwind: bb4];
44+
- _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]>::new(move _3) -> [return: bb2, unwind: bb4];
4545
- }
4646
-
4747
- bb2: {
48-
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _3 };
48+
+ _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:16:5: 16:8]> { pointer: move _3 };
4949
StorageDead(_3);
50-
- _1 = <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb4];
50+
- _1 = <[generator@$DIR/inline_generator.rs:16:5: 16:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb4];
5151
+ StorageLive(_5);
5252
+ _5 = const false;
53-
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
53+
+ _6 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
5454
+ _7 = discriminant((*_6));
5555
+ switchInt(move _7) -> [0: bb3, 1: bb7, 3: bb8, otherwise: bb9];
5656
}
@@ -87,7 +87,7 @@
8787
+
8888
+ bb6: {
8989
+ _1 = GeneratorState::<i32, bool>::Yielded(move _8);
90-
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
90+
+ _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
9191
+ discriminant((*_9)) = 3;
9292
+ goto -> bb1;
9393
+ }
@@ -100,7 +100,7 @@
100100
+ StorageLive(_8);
101101
+ StorageDead(_8);
102102
+ _1 = GeneratorState::<i32, bool>::Complete(_5);
103-
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]);
103+
+ _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:16:5: 16:8]);
104104
+ discriminant((*_10)) = 1;
105105
+ goto -> bb1;
106106
+ }

‎tests/mir-opt/inline/inline_generator.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
2+
// compile-flags: -Zinline-mir-hint-threshold=1000
23
#![feature(generators, generator_trait)]
34

45
use std::ops::Generator;
@@ -9,8 +10,8 @@ fn main() {
910
let _r = Pin::new(&mut g()).resume(false);
1011
}
1112

12-
#[inline(always)]
13+
#[inline]
1314
pub fn g() -> impl Generator<bool> {
14-
#[inline(always)]
15+
#[inline]
1516
|a| { yield if a { 7 } else { 13 } }
1617
}

0 commit comments

Comments
 (0)