Skip to content

Commit df19914

Browse files
saethlinsaethlin
committed
Monomorphize ahead-of-time
1 parent 0c3c070 commit df19914

File tree

12 files changed

+96
-50
lines changed

12 files changed

+96
-50
lines changed

‎compiler/rustc_codegen_ssa/src/mir/analyze.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind};
1212
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
1313
use rustc_middle::{bug, span_bug};
1414

15-
pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
16-
fx: &FunctionCx<'a, 'tcx, Bx>,
15+
pub fn non_ssa_locals<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
16+
fx: &FunctionCx<'body, 'a, 'tcx, Bx>,
1717
) -> BitSet<mir::Local> {
1818
let mir = fx.mir;
1919
let dominators = mir.basic_blocks.dominators();
@@ -68,9 +68,9 @@ enum LocalKind {
6868
SSA(DefLocation),
6969
}
7070

71-
struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
72-
fx: &'mir FunctionCx<'a, 'tcx, Bx>,
73-
dominators: &'mir Dominators<mir::BasicBlock>,
71+
struct LocalAnalyzer<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
72+
fx: &'body FunctionCx<'body, 'a, 'tcx, Bx>,
73+
dominators: &'body Dominators<mir::BasicBlock>,
7474
locals: IndexVec<mir::Local, LocalKind>,
7575
}
7676

‎compiler/rustc_codegen_ssa/src/mir/block.rs‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -38,17 +38,17 @@ enum MergingSucc {
3838

3939
/// Used by `FunctionCx::codegen_terminator` for emitting common patterns
4040
/// e.g., creating a basic block, calling a function, etc.
41-
struct TerminatorCodegenHelper<'tcx> {
41+
struct TerminatorCodegenHelper<'body, 'tcx> {
4242
bb: mir::BasicBlock,
43-
terminator: &'tcx mir::Terminator<'tcx>,
43+
terminator: &'body mir::Terminator<'tcx>,
4444
}
4545

46-
impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
46+
impl<'body, 'a, 'tcx> TerminatorCodegenHelper<'body, 'tcx> {
4747
/// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
4848
/// either already previously cached, or newly created, by `landing_pad_for`.
4949
fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
5050
&self,
51-
fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
51+
fx: &'b mut FunctionCx<'body, 'a, 'tcx, Bx>,
5252
) -> Option<&'b Bx::Funclet> {
5353
let cleanup_kinds = fx.cleanup_kinds.as_ref()?;
5454
let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
@@ -74,7 +74,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
7474
/// stuff in it or next to it.
7575
fn llbb_with_cleanup<Bx: BuilderMethods<'a, 'tcx>>(
7676
&self,
77-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
77+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
7878
target: mir::BasicBlock,
7979
) -> Bx::BasicBlock {
8080
let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target);
@@ -98,7 +98,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
9898

9999
fn llbb_characteristics<Bx: BuilderMethods<'a, 'tcx>>(
100100
&self,
101-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
101+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
102102
target: mir::BasicBlock,
103103
) -> (bool, bool) {
104104
if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
@@ -123,7 +123,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
123123

124124
fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
125125
&self,
126-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
126+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
127127
bx: &mut Bx,
128128
target: mir::BasicBlock,
129129
mergeable_succ: bool,
@@ -152,7 +152,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
152152
/// return destination `destination` and the unwind action `unwind`.
153153
fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
154154
&self,
155-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
155+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
156156
bx: &mut Bx,
157157
fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
158158
fn_ptr: Bx::Value,
@@ -271,7 +271,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
271271
/// Generates inline assembly with optional `destination` and `unwind`.
272272
fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
273273
&self,
274-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
274+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
275275
bx: &mut Bx,
276276
template: &[InlineAsmTemplatePiece],
277277
operands: &[InlineAsmOperandRef<'tcx, Bx>],
@@ -338,9 +338,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
338338
}
339339

340340
/// Codegen implementations for some terminator variants.
341-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
341+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
342342
/// Generates code for a `Resume` terminator.
343-
fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) {
343+
fn codegen_resume_terminator(
344+
&mut self,
345+
helper: TerminatorCodegenHelper<'body, 'tcx>,
346+
bx: &mut Bx,
347+
) {
344348
if let Some(funclet) = helper.funclet(self) {
345349
bx.cleanup_ret(funclet, None);
346350
} else {
@@ -357,7 +361,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
357361

358362
fn codegen_switchint_terminator(
359363
&mut self,
360-
helper: TerminatorCodegenHelper<'tcx>,
364+
helper: TerminatorCodegenHelper<'body, 'tcx>,
361365
bx: &mut Bx,
362366
discr: &mir::Operand<'tcx>,
363367
targets: &SwitchTargets,
@@ -497,7 +501,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
497501
#[tracing::instrument(level = "trace", skip(self, helper, bx))]
498502
fn codegen_drop_terminator(
499503
&mut self,
500-
helper: TerminatorCodegenHelper<'tcx>,
504+
helper: TerminatorCodegenHelper<'body, 'tcx>,
501505
bx: &mut Bx,
502506
location: mir::Place<'tcx>,
503507
target: mir::BasicBlock,
@@ -622,7 +626,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
622626

623627
fn codegen_assert_terminator(
624628
&mut self,
625-
helper: TerminatorCodegenHelper<'tcx>,
629+
helper: TerminatorCodegenHelper<'body, 'tcx>,
626630
bx: &mut Bx,
627631
terminator: &mir::Terminator<'tcx>,
628632
cond: &mir::Operand<'tcx>,
@@ -701,7 +705,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
701705

702706
fn codegen_terminate_terminator(
703707
&mut self,
704-
helper: TerminatorCodegenHelper<'tcx>,
708+
helper: TerminatorCodegenHelper<'body, 'tcx>,
705709
bx: &mut Bx,
706710
terminator: &mir::Terminator<'tcx>,
707711
reason: UnwindTerminateReason,
@@ -731,7 +735,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
731735
/// Returns `Some` if this is indeed a panic intrinsic and codegen is done.
732736
fn codegen_panic_intrinsic(
733737
&mut self,
734-
helper: &TerminatorCodegenHelper<'tcx>,
738+
helper: &TerminatorCodegenHelper<'body, 'tcx>,
735739
bx: &mut Bx,
736740
intrinsic: Option<ty::IntrinsicDef>,
737741
instance: Option<Instance<'tcx>>,
@@ -800,7 +804,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
800804

801805
fn codegen_call_terminator(
802806
&mut self,
803-
helper: TerminatorCodegenHelper<'tcx>,
807+
helper: TerminatorCodegenHelper<'body, 'tcx>,
804808
bx: &mut Bx,
805809
terminator: &mir::Terminator<'tcx>,
806810
func: &mir::Operand<'tcx>,
@@ -1152,7 +1156,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
11521156

11531157
fn codegen_asm_terminator(
11541158
&mut self,
1155-
helper: TerminatorCodegenHelper<'tcx>,
1159+
helper: TerminatorCodegenHelper<'body, 'tcx>,
11561160
bx: &mut Bx,
11571161
terminator: &mir::Terminator<'tcx>,
11581162
template: &[ast::InlineAsmTemplatePiece],
@@ -1236,7 +1240,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
12361240
}
12371241
}
12381242

1239-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
1243+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
12401244
pub fn codegen_block(&mut self, mut bb: mir::BasicBlock) {
12411245
let llbb = match self.try_llbb(bb) {
12421246
Some(llbb) => llbb,
@@ -1291,7 +1295,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
12911295
&mut self,
12921296
bx: &mut Bx,
12931297
bb: mir::BasicBlock,
1294-
terminator: &'tcx mir::Terminator<'tcx>,
1298+
terminator: &'body mir::Terminator<'tcx>,
12951299
) -> MergingSucc {
12961300
debug!("codegen_terminator: {:?}", terminator);
12971301

‎compiler/rustc_codegen_ssa/src/mir/constant.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use rustc_target::abi::Abi;
1010

1111
use super::FunctionCx;
1212

13-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
13+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
1414
pub fn eval_mir_constant_to_operand(
1515
&self,
1616
bx: &mut Bx,

‎compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use rustc_middle::mir::SourceScope;
55

66
use super::FunctionCx;
77

8-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
8+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
99
pub fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) {
1010
// Determine the instance that coverage data was originally generated for.
1111
let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) {

‎compiler/rustc_codegen_ssa/src/mir/debuginfo.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ fn calculate_debuginfo_offset<
203203
DebugInfoOffset { direct_offset, indirect_offsets, result: place }
204204
}
205205

206-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
206+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
207207
pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) {
208208
bx.set_span(source_info.span);
209209
if let Some(dbg_loc) = self.dbg_loc(source_info) {

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
5454
bx.memset(dst, val, size, align, flags);
5555
}
5656

57-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
57+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
5858
/// In the `Err` case, returns the instance that should be called instead.
5959
pub fn codegen_intrinsic_call(
6060
bx: &mut Bx,

‎compiler/rustc_codegen_ssa/src/mir/locals.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ impl<'tcx, V> Locals<'tcx, V> {
3232
}
3333
}
3434

35-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
35+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
3636
pub(super) fn initialize_locals(&mut self, values: Vec<LocalRef<'tcx, Bx::Value>>) {
3737
assert!(self.locals.values.is_empty());
3838
// FIXME(#115215): After #115025 get's merged this might not be necessary

‎compiler/rustc_codegen_ssa/src/mir/mod.rs‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ enum CachedLlbb<T> {
4242
}
4343

4444
/// Master context for codegenning from MIR.
45-
pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
45+
pub struct FunctionCx<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
4646
instance: Instance<'tcx>,
4747

48-
mir: &'tcx mir::Body<'tcx>,
48+
mir: &'body mir::Body<'tcx>,
4949

5050
debug_context: Option<FunctionDebugContext<'tcx, Bx::DIScope, Bx::DILocation>>,
5151

@@ -114,17 +114,12 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
114114
caller_location: Option<OperandRef<'tcx, Bx::Value>>,
115115
}
116116

117-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
117+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
118118
pub fn monomorphize<T>(&self, value: T) -> T
119119
where
120120
T: Copy + TypeFoldable<TyCtxt<'tcx>>,
121121
{
122-
debug!("monomorphize: self.instance={:?}", self.instance);
123-
self.instance.instantiate_mir_and_normalize_erasing_regions(
124-
self.cx.tcx(),
125-
ty::ParamEnv::reveal_all(),
126-
ty::EarlyBinder::bind(value),
127-
)
122+
value
128123
}
129124
}
130125

@@ -154,7 +149,51 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
154149
}
155150
}
156151

157-
///////////////////////////////////////////////////////////////////////////
152+
struct Folder<'tcx> {
153+
instance: Instance<'tcx>,
154+
tcx: TyCtxt<'tcx>,
155+
}
156+
157+
impl<'tcx> rustc_middle::ty::TypeFolder<TyCtxt<'tcx>> for Folder<'tcx> {
158+
fn interner(&self) -> TyCtxt<'tcx> {
159+
self.tcx
160+
}
161+
162+
fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
163+
&mut self,
164+
t: ty::Binder<'tcx, T>,
165+
) -> ty::Binder<'tcx, T> {
166+
self.instance.instantiate_mir_and_normalize_erasing_regions(
167+
self.tcx,
168+
ty::ParamEnv::reveal_all(),
169+
ty::EarlyBinder::bind(t),
170+
)
171+
}
172+
173+
fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
174+
self.instance.instantiate_mir_and_normalize_erasing_regions(
175+
self.tcx,
176+
ty::ParamEnv::reveal_all(),
177+
ty::EarlyBinder::bind(ty),
178+
)
179+
}
180+
181+
fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
182+
self.instance.instantiate_mir_and_normalize_erasing_regions(
183+
self.tcx,
184+
ty::ParamEnv::reveal_all(),
185+
ty::EarlyBinder::bind(ct),
186+
)
187+
}
188+
189+
fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
190+
self.instance.instantiate_mir_and_normalize_erasing_regions(
191+
self.tcx,
192+
ty::ParamEnv::reveal_all(),
193+
ty::EarlyBinder::bind(p),
194+
)
195+
}
196+
}
158197

159198
#[instrument(level = "debug", skip(cx))]
160199
pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
@@ -164,10 +203,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
164203
assert!(!instance.args.has_infer());
165204

166205
let llfn = cx.get_fn(instance);
206+
let tcx = cx.tcx();
167207

168-
let mir = cx.tcx().instance_mir(instance.def);
169-
let body: rustc_middle::mir::Body<'tcx> = mir.clone();
170-
drop(std::hint::black_box(body));
208+
let mir = tcx
209+
.instance_mir(instance.def)
210+
.clone()
211+
.fold_with(&mut Folder { tcx, instance: instance.polymorphize(tcx) });
212+
let mir = &mir;
171213

172214
let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
173215
debug!("fn_abi: {:?}", fn_abi);
@@ -286,9 +328,9 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
286328
/// Produces, for each argument, a `Value` pointing at the
287329
/// argument's value. As arguments are places, these are always
288330
/// indirect.
289-
fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
331+
fn arg_local_refs<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
290332
bx: &mut Bx,
291-
fx: &mut FunctionCx<'a, 'tcx, Bx>,
333+
fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>,
292334
memory_locals: &BitSet<mir::Local>,
293335
) -> Vec<LocalRef<'tcx, Bx::Value>> {
294336
let mir = fx.mir;

‎compiler/rustc_codegen_ssa/src/mir/operand.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
512512
}
513513
}
514514

515-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
515+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
516516
fn maybe_codegen_consume_direct(
517517
&mut self,
518518
bx: &mut Bx,

‎compiler/rustc_codegen_ssa/src/mir/place.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
430430
}
431431
}
432432

433-
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
433+
impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> {
434434
#[instrument(level = "trace", skip(self, bx))]
435435
pub fn codegen_place(
436436
&mut self,

0 commit comments

Comments
 (0)