Rollup merge of rust-lang#57677 - dotdash:locals, r=michaelwoerister

const_eval: Predetermine the layout of all locals when pushing a stack frame

Usually the layout of any locals is required at least three times, once
when it becomes live, once when it is written to, and once it is read
from. By adding a cache for them, we can reduce the number of layout
queries speeding up code that is heavy on const_eval.

Author: Centril

src/librustc_mir/const_eval.rs

@@ -72,6 +72,7 @@ fn mk_eval_cx_inner<'a, 'mir, 'tcx>(
     ecx.stack.push(interpret::Frame {
         block: mir::START_BLOCK,
         locals: IndexVec::new(),
+        local_layouts: IndexVec::new(),
         instance,
         span,
         mir,

src/librustc_mir/interpret/eval_context.rs

@@ -1,3 +1,4 @@
+use std::cell::Cell;
 use std::fmt::Write;
 use std::mem;
 
@@ -76,6 +77,7 @@ pub struct Frame<'mir, 'tcx: 'mir, Tag=(), Extra=()> {
     /// `None` represents a local that is currently dead, while a live local
     /// can either directly contain `Scalar` or refer to some part of an `Allocation`.
     pub locals: IndexVec<mir::Local, LocalValue<Tag>>,
+    pub local_layouts: IndexVec<mir::Local, Cell<Option<TyLayout<'tcx>>>>,
 
     ////////////////////////////////////////////////////////////////////////////////
     // Current position within the function
@@ -290,9 +292,15 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
         frame: &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>,
         local: mir::Local
     ) -> EvalResult<'tcx, TyLayout<'tcx>> {
-        let local_ty = frame.mir.local_decls[local].ty;
-        let local_ty = self.monomorphize(local_ty, frame.instance.substs);
-        self.layout_of(local_ty)
+        let cell = &frame.local_layouts[local];
+        if cell.get().is_none() {
+            let local_ty = frame.mir.local_decls[local].ty;
+            let local_ty = self.monomorphize(local_ty, frame.instance.substs);
+            let layout = self.layout_of(local_ty)?;
+            cell.set(Some(layout));
+        }
+
+        Ok(cell.get().unwrap())
     }
 
     pub fn str_to_immediate(&mut self, s: &str) -> EvalResult<'tcx, Immediate<M::PointerTag>> {
@@ -426,6 +434,7 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
             // empty local array, we fill it in below, after we are inside the stack frame and
             // all methods actually know about the frame
             locals: IndexVec::new(),
+            local_layouts: IndexVec::from_elem_n(Default::default(), mir.local_decls.len()),
             span,
             instance,
             stmt: 0,
@@ -464,11 +473,11 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
                 },
             }
             // Finally, properly initialize all those that still have the dummy value
-            for (local, decl) in locals.iter_mut().zip(mir.local_decls.iter()) {
+            for (idx, local) in locals.iter_enumerated_mut() {
                 match *local {
                     LocalValue::Live(_) => {
                         // This needs to be peoperly initialized.
-                        let layout = self.layout_of(self.monomorphize(decl.ty, instance.substs))?;
+                        let layout = self.layout_of_local(self.frame(), idx)?;
                         *local = LocalValue::Live(self.uninit_operand(layout)?);
                     }
                     LocalValue::Dead => {

src/librustc_mir/interpret/operand.rs

@@ -457,36 +457,30 @@ impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M>
     }
 
     /// This is used by [priroda](https://github.com/oli-obk/priroda) to get an OpTy from a local
-    ///
-    /// When you know the layout of the local in advance, you can pass it as last argument
-    pub fn access_local(
+    fn access_local(
         &self,
         frame: &super::Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>,
         local: mir::Local,
-        layout: Option<TyLayout<'tcx>>,
     ) -> EvalResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         assert_ne!(local, mir::RETURN_PLACE);
         let op = *frame.locals[local].access()?;
-        let layout = from_known_layout(layout,
-                    || self.layout_of_local(frame, local))?;
+        let layout = self.layout_of_local(frame, local)?;
         Ok(OpTy { op, layout })
     }
 
     // Evaluate a place with the goal of reading from it.  This lets us sometimes
-    // avoid allocations.  If you already know the layout, you can pass it in
-    // to avoid looking it up again.
+    // avoid allocations.
     fn eval_place_to_op(
         &self,
         mir_place: &mir::Place<'tcx>,
-        layout: Option<TyLayout<'tcx>>,
     ) -> EvalResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         use rustc::mir::Place::*;
         let op = match *mir_place {
             Local(mir::RETURN_PLACE) => return err!(ReadFromReturnPointer),
-            Local(local) => self.access_local(self.frame(), local, layout)?,
+            Local(local) => self.access_local(self.frame(), local)?,
 
             Projection(ref proj) => {
-                let op = self.eval_place_to_op(&proj.base, None)?;
+                let op = self.eval_place_to_op(&proj.base)?;
                 self.operand_projection(op, &proj.elem)?
             }
 
@@ -510,7 +504,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M>
             // FIXME: do some more logic on `move` to invalidate the old location
             Copy(ref place) |
             Move(ref place) =>
-                self.eval_place_to_op(place, layout)?,
+                self.eval_place_to_op(place)?,
 
             Constant(ref constant) => {
                 let layout = from_known_layout(layout, || {

src/librustc_mir/interpret/snapshot.rs

@@ -314,13 +314,14 @@ struct FrameSnapshot<'a, 'tcx: 'a> {
     stmt: usize,
 }
 
-impl_stable_hash_for!(impl<'tcx, 'mir: 'tcx> for struct Frame<'mir, 'tcx> {
+impl_stable_hash_for!(impl<'mir, 'tcx: 'mir> for struct Frame<'mir, 'tcx> {
     mir,
     instance,
     span,
     return_to_block,
     return_place -> (return_place.as_ref().map(|r| &**r)),
     locals,
+    local_layouts -> _,
     block,
     stmt,
     extra,
@@ -339,6 +340,7 @@ impl<'a, 'mir, 'tcx, Ctx> Snapshot<'a, Ctx> for &'a Frame<'mir, 'tcx>
             return_to_block,
             return_place,
             locals,
+            local_layouts: _,
             block,
             stmt,
             extra: _,