Rollup merge of #82682 - petrochenkov:cfgeval, r=Aaron1011

Implement built-in attribute macro `#[cfg_eval]` + some refactoring

This PR implements a built-in attribute macro `#[cfg_eval]` as it was suggested in #79078 to avoid `#[derive()]` without arguments being abused as a way to configure input for other attributes.

The macro is used for eagerly expanding all `#[cfg]` and `#[cfg_attr]` attributes in its input ("fully configuring" the input).
The effect is identical to effect of `#[derive(Foo, Bar)]` which also fully configures its input before passing it to macros `Foo` and `Bar`, but unlike `#[derive]` `#[cfg_eval]` can be applied to any syntax nodes supporting macro attributes, not only certain items.

`cfg_eval` was the first name suggested in #79078, but other alternatives are also possible, e.g. `cfg_expand`.

```rust
#[cfg_eval]
#[my_attr] // Receives `struct S {}` as input, the field is configured away by `#[cfg_eval]`
struct S {
    #[cfg(FALSE)]
    field: u8,
}
```

Tracking issue: #82679

Author: Dylan-DPC

compiler/rustc_ast/src/ast.rs

@@ -915,16 +915,6 @@ impl Stmt {
         }
     }
 
-    pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> {
-        match self.kind {
-            StmtKind::Local(ref mut local) => local.tokens.as_mut(),
-            StmtKind::Item(ref mut item) => item.tokens.as_mut(),
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(),
-            StmtKind::Empty => None,
-            StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(),
-        }
-    }
-
     pub fn has_trailing_semicolon(&self) -> bool {
         match &self.kind {
             StmtKind::Semi(_) => true,

compiler/rustc_ast/src/ast_like.rs

@@ -11,13 +11,7 @@ use super::{AttrVec, Attribute, Stmt, StmtKind};
 pub trait AstLike: Sized {
     fn attrs(&self) -> &[Attribute];
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
-    /// Called by `Parser::collect_tokens` to store the collected
-    /// tokens inside an AST node
-    fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {
-        // This default impl makes this trait easier to implement
-        // in tools like `rust-analyzer`
-        panic!("`finalize_tokens` is not supported!")
-    }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
 }
 
 impl<T: AstLike + 'static> AstLike for P<T> {
@@ -27,8 +21,8 @@ impl<T: AstLike + 'static> AstLike for P<T> {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         (**self).visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        (**self).finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        (**self).tokens_mut()
     }
 }
 
@@ -42,12 +36,12 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
 
 impl AstLike for StmtKind {
     fn attrs(&self) -> &[Attribute] {
-        match *self {
-            StmtKind::Local(ref local) => local.attrs(),
-            StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(),
-            StmtKind::Item(ref item) => item.attrs(),
+        match self {
+            StmtKind::Local(local) => local.attrs(),
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.attrs(),
+            StmtKind::Item(item) => item.attrs(),
             StmtKind::Empty => &[],
-            StmtKind::MacCall(ref mac) => &*mac.attrs,
+            StmtKind::MacCall(mac) => &mac.attrs,
         }
     }
 
@@ -60,17 +54,14 @@ impl AstLike for StmtKind {
             StmtKind::MacCall(mac) => visit_attrvec(&mut mac.attrs, f),
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        let stmt_tokens = match self {
-            StmtKind::Local(ref mut local) => &mut local.tokens,
-            StmtKind::Item(ref mut item) => &mut item.tokens,
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens,
-            StmtKind::Empty => return,
-            StmtKind::MacCall(ref mut mac) => &mut mac.tokens,
-        };
-        if stmt_tokens.is_none() {
-            *stmt_tokens = Some(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match self {
+            StmtKind::Local(local) => &mut local.tokens,
+            StmtKind::Item(item) => &mut item.tokens,
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => &mut expr.tokens,
+            StmtKind::Empty => return None,
+            StmtKind::MacCall(mac) => &mut mac.tokens,
+        })
     }
 }
 
@@ -82,8 +73,8 @@ impl AstLike for Stmt {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         self.kind.visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        self.kind.finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.kind.tokens_mut()
     }
 }
 
@@ -92,17 +83,13 @@ impl AstLike for Attribute {
         &[]
     }
     fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        match &mut self.kind {
-            AttrKind::Normal(_, attr_tokens) => {
-                if attr_tokens.is_none() {
-                    *attr_tokens = Some(tokens);
-                }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match &mut self.kind {
+            AttrKind::Normal(_, tokens) => tokens,
+            kind @ AttrKind::DocComment(..) => {
+                panic!("Called tokens_mut on doc comment attr {:?}", kind)
             }
-            AttrKind::DocComment(..) => {
-                panic!("Called finalize_tokens on doc comment attr {:?}", self)
-            }
-        }
+        })
     }
 }
 
@@ -115,10 +102,8 @@ impl<T: AstLike> AstLike for Option<T> {
             inner.visit_attrs(f);
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        if let Some(inner) = self {
-            inner.finalize_tokens(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
@@ -152,11 +137,8 @@ macro_rules! derive_has_tokens_and_attrs {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
-
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
         }
     )* }
@@ -173,7 +155,9 @@ macro_rules! derive_has_attrs_no_tokens {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {}
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                None
+            }
         }
     )* }
 }
@@ -185,14 +169,10 @@ macro_rules! derive_has_tokens_no_attrs {
                 &[]
             }
 
-            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {
-            }
-
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
+            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
 
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
         }
     )* }

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -0,0 +1,157 @@
+use crate::util::check_builtin_macro_attribute;
+
+use rustc_ast::mut_visit::{self, MutVisitor};
+use rustc_ast::ptr::P;
+use rustc_ast::{self as ast, AstLike};
+use rustc_expand::base::{Annotatable, ExtCtxt};
+use rustc_expand::config::StripUnconfigured;
+use rustc_expand::configure;
+use rustc_span::symbol::sym;
+use rustc_span::Span;
+use smallvec::SmallVec;
+
+crate fn expand(
+    ecx: &mut ExtCtxt<'_>,
+    _span: Span,
+    meta_item: &ast::MetaItem,
+    annotatable: Annotatable,
+) -> Vec<Annotatable> {
+    check_builtin_macro_attribute(ecx, meta_item, sym::cfg_eval);
+    cfg_eval(ecx, annotatable)
+}
+
+crate fn cfg_eval(ecx: &ExtCtxt<'_>, annotatable: Annotatable) -> Vec<Annotatable> {
+    let mut visitor = CfgEval {
+        cfg: StripUnconfigured { sess: ecx.sess, features: ecx.ecfg.features, modified: false },
+    };
+    let mut annotatable = visitor.configure_annotatable(annotatable);
+    if visitor.cfg.modified {
+        // Erase the tokens if cfg-stripping modified the item
+        // This will cause us to synthesize fake tokens
+        // when `nt_to_tokenstream` is called on this item.
+        if let Some(tokens) = annotatable.tokens_mut() {
+            *tokens = None;
+        }
+    }
+    vec![annotatable]
+}
+
+struct CfgEval<'a> {
+    cfg: StripUnconfigured<'a>,
+}
+
+impl CfgEval<'_> {
+    fn configure<T: AstLike>(&mut self, node: T) -> Option<T> {
+        self.cfg.configure(node)
+    }
+
+    fn configure_annotatable(&mut self, annotatable: Annotatable) -> Annotatable {
+        // Since the item itself has already been configured by the InvocationCollector,
+        // we know that fold result vector will contain exactly one element
+        match annotatable {
+            Annotatable::Item(item) => Annotatable::Item(self.flat_map_item(item).pop().unwrap()),
+            Annotatable::TraitItem(item) => {
+                Annotatable::TraitItem(self.flat_map_trait_item(item).pop().unwrap())
+            }
+            Annotatable::ImplItem(item) => {
+                Annotatable::ImplItem(self.flat_map_impl_item(item).pop().unwrap())
+            }
+            Annotatable::ForeignItem(item) => {
+                Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
+            }
+            Annotatable::Stmt(stmt) => {
+                Annotatable::Stmt(stmt.map(|stmt| self.flat_map_stmt(stmt).pop().unwrap()))
+            }
+            Annotatable::Expr(mut expr) => Annotatable::Expr({
+                self.visit_expr(&mut expr);
+                expr
+            }),
+            Annotatable::Arm(arm) => Annotatable::Arm(self.flat_map_arm(arm).pop().unwrap()),
+            Annotatable::Field(field) => {
+                Annotatable::Field(self.flat_map_field(field).pop().unwrap())
+            }
+            Annotatable::FieldPat(fp) => {
+                Annotatable::FieldPat(self.flat_map_field_pattern(fp).pop().unwrap())
+            }
+            Annotatable::GenericParam(param) => {
+                Annotatable::GenericParam(self.flat_map_generic_param(param).pop().unwrap())
+            }
+            Annotatable::Param(param) => {
+                Annotatable::Param(self.flat_map_param(param).pop().unwrap())
+            }
+            Annotatable::StructField(sf) => {
+                Annotatable::StructField(self.flat_map_struct_field(sf).pop().unwrap())
+            }
+            Annotatable::Variant(v) => {
+                Annotatable::Variant(self.flat_map_variant(v).pop().unwrap())
+            }
+        }
+    }
+}
+
+impl MutVisitor for CfgEval<'_> {
+    fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
+        self.cfg.configure_expr(expr);
+        mut_visit::noop_visit_expr(expr, self);
+    }
+
+    fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        let mut expr = configure!(self, expr);
+        mut_visit::noop_visit_expr(&mut expr, self);
+        Some(expr)
+    }
+
+    fn flat_map_generic_param(
+        &mut self,
+        param: ast::GenericParam,
+    ) -> SmallVec<[ast::GenericParam; 1]> {
+        mut_visit::noop_flat_map_generic_param(configure!(self, param), self)
+    }
+
+    fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> {
+        mut_visit::noop_flat_map_stmt(configure!(self, stmt), self)
+    }
+
+    fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
+        mut_visit::noop_flat_map_item(configure!(self, item), self)
+    }
+
+    fn flat_map_impl_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
+        mut_visit::noop_flat_map_assoc_item(configure!(self, item), self)
+    }
+
+    fn flat_map_trait_item(&mut self, item: P<ast::AssocItem>) -> SmallVec<[P<ast::AssocItem>; 1]> {
+        mut_visit::noop_flat_map_assoc_item(configure!(self, item), self)
+    }
+
+    fn flat_map_foreign_item(
+        &mut self,
+        foreign_item: P<ast::ForeignItem>,
+    ) -> SmallVec<[P<ast::ForeignItem>; 1]> {
+        mut_visit::noop_flat_map_foreign_item(configure!(self, foreign_item), self)
+    }
+
+    fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> {
+        mut_visit::noop_flat_map_arm(configure!(self, arm), self)
+    }
+
+    fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> {
+        mut_visit::noop_flat_map_field(configure!(self, field), self)
+    }
+
+    fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> {
+        mut_visit::noop_flat_map_field_pattern(configure!(self, fp), self)
+    }
+
+    fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> {
+        mut_visit::noop_flat_map_param(configure!(self, p), self)
+    }
+
+    fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> {
+        mut_visit::noop_flat_map_struct_field(configure!(self, sf), self)
+    }
+
+    fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> {
+        mut_visit::noop_flat_map_variant(configure!(self, variant), self)
+    }
+}

compiler/rustc_builtin_macros/src/derive.rs

@@ -1,7 +1,8 @@
+use crate::cfg_eval::cfg_eval;
+
 use rustc_ast::{self as ast, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
-use rustc_expand::config::StripUnconfigured;
 use rustc_feature::AttributeTemplate;
 use rustc_parse::validate_attr;
 use rustc_session::Session;
@@ -51,26 +52,7 @@ impl MultiItemModifier for Expander {
 
         // FIXME: Try to cache intermediate results to avoid collecting same paths multiple times.
         match ecx.resolver.resolve_derives(ecx.current_expansion.id, derives, ecx.force_mode) {
-            Ok(()) => {
-                let mut visitor =
-                    StripUnconfigured { sess, features: ecx.ecfg.features, modified: false };
-                let mut item = visitor.fully_configure(item);
-                if visitor.modified {
-                    // Erase the tokens if cfg-stripping modified the item
-                    // This will cause us to synthesize fake tokens
-                    // when `nt_to_tokenstream` is called on this item.
-                    match &mut item {
-                        Annotatable::Item(item) => item,
-                        Annotatable::Stmt(stmt) => match &mut stmt.kind {
-                            StmtKind::Item(item) => item,
-                            _ => unreachable!(),
-                        },
-                        _ => unreachable!(),
-                    }
-                    .tokens = None;
-                }
-                ExpandResult::Ready(vec![item])
-            }
+            Ok(()) => ExpandResult::Ready(cfg_eval(ecx, item)),
             Err(Indeterminate) => ExpandResult::Retry(item),
         }
     }

compiler/rustc_builtin_macros/src/lib.rs

@@ -24,6 +24,7 @@ mod asm;
 mod assert;
 mod cfg;
 mod cfg_accessible;
+mod cfg_eval;
 mod compile_error;
 mod concat;
 mod concat_idents;
@@ -89,6 +90,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     register_attr! {
         bench: test::expand_bench,
         cfg_accessible: cfg_accessible::Expander,
+        cfg_eval: cfg_eval::expand,
         derive: derive::Expander,
         global_allocator: global_allocator::expand,
         test: test::expand_test,