macros: Clean up code with non-optional NonterminalKind

Since [1], the fragment specifier is unconditionally required in all
editions. This means `NonTerminalKind` no longer needs to be optional,
as we can reject this code during the expansion of `macro_rules!` rather
than handling it throughout the code. Do this cleanup here.

[1]: #128425

Author: tgross35

compiler/rustc_ast/src/token.rs

@@ -1085,6 +1085,7 @@ pub enum NtExprKind {
     Expr2021 { inferred: bool },
 }
 
+/// A macro nonterminal, known in documentation as a fragment specifier.
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NonterminalKind {
     Item,

compiler/rustc_expand/src/mbe.rs

@@ -78,7 +78,13 @@ enum TokenTree {
     /// only covers the ident, e.g. `var`.)
     MetaVar(Span, Ident),
     /// e.g., `$var:expr`. Only appears on the LHS.
-    MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
+    MetaVarDecl {
+        span: Span,
+        /// Name to bind.
+        name: Ident,
+        /// The fragment specifier.
+        kind: NonterminalKind,
+    },
     /// A meta-variable expression inside `${...}`.
     MetaVarExpr(DelimSpan, MetaVarExpr),
 }
@@ -102,7 +108,7 @@ impl TokenTree {
         match *self {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
-            | TokenTree::MetaVarDecl(span, _, _) => span,
+            | TokenTree::MetaVarDecl { span, .. } => span,
             TokenTree::Delimited(span, ..)
             | TokenTree::MetaVarExpr(span, _)
             | TokenTree::Sequence(span, _) => span.entire(),

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -24,6 +24,7 @@ pub(super) fn failed_to_match_macro(
     arg: TokenStream,
     lhses: &[Vec<MatcherLoc>],
 ) -> (Span, ErrorGuaranteed) {
+    debug!("failed to match macro");
     // An error occurred, try the expansion again, tracking the expansion closely for better
     // diagnostics.
     let mut tracker = CollectTrackerAndEmitter::new(psess.dcx(), sp);

compiler/rustc_expand/src/mbe/macro_check.rs

@@ -117,7 +117,6 @@ use rustc_session::parse::ParseSess;
 use rustc_span::{ErrorGuaranteed, MacroRulesNormalizedIdent, Span, kw};
 use smallvec::SmallVec;
 
-use super::quoted::VALID_FRAGMENT_NAMES_MSG;
 use crate::errors;
 use crate::mbe::{KleeneToken, TokenTree};
 
@@ -263,14 +262,7 @@ fn check_binders(
             }
         }
         // Similarly, this can only happen when checking a toplevel macro.
-        TokenTree::MetaVarDecl(span, name, kind) => {
-            if kind.is_none() && node_id != DUMMY_NODE_ID {
-                psess.dcx().emit_err(errors::MissingFragmentSpecifier {
-                    span,
-                    add_span: span.shrink_to_hi(),
-                    valid: VALID_FRAGMENT_NAMES_MSG,
-                });
-            }
+        TokenTree::MetaVarDecl { span, name, .. } => {
             if !macros.is_empty() {
                 psess.dcx().span_bug(span, "unexpected MetaVarDecl in nested lhs");
             }
@@ -339,7 +331,7 @@ fn check_occurrences(
 ) {
     match *rhs {
         TokenTree::Token(..) => {}
-        TokenTree::MetaVarDecl(span, _name, _kind) => {
+        TokenTree::MetaVarDecl { span, .. } => {
             psess.dcx().span_bug(span, "unexpected MetaVarDecl in rhs")
         }
         TokenTree::MetaVar(span, name) => {

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -122,7 +122,7 @@ pub(crate) enum MatcherLoc {
     MetaVarDecl {
         span: Span,
         bind: Ident,
-        kind: Option<NonterminalKind>,
+        kind: NonterminalKind,
         next_metavar: usize,
         seq_depth: usize,
     },
@@ -151,12 +151,7 @@ impl Display for MatcherLoc {
                 write!(f, "{}", token_descr(token))
             }
             MatcherLoc::MetaVarDecl { bind, kind, .. } => {
-                write!(f, "meta-variable `${bind}")?;
-                if let Some(kind) = kind {
-                    write!(f, ":{kind}")?;
-                }
-                write!(f, "`")?;
-                Ok(())
+                write!(f, "meta-variable `${bind}:{kind}`")
             }
             MatcherLoc::Eof => f.write_str("end of macro"),
 
@@ -220,7 +215,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
                         seq_depth,
                     };
                 }
-                &TokenTree::MetaVarDecl(span, bind, kind) => {
+                &TokenTree::MetaVarDecl { span, name: bind, kind } => {
                     locs.push(MatcherLoc::MetaVarDecl {
                         span,
                         bind,
@@ -330,7 +325,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
     matcher
         .iter()
         .map(|tt| match tt {
-            TokenTree::MetaVarDecl(..) => 1,
+            TokenTree::MetaVarDecl { .. } => 1,
             TokenTree::Sequence(_, seq) => seq.num_captures,
             TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts),
             TokenTree::Token(..) => 0,
@@ -551,18 +546,12 @@ impl TtParser {
                     mp.idx = idx_first;
                     self.cur_mps.push(mp);
                 }
-                &MatcherLoc::MetaVarDecl { span, kind, .. } => {
+                &MatcherLoc::MetaVarDecl { kind, .. } => {
                     // Built-in nonterminals never start with these tokens, so we can eliminate
                     // them from consideration. We use the span of the metavariable declaration
                     // to determine any edition-specific matching behavior for non-terminals.
-                    if let Some(kind) = kind {
-                        if Parser::nonterminal_may_begin_with(kind, token) {
-                            self.bb_mps.push(mp);
-                        }
-                    } else {
-                        // E.g. `$e` instead of `$e:expr`, reported as a hard error if actually used.
-                        // Both this check and the one in `nameize` are necessary, surprisingly.
-                        return Some(Error(span, "missing fragment specifier".to_string()));
+                    if Parser::nonterminal_may_begin_with(kind, token) {
+                        self.bb_mps.push(mp);
                     }
                 }
                 MatcherLoc::Eof => {
@@ -666,11 +655,7 @@ impl TtParser {
                     let mut mp = self.bb_mps.pop().unwrap();
                     let loc = &matcher[mp.idx];
                     if let &MatcherLoc::MetaVarDecl {
-                        span,
-                        kind: Some(kind),
-                        next_metavar,
-                        seq_depth,
-                        ..
+                        span, kind, next_metavar, seq_depth, ..
                     } = loc
                     {
                         // We use the span of the metavariable declaration to determine any
@@ -715,7 +700,7 @@ impl TtParser {
             .bb_mps
             .iter()
             .map(|mp| match &matcher[mp.idx] {
-                MatcherLoc::MetaVarDecl { bind, kind: Some(kind), .. } => {
+                MatcherLoc::MetaVarDecl { bind, kind, .. } => {
                     format!("{kind} ('{bind}')")
                 }
                 _ => unreachable!(),
@@ -745,19 +730,13 @@ impl TtParser {
         // `NamedParseResult`. Otherwise, it's an error.
         let mut ret_val = FxHashMap::default();
         for loc in matcher {
-            if let &MatcherLoc::MetaVarDecl { span, bind, kind, .. } = loc {
-                if kind.is_some() {
-                    match ret_val.entry(MacroRulesNormalizedIdent::new(bind)) {
-                        Vacant(spot) => spot.insert(res.next().unwrap()),
-                        Occupied(..) => {
-                            return Error(span, format!("duplicated bind name: {bind}"));
-                        }
-                    };
-                } else {
-                    // E.g. `$e` instead of `$e:expr`, reported as a hard error if actually used.
-                    // Both this check and the one in `parse_tt_inner` are necessary, surprisingly.
-                    return Error(span, "missing fragment specifier".to_string());
-                }
+            if let &MatcherLoc::MetaVarDecl { span, bind, .. } = loc {
+                match ret_val.entry(MacroRulesNormalizedIdent::new(bind)) {
+                    Vacant(spot) => spot.insert(res.next().unwrap()),
+                    Occupied(..) => {
+                        return Error(span, format!("duplicated bind name: {bind}"));
+                    }
+                };
             }
         }
         Success(ret_val)

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -392,7 +392,7 @@ pub fn compile_declarative_macro(
 
     let lhs_nm = Ident::new(sym::lhs, span);
     let rhs_nm = Ident::new(sym::rhs, span);
-    let tt_spec = Some(NonterminalKind::TT);
+    let tt_spec = NonterminalKind::TT;
     let macro_rules = macro_def.macro_rules;
 
     // Parse the macro_rules! invocation
@@ -407,9 +407,9 @@ pub fn compile_declarative_macro(
             DelimSpan::dummy(),
             mbe::SequenceRepetition {
                 tts: vec![
-                    mbe::TokenTree::MetaVarDecl(span, lhs_nm, tt_spec),
+                    mbe::TokenTree::MetaVarDecl { span, name: lhs_nm, kind: tt_spec },
                     mbe::TokenTree::token(token::FatArrow, span),
-                    mbe::TokenTree::MetaVarDecl(span, rhs_nm, tt_spec),
+                    mbe::TokenTree::MetaVarDecl { span, name: rhs_nm, kind: tt_spec },
                 ],
                 separator: Some(Token::new(
                     if macro_rules { token::Semi } else { token::Comma },
@@ -448,6 +448,7 @@ pub fn compile_declarative_macro(
         match tt_parser.parse_tt(&mut Cow::Owned(parser), &argument_gram, &mut NoopTracker) {
             Success(m) => m,
             Failure(()) => {
+                debug!("failed to parse macro tt");
                 // The fast `NoopTracker` doesn't have any info on failure, so we need to retry it
                 // with another one that gives us the information we need.
                 // For this we need to reclone the macro body as the previous parser consumed it.
@@ -616,7 +617,7 @@ fn is_empty_token_tree(sess: &Session, seq: &mbe::SequenceRepetition) -> bool {
         let mut iter = seq.tts.iter().peekable();
         while let Some(tt) = iter.next() {
             match tt {
-                mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)) => {}
+                mbe::TokenTree::MetaVarDecl { kind: NonterminalKind::Vis, .. } => {}
                 mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }) => {
                     let mut now = t;
                     while let Some(&mbe::TokenTree::Token(
@@ -651,7 +652,7 @@ fn check_redundant_vis_repetition(
 ) {
     let is_zero_or_one: bool = seq.kleene.op == KleeneOp::ZeroOrOne;
     let is_vis = seq.tts.first().map_or(false, |tt| {
-        matches!(tt, mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)))
+        matches!(tt, mbe::TokenTree::MetaVarDecl { kind: NonterminalKind::Vis, .. })
     });
 
     if is_vis && is_zero_or_one {
@@ -678,7 +679,7 @@ fn check_lhs_no_empty_seq(sess: &Session, tts: &[mbe::TokenTree]) -> Result<(),
         match tt {
             TokenTree::Token(..)
             | TokenTree::MetaVar(..)
-            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarDecl { .. }
             | TokenTree::MetaVarExpr(..) => (),
             TokenTree::Delimited(.., del) => check_lhs_no_empty_seq(sess, &del.tts)?,
             TokenTree::Sequence(span, seq) => {
@@ -777,7 +778,7 @@ impl<'tt> FirstSets<'tt> {
                 match tt {
                     TokenTree::Token(..)
                     | TokenTree::MetaVar(..)
-                    | TokenTree::MetaVarDecl(..)
+                    | TokenTree::MetaVarDecl { .. }
                     | TokenTree::MetaVarExpr(..) => {
                         first.replace_with(TtHandle::TtRef(tt));
                     }
@@ -845,7 +846,7 @@ impl<'tt> FirstSets<'tt> {
             match tt {
                 TokenTree::Token(..)
                 | TokenTree::MetaVar(..)
-                | TokenTree::MetaVarDecl(..)
+                | TokenTree::MetaVarDecl { .. }
                 | TokenTree::MetaVarExpr(..) => {
                     first.add_one(TtHandle::TtRef(tt));
                     return first;
@@ -1084,7 +1085,7 @@ fn check_matcher_core<'tt>(
         match token {
             TokenTree::Token(..)
             | TokenTree::MetaVar(..)
-            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarDecl { .. }
             | TokenTree::MetaVarExpr(..) => {
                 if token_can_be_followed_by_any(token) {
                     // don't need to track tokens that work with any,
@@ -1152,7 +1153,7 @@ fn check_matcher_core<'tt>(
         // Now `last` holds the complete set of NT tokens that could
         // end the sequence before SUFFIX. Check that every one works with `suffix`.
         for tt in &last.tokens {
-            if let &TokenTree::MetaVarDecl(span, name, Some(kind)) = tt.get() {
+            if let &TokenTree::MetaVarDecl { span, name, kind } = tt.get() {
                 for next_token in &suffix_first.tokens {
                     let next_token = next_token.get();
 
@@ -1172,11 +1173,11 @@ fn check_matcher_core<'tt>(
                         )
                     {
                         // It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
-                        let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
+                        let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl {
                             span,
                             name,
-                            Some(NonterminalKind::Pat(PatParam { inferred: false })),
-                        ));
+                            kind: NonterminalKind::Pat(PatParam { inferred: false }),
+                        });
                         sess.psess.buffer_lint(
                             RUST_2021_INCOMPATIBLE_OR_PATTERNS,
                             span,
@@ -1212,11 +1213,11 @@ fn check_matcher_core<'tt>(
                                 && sess.psess.edition.at_least_rust_2021()
                                 && next_token.is_token(&token::Or)
                             {
-                                let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
+                                let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl {
                                     span,
                                     name,
-                                    Some(NonterminalKind::Pat(PatParam { inferred: false })),
-                                ));
+                                    kind: NonterminalKind::Pat(PatParam { inferred: false }),
+                                });
                                 err.span_suggestion(
                                     span,
                                     "try a `pat_param` fragment specifier instead",
@@ -1254,7 +1255,7 @@ fn check_matcher_core<'tt>(
 }
 
 fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool {
-    if let mbe::TokenTree::MetaVarDecl(_, _, Some(kind)) = *tok {
+    if let mbe::TokenTree::MetaVarDecl { kind, .. } = *tok {
         frag_can_be_followed_by_any(kind)
     } else {
         // (Non NT's can always be followed by anything in matchers.)
@@ -1367,7 +1368,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                         }
                         _ => IsInFollow::No(TOKENS),
                     },
-                    TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Block)) => IsInFollow::Yes,
+                    TokenTree::MetaVarDecl { kind: NonterminalKind::Block, .. } => IsInFollow::Yes,
                     _ => IsInFollow::No(TOKENS),
                 }
             }
@@ -1400,11 +1401,10 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                             }
                         }
                     },
-                    TokenTree::MetaVarDecl(
-                        _,
-                        _,
-                        Some(NonterminalKind::Ident | NonterminalKind::Ty | NonterminalKind::Path),
-                    ) => IsInFollow::Yes,
+                    TokenTree::MetaVarDecl {
+                        kind: NonterminalKind::Ident | NonterminalKind::Ty | NonterminalKind::Path,
+                        ..
+                    } => IsInFollow::Yes,
                     _ => IsInFollow::No(TOKENS),
                 }
             }
@@ -1416,8 +1416,7 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     match tt {
         mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
         mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
-        mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
-        mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
+        mbe::TokenTree::MetaVarDecl { name, kind, .. } => format!("${name}:{kind}"),
         _ => panic!(
             "{}",
             "unexpected mbe::TokenTree::{Sequence or Delimited} \

compiler/rustc_expand/src/mbe/quoted.rs

@@ -68,8 +68,15 @@ pub(super) fn parse(
         };
 
         // Push a metavariable with no fragment specifier at the given span
-        let mut push_empty_metavar = |span| {
-            result.push(TokenTree::MetaVarDecl(span, ident, None));
+        let mut missing_fragment_specifier = |span| {
+            sess.dcx().emit_err(errors::MissingFragmentSpecifier {
+                span,
+                add_span: span.shrink_to_hi(),
+                valid: VALID_FRAGMENT_NAMES_MSG,
+            });
+
+            // Fall back to a `TokenTree` since that will match anything if we continue expanding.
+            result.push(TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT });
         };
 
         // Not consuming the next token immediately, as it may not be a colon
@@ -84,13 +91,13 @@ pub(super) fn parse(
             // since if it's not a token then it will be an invalid declaration.
             let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else {
                 // Invalid, return a nice source location as `var:`
-                push_empty_metavar(colon_span.with_lo(start_sp.lo()));
+                missing_fragment_specifier(colon_span.with_lo(start_sp.lo()));
                 continue;
             };
 
             let Some((fragment, _)) = token.ident() else {
                 // No identifier for the fragment specifier;
-                push_empty_metavar(token.span);
+                missing_fragment_specifier(token.span);
                 continue;
             };
 
@@ -114,11 +121,11 @@ pub(super) fn parse(
                 });
                 NonterminalKind::Ident
             });
-            result.push(TokenTree::MetaVarDecl(span, ident, Some(kind)));
+            result.push(TokenTree::MetaVarDecl { span, name: ident, kind });
         } else {
             // Whether it's none or some other tree, it doesn't belong to
             // the current meta variable, returning the original span.
-            push_empty_metavar(start_sp);
+            missing_fragment_specifier(start_sp);
         }
     }
     result