Auto merge of #60932 - Centril:macro-at-most-once-2015, r=mark-i-m

Support ? Kleene macro operator in 2015

Closes #56668.

See that issue for rationale and discussion.

Crater will be needed (done in #60932 (comment), zero regressions) and then, if all goes well, FCP (in #60932 (comment)).

Author: bors

src/libsyntax/ext/tt/quoted.rs

@@ -1,5 +1,4 @@
 use crate::ast::NodeId;
-use crate::early_buffered_lints::BufferedEarlyLintId;
 use crate::ext::tt::macro_parser;
 use crate::feature_gate::Features;
 use crate::parse::token::{self, Token, TokenKind};
@@ -250,19 +249,16 @@ pub fn parse(
 /// - `sess`: the parsing session. Any errors will be emitted to this session.
 /// - `features`, `attrs`: language feature flags and attributes so that we know whether to use
 ///   unstable features or not.
-fn parse_tree<I>(
+fn parse_tree(
     tree: tokenstream::TokenTree,
-    trees: &mut Peekable<I>,
+    trees: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>,
     expect_matchers: bool,
     sess: &ParseSess,
     features: &Features,
     attrs: &[ast::Attribute],
     edition: Edition,
     macro_node_id: NodeId,
-) -> TokenTree
-where
-    I: Iterator<Item = tokenstream::TokenTree>,
-{
+) -> TokenTree {
     // Depending on what `tree` is, we could be parsing different parts of a macro
     match tree {
         // `tree` is a `$` token. Look at the next token in `trees`
@@ -287,16 +283,7 @@ where
                     macro_node_id,
                 );
                 // Get the Kleene operator and optional separator
-                let (separator, op) =
-                    parse_sep_and_kleene_op(
-                        trees,
-                        span.entire(),
-                        sess,
-                        features,
-                        attrs,
-                        edition,
-                        macro_node_id,
-                    );
+                let (separator, op) = parse_sep_and_kleene_op(trees, span.entire(), sess);
                 // Count the number of captured "names" (i.e., named metavars)
                 let name_captures = macro_parser::count_names(&sequence);
                 TokenTree::Sequence(
@@ -375,10 +362,10 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
 /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
 /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
 /// - Err(span) if the next token tree is not a token
-fn parse_kleene_op<I>(input: &mut I, span: Span) -> Result<Result<(KleeneOp, Span), Token>, Span>
-where
-    I: Iterator<Item = tokenstream::TokenTree>,
-{
+fn parse_kleene_op(
+    input: &mut impl Iterator<Item = tokenstream::TokenTree>,
+    span: Span,
+) -> Result<Result<(KleeneOp, Span), Token>, Span> {
     match input.next() {
         Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) {
             Some(op) => Ok(Ok((op, token.span))),
@@ -403,178 +390,20 @@ where
 /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
 /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
 /// error with the appropriate span is emitted to `sess` and a dummy value is returned.
-///
-/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator.
-/// In the 2018 edition however, `?` is a Kleene operator, and not a separator.
-fn parse_sep_and_kleene_op<I>(
-    input: &mut Peekable<I>,
-    span: Span,
-    sess: &ParseSess,
-    features: &Features,
-    attrs: &[ast::Attribute],
-    edition: Edition,
-    macro_node_id: NodeId,
-) -> (Option<Token>, KleeneOp)
-where
-    I: Iterator<Item = tokenstream::TokenTree>,
-{
-    match edition {
-        Edition::Edition2015 => parse_sep_and_kleene_op_2015(
-            input,
-            span,
-            sess,
-            features,
-            attrs,
-            macro_node_id,
-        ),
-        Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs),
-    }
-}
-
-// `?` is a separator (with a migration warning) and never a KleeneOp.
-fn parse_sep_and_kleene_op_2015<I>(
-    input: &mut Peekable<I>,
-    span: Span,
-    sess: &ParseSess,
-    _features: &Features,
-    _attrs: &[ast::Attribute],
-    macro_node_id: NodeId,
-) -> (Option<Token>, KleeneOp)
-where
-    I: Iterator<Item = tokenstream::TokenTree>,
-{
-    // We basically look at two token trees here, denoted as #1 and #2 below
-    let span = match parse_kleene_op(input, span) {
-        // #1 is a `+` or `*` KleeneOp
-        //
-        // `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so
-        // we need to look ahead one more token to be sure.
-        Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op),
-
-        // #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator
-        // or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1
-        // token to find out which.
-        Ok(Ok((op, op1_span))) => {
-            assert_eq!(op, KleeneOp::ZeroOrOne);
-
-            // Lookahead at #2. If it is a KleenOp, then #1 is a separator.
-            let is_1_sep = if let Some(tokenstream::TokenTree::Token(tok2)) = input.peek() {
-                kleene_op(tok2).is_some()
-            } else {
-                false
-            };
-
-            if is_1_sep {
-                // #1 is a separator and #2 should be a KleepeOp.
-                // (N.B. We need to advance the input iterator.)
-                match parse_kleene_op(input, span) {
-                    // #2 is `?`, which is not allowed as a Kleene op in 2015 edition,
-                    // but is allowed in the 2018 edition.
-                    Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
-                        sess.span_diagnostic
-                            .struct_span_err(op2_span, "expected `*` or `+`")
-                            .note("`?` is not a macro repetition operator in the 2015 edition, \
-                                 but is accepted in the 2018 edition")
-                            .emit();
-
-                        // Return a dummy
-                        return (None, KleeneOp::ZeroOrMore);
-                    }
-
-                    // #2 is a Kleene op, which is the only valid option
-                    Ok(Ok((op, _))) => {
-                        // Warn that `?` as a separator will be deprecated
-                        sess.buffer_lint(
-                            BufferedEarlyLintId::QuestionMarkMacroSep,
-                            op1_span,
-                            macro_node_id,
-                            "using `?` as a separator is deprecated and will be \
-                             a hard error in an upcoming edition",
-                        );
-
-                        return (Some(Token::new(token::Question, op1_span)), op);
-                    }
-
-                    // #2 is a random token (this is an error) :(
-                    Ok(Err(_)) => op1_span,
-
-                    // #2 is not even a token at all :(
-                    Err(_) => op1_span,
-                }
-            } else {
-                // `?` is not allowed as a Kleene op in 2015,
-                // but is allowed in the 2018 edition
-                sess.span_diagnostic
-                    .struct_span_err(op1_span, "expected `*` or `+`")
-                    .note("`?` is not a macro repetition operator in the 2015 edition, \
-                         but is accepted in the 2018 edition")
-                    .emit();
-
-                // Return a dummy
-                return (None, KleeneOp::ZeroOrMore);
-            }
-        }
-
-        // #1 is a separator followed by #2, a KleeneOp
-        Ok(Err(token)) => match parse_kleene_op(input, token.span) {
-            // #2 is a `?`, which is not allowed as a Kleene op in 2015 edition,
-            // but is allowed in the 2018 edition
-            Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
-                sess.span_diagnostic
-                    .struct_span_err(op2_span, "expected `*` or `+`")
-                    .note("`?` is not a macro repetition operator in the 2015 edition, \
-                        but is accepted in the 2018 edition")
-                    .emit();
-
-                // Return a dummy
-                return (None, KleeneOp::ZeroOrMore);
-            }
-
-            // #2 is a KleeneOp :D
-            Ok(Ok((op, _))) => return (Some(token), op),
-
-            // #2 is a random token :(
-            Ok(Err(token)) => token.span,
-
-            // #2 is not a token at all :(
-            Err(span) => span,
-        },
-
-        // #1 is not a token
-        Err(span) => span,
-    };
-
-    sess.span_diagnostic.span_err(span, "expected `*` or `+`");
-
-    // Return a dummy
-    (None, KleeneOp::ZeroOrMore)
-}
-
-// `?` is a Kleene op, not a separator
-fn parse_sep_and_kleene_op_2018<I>(
-    input: &mut Peekable<I>,
+fn parse_sep_and_kleene_op(
+    input: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>,
     span: Span,
     sess: &ParseSess,
-    _features: &Features,
-    _attrs: &[ast::Attribute],
-) -> (Option<Token>, KleeneOp)
-where
-    I: Iterator<Item = tokenstream::TokenTree>,
-{
+) -> (Option<Token>, KleeneOp) {
     // We basically look at two token trees here, denoted as #1 and #2 below
     let span = match parse_kleene_op(input, span) {
-        // #1 is a `?` (needs feature gate)
-        Ok(Ok((op, _op1_span))) if op == KleeneOp::ZeroOrOne => {
-            return (None, op);
-        }
-
-        // #1 is a `+` or `*` KleeneOp
+        // #1 is a `?`, `+`, or `*` KleeneOp
         Ok(Ok((op, _))) => return (None, op),
 
         // #1 is a separator followed by #2, a KleeneOp
         Ok(Err(token)) => match parse_kleene_op(input, token.span) {
             // #2 is the `?` Kleene op, which does not take a separator (error)
-            Ok(Ok((op, _op2_span))) if op == KleeneOp::ZeroOrOne => {
+            Ok(Ok((KleeneOp::ZeroOrOne, _))) => {
                 // Error!
                 sess.span_diagnostic.span_err(
                     token.span,
@@ -588,11 +417,8 @@ where
             // #2 is a KleeneOp :D
             Ok(Ok((op, _))) => return (Some(token), op),
 
-            // #2 is a random token :(
-            Ok(Err(token)) => token.span,
-
-            // #2 is not a token at all :(
-            Err(span) => span,
+            // #2 is a random token or not a token at all :(
+            Ok(Err(Token { span, .. })) | Err(span) => span,
         },
 
         // #1 is not a token

src/test/run-pass/macros/macro-at-most-once-rep-2015.rs

@@ -0,0 +1,50 @@
+// run-pass
+
+#![allow(unused_mut)]
+
+// Check that when `?` is followed by what looks like a Kleene operator (?, +, and *)
+// then that `?` is not interpreted as a separator. In other words, `$(pat)?+` matches `pat +`
+// or `+` but does not match `pat` or `pat ? pat`.
+
+// edition:2015
+
+macro_rules! foo {
+    // Check for `?`.
+    ($($a:ident)? ? $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `+`.
+    ($($a:ident)? + $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `*`.
+    ($($a:ident)? * $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `;`, not a kleene operator.
+    ($($a:ident)? ; $num:expr) => {
+        let mut x = 0;
+
+        $(
+            x += $a;
+        )?
+
+        assert_eq!(x, $num);
+    };
+}
+
+pub fn main() {
+    let a = 1;
+
+    // Accept 0 repetitions.
+    foo!( ; 0);
+    foo!( + 0);
+    foo!( * 0);
+    foo!( ? 0);
+
+    // Accept 1 repetition.
+    foo!(a ; 1);
+    foo!(a + 1);
+    foo!(a * 1);
+    foo!(a ? 1);
+}

src/test/run-pass/macros/macro-at-most-once-rep-2018.rs

@@ -0,0 +1,50 @@
+// run-pass
+
+#![allow(unused_mut)]
+
+// Check that when `?` is followed by what looks like a Kleene operator (?, +, and *)
+// then that `?` is not interpreted as a separator. In other words, `$(pat)?+` matches `pat +`
+// or `+` but does not match `pat` or `pat ? pat`.
+
+// edition:2018
+
+macro_rules! foo {
+    // Check for `?`.
+    ($($a:ident)? ? $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `+`.
+    ($($a:ident)? + $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `*`.
+    ($($a:ident)? * $num:expr) => {
+        foo!($($a)? ; $num);
+    };
+    // Check for `;`, not a kleene operator.
+    ($($a:ident)? ; $num:expr) => {
+        let mut x = 0;
+
+        $(
+            x += $a;
+        )?
+
+        assert_eq!(x, $num);
+    };
+}
+
+pub fn main() {
+    let a = 1;
+
+    // Accept 0 repetitions.
+    foo!( ; 0);
+    foo!( + 0);
+    foo!( * 0);
+    foo!( ? 0);
+
+    // Accept 1 repetition.
+    foo!(a ; 1);
+    foo!(a + 1);
+    foo!(a * 1);
+    foo!(a ? 1);
+}