add unknown token

Author: matklad

src/librustc/ich/impls_syntax.rs

@@ -363,7 +363,8 @@ impl<'a> HashStable<StableHashingContext<'a>> for token::TokenKind {
             }
 
             token::DocComment(val) |
-            token::Shebang(val) => val.hash_stable(hcx, hasher),
+            token::Shebang(val) |
+            token::Unknown(val) => val.hash_stable(hcx, hasher),
         }
     }
 }

src/librustdoc/html/highlight.rs

@@ -237,7 +237,7 @@ impl<'a> Classifier<'a> {
                 return Ok(());
             },
 
-            token::Whitespace => Class::None,
+            token::Whitespace | token::Unknown(..) => Class::None,
             token::Comment => Class::Comment,
             token::DocComment(..) => Class::DocComment,
 

src/libsyntax/ext/proc_macro_server.rs

@@ -184,7 +184,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
             }
 
             OpenDelim(..) | CloseDelim(..) => unreachable!(),
-            Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
+            Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(),
         }
     }
 }

src/libsyntax/parse/lexer/mod.rs

@@ -354,7 +354,7 @@ impl<'a> StringReader<'a> {
                 // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it,
                 // as there will be less overall work to do this way.
                 let token = unicode_chars::check_for_substitution(self, start, c, &mut err)
-                    .unwrap_or(token::Whitespace);
+                    .unwrap_or_else(|| token::Unknown(self.symbol_from(start)));
                 err.emit();
                 token
             }

src/libsyntax/parse/lexer/tokentrees.rs

@@ -217,7 +217,7 @@ impl<'a> TokenTreesReader<'a> {
         loop {
             let token = self.string_reader.next_token();
             match token.kind {
-                token::Whitespace | token::Comment | token::Shebang(_) => {
+                token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => {
                     self.joint_to_prev = NonJoint;
                 }
                 _ => {

src/libsyntax/parse/token.rs

@@ -255,6 +255,8 @@ pub enum TokenKind {
     /// A comment.
     Comment,
     Shebang(ast::Name),
+    /// A completely invalid token which should be skipped.
+    Unknown(ast::Name),
 
     Eof,
 }
@@ -603,7 +605,7 @@ impl Token {
             DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar |
             Question | OpenDelim(..) | CloseDelim(..) |
             Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
-            Whitespace | Comment | Shebang(..) | Eof => return None,
+            Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None,
         };
 
         Some(Token::new(kind, self.span.to(joint.span)))

src/libsyntax/print/pprust.rs

@@ -288,6 +288,7 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>)
         token::Whitespace           => " ".to_string(),
         token::Comment              => "/* */".to_string(),
         token::Shebang(s)           => format!("/* shebang: {}*/", s),
+        token::Unknown(s)           => s.to_string(),
 
         token::Interpolated(ref nt) => nonterminal_to_string(nt),
     }