Rename TokenStream::concat and remove TokenStream::concat_rc_vec.

nnethercote · nnethercote · commit e80c7ddb05ee · 2018-12-12T20:36:00.000+11:00
`TokenStream::new` is a better name for the former, and the latter is
now just equivalent to `TokenStream::Stream`.
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
@@ -483,7 +483,7 @@ impl MetaItem {
             last_pos = segment.ident.span.hi();
         }
         idents.push(self.node.tokens(self.span));
-        TokenStream::concat(idents)
+        TokenStream::new(idents)
     }
 
     fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
@@ -539,7 +539,7 @@ impl MetaItemKind {
         match *self {
             MetaItemKind::Word => TokenStream::empty(),
             MetaItemKind::NameValue(ref lit) => {
-                TokenStream::concat(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])
+                TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])
             }
             MetaItemKind::List(ref list) => {
                 let mut tokens = Vec::new();
@@ -552,7 +552,7 @@ impl MetaItemKind {
                 TokenTree::Delimited(
                     DelimSpan::from_single(span),
                     token::Paren,
-                    TokenStream::concat(tokens).into(),
+                    TokenStream::new(tokens).into(),
                 ).into()
             }
         }
diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
@@ -247,7 +247,7 @@ pub mod rt {
 
             let delim_span = DelimSpan::from_single(self.span);
             r.push(TokenTree::Delimited(
-                delim_span, token::Bracket, TokenStream::concat(inner).into()
+                delim_span, token::Bracket, TokenStream::new(inner).into()
             ));
             r
         }
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
@@ -103,12 +103,12 @@ pub fn transcribe(cx: &ExtCtxt,
                 }
                 Frame::Delimited { forest, span, .. } => {
                     if result_stack.is_empty() {
-                        return TokenStream::concat(result);
+                        return TokenStream::new(result);
                     }
                     let tree = TokenTree::Delimited(
                         span,
                         forest.delim,
-                        TokenStream::concat(result).into(),
+                        TokenStream::new(result).into(),
                     );
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs
@@ -170,7 +170,7 @@ impl<'a> Parser<'a> {
                     token::CloseDelim(_) | token::Eof => self.unexpected()?,
                     _ => self.parse_token_tree(),
                 };
-                TokenStream::concat(vec![eq.into(), tree.into()])
+                TokenStream::new(vec![eq.into(), tree.into()])
             } else {
                 TokenStream::empty()
             };
diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs
@@ -22,22 +22,22 @@ impl<'a> StringReader<'a> {
             tts.push(self.parse_token_tree()?);
         }
 
-        Ok(TokenStream::concat(tts))
+        Ok(TokenStream::new(tts))
     }
 
     // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
     fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
         let mut tts = vec![];
         loop {
             if let token::CloseDelim(..) = self.token {
-                return TokenStream::concat(tts);
+                return TokenStream::new(tts);
             }
 
             match self.parse_token_tree() {
                 Ok(tree) => tts.push(tree),
                 Err(mut e) => {
                     e.emit();
-                    return TokenStream::concat(tts);
+                    return TokenStream::new(tts);
                 }
             }
         }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
@@ -842,13 +842,13 @@ mod tests {
         with_globals(|| {
             let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
 
-            let expected = TokenStream::concat(vec![
+            let expected = TokenStream::new(vec![
                 TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(),
                 TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
                     token::DelimToken::Paren,
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         TokenTree::Token(sp(6, 7),
                                          token::Ident(Ident::from_str("b"), false)).into(),
                         TokenTree::Token(sp(8, 9), token::Colon).into(),
@@ -859,7 +859,7 @@ mod tests {
                 TokenTree::Delimited(
                     DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
                     token::DelimToken::Brace,
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         TokenTree::Token(sp(17, 18),
                                          token::Ident(Ident::from_str("b"), false)).into(),
                         TokenTree::Token(sp(18, 19), token::Semi).into(),
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
@@ -2928,7 +2928,7 @@ impl<'a> Parser<'a> {
                 _ => result.push(self.parse_token_tree().into()),
             }
         }
-        TokenStream::concat(result)
+        TokenStream::new(result)
     }
 
     /// Parse a prefix-unary-operator expr
@@ -4624,7 +4624,7 @@ impl<'a> Parser<'a> {
                         self.unexpected()?;
                         unreachable!()
                     };
-                    TokenStream::concat(vec![
+                    TokenStream::new(vec![
                         args.into(),
                         TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(),
                         body.into(),
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
@@ -195,7 +195,7 @@ impl TokenStream {
                 new_stream.extend_from_slice(parts.0);
                 new_stream.push(comma);
                 new_stream.extend_from_slice(parts.1);
-                return Some((TokenStream::concat(new_stream), sp));
+                return Some((TokenStream::new(new_stream), sp));
             }
         }
         None
@@ -216,7 +216,7 @@ impl From<Token> for TokenStream {
 
 impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
     fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        TokenStream::concat(iter.into_iter().map(Into::into).collect::<Vec<_>>())
+        TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<_>>())
     }
 }
 
@@ -265,7 +265,7 @@ impl Extend<TokenStream> for TokenStream {
         // Build the resulting token stream. If it contains more than one token,
         // preserve capacity in the vector in anticipation of the caller
         // performing additional calls to extend.
-        *self = TokenStream::concat(builder.0);
+        *self = TokenStream::new(builder.0);
     }
 }
 
@@ -297,18 +297,14 @@ impl TokenStream {
         }
     }
 
-    pub fn concat(mut streams: Vec<TokenStream>) -> TokenStream {
+    pub fn new(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
             1 => streams.pop().unwrap(),
-            _ => TokenStream::concat_rc_vec(Lrc::new(streams)),
+            _ => TokenStream::Stream(Lrc::new(streams)),
         }
     }
 
-    fn concat_rc_vec(streams: Lrc<Vec<TokenStream>>) -> TokenStream {
-        TokenStream::Stream(streams)
-    }
-
     pub fn trees(&self) -> Cursor {
         self.clone().into_trees()
     }
@@ -389,7 +385,7 @@ impl TokenStream {
             });
             i += 1;
         }
-        TokenStream::concat(result)
+        TokenStream::new(result)
     }
 
     pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
@@ -402,7 +398,7 @@ impl TokenStream {
                 _ => unreachable!()
             });
         }
-        TokenStream::concat(result)
+        TokenStream::new(result)
     }
 
     fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
@@ -461,7 +457,7 @@ impl TokenStreamBuilder {
     }
 
     pub fn build(self) -> TokenStream {
-        TokenStream::concat(self.0)
+        TokenStream::new(self.0)
     }
 
     fn push_all_but_last_tree(&mut self, stream: &TokenStream) {
@@ -470,7 +466,7 @@ impl TokenStreamBuilder {
             match len {
                 1 => {}
                 2 => self.0.push(streams[0].clone().into()),
-                _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())),
+                _ => self.0.push(TokenStream::new(streams[0 .. len - 1].to_vec())),
             }
             self.push_all_but_last_tree(&streams[len - 1])
         }
@@ -482,7 +478,7 @@ impl TokenStreamBuilder {
             match len {
                 1 => {}
                 2 => self.0.push(streams[1].clone().into()),
-                _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())),
+                _ => self.0.push(TokenStream::new(streams[1 .. len].to_vec())),
             }
             self.push_all_but_first_tree(&streams[0])
         }
@@ -577,7 +573,7 @@ impl Cursor {
             _ if stream.is_empty() => return,
             CursorKind::Empty => *self = stream.trees(),
             CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => {
-                *self = TokenStream::concat(vec![self.original_stream(), stream]).trees();
+                *self = TokenStream::new(vec![self.original_stream(), stream]).trees();
                 if consumed {
                     self.next();
                 }
@@ -593,10 +589,10 @@ impl Cursor {
             CursorKind::Empty => TokenStream::empty(),
             CursorKind::Tree(ref tree, _) => tree.clone().into(),
             CursorKind::JointTree(ref tree, _) => tree.clone().joint(),
-            CursorKind::Stream(ref cursor) => TokenStream::concat_rc_vec({
+            CursorKind::Stream(ref cursor) => TokenStream::Stream(
                 cursor.stack.get(0).cloned().map(|(stream, _)| stream)
                     .unwrap_or_else(|| cursor.stream.clone())
-            }),
+            ),
         }
     }
 
@@ -664,7 +660,7 @@ impl From<TokenStream> for ThinTokenStream {
 
 impl From<ThinTokenStream> for TokenStream {
     fn from(stream: ThinTokenStream) -> TokenStream {
-        stream.0.map(TokenStream::concat_rc_vec).unwrap_or_else(TokenStream::empty)
+        stream.0.map(TokenStream::Stream).unwrap_or_else(TokenStream::empty)
     }
 }
 
@@ -763,7 +759,7 @@ mod tests {
             let test_res = string_to_ts("foo::bar::baz");
             let test_fst = string_to_ts("foo::bar");
             let test_snd = string_to_ts("::baz");
-            let eq_res = TokenStream::concat(vec![test_fst, test_snd]);
+            let eq_res = TokenStream::new(vec![test_fst, test_snd]);
             assert_eq!(test_res.trees().count(), 5);
             assert_eq!(eq_res.trees().count(), 5);
             assert_eq!(test_res.eq_unspanned(&eq_res), true);

Original file line number	Diff line number	Diff line change
`@@ -483,7 +483,7 @@ impl MetaItem {`
`483`	`483`	`last_pos = segment.ident.span.hi();`
`484`	`484`	`}`
`485`	`485`	`idents.push(self.node.tokens(self.span));`
`486`		`- TokenStream::concat(idents)`
	`486`	`+ TokenStream::new(idents)`
`487`	`487`	`}`
`488`	`488`
`489`	`489`	`fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>`
`@@ -539,7 +539,7 @@ impl MetaItemKind {`
`539`	`539`	`match *self {`
`540`	`540`	`MetaItemKind::Word => TokenStream::empty(),`
`541`	`541`	`MetaItemKind::NameValue(ref lit) => {`
`542`		`- TokenStream::concat(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])`
	`542`	`+ TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()])`
`543`	`543`	`}`
`544`	`544`	`MetaItemKind::List(ref list) => {`
`545`	`545`	`let mut tokens = Vec::new();`
`@@ -552,7 +552,7 @@ impl MetaItemKind {`
`552`	`552`	`TokenTree::Delimited(`
`553`	`553`	`DelimSpan::from_single(span),`
`554`	`554`	`token::Paren,`
`555`		`- TokenStream::concat(tokens).into(),`
	`555`	`+ TokenStream::new(tokens).into(),`
`556`	`556`	`).into()`
`557`	`557`	`}`
`558`	`558`	`}`
Original file line number	Diff line number	Diff line change
`@@ -247,7 +247,7 @@ pub mod rt {`
`247`	`247`
`248`	`248`	`let delim_span = DelimSpan::from_single(self.span);`
`249`	`249`	`r.push(TokenTree::Delimited(`
`250`		`- delim_span, token::Bracket, TokenStream::concat(inner).into()`
	`250`	`+ delim_span, token::Bracket, TokenStream::new(inner).into()`
`251`	`251`	`));`
`252`	`252`	`r`
`253`	`253`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,22 +22,22 @@ impl<'a> StringReader<'a> {`
`22`	`22`	`tts.push(self.parse_token_tree()?);`
`23`	`23`	`}`
`24`	`24`
`25`		`- Ok(TokenStream::concat(tts))`
	`25`	`+ Ok(TokenStream::new(tts))`
`26`	`26`	`}`
`27`	`27`
`28`	`28`	// Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
`29`	`29`	`fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {`
`30`	`30`	`let mut tts = vec![];`
`31`	`31`	`loop {`
`32`	`32`	`if let token::CloseDelim(..) = self.token {`
`33`		`- return TokenStream::concat(tts);`
	`33`	`+ return TokenStream::new(tts);`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`match self.parse_token_tree() {`
`37`	`37`	`Ok(tree) => tts.push(tree),`
`38`	`38`	`Err(mut e) => {`
`39`	`39`	`e.emit();`
`40`		`- return TokenStream::concat(tts);`
	`40`	`+ return TokenStream::new(tts);`
`41`	`41`	`}`
`42`	`42`	`}`
`43`	`43`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2928,7 +2928,7 @@ impl<'a> Parser<'a> {`
`2928`	`2928`	`_ => result.push(self.parse_token_tree().into()),`
`2929`	`2929`	`}`
`2930`	`2930`	`}`
`2931`		`- TokenStream::concat(result)`
	`2931`	`+ TokenStream::new(result)`
`2932`	`2932`	`}`
`2933`	`2933`
`2934`	`2934`	`/// Parse a prefix-unary-operator expr`
`@@ -4624,7 +4624,7 @@ impl<'a> Parser<'a> {`
`4624`	`4624`	`self.unexpected()?;`
`4625`	`4625`	`unreachable!()`
`4626`	`4626`	`};`
`4627`		`- TokenStream::concat(vec![`
	`4627`	`+ TokenStream::new(vec![`
`4628`	`4628`	`args.into(),`
`4629`	`4629`	`TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(),`
`4630`	`4630`	`body.into(),`
Original file line number	Diff line number	Diff line change
`@@ -195,7 +195,7 @@ impl TokenStream {`
`195`	`195`	`new_stream.extend_from_slice(parts.0);`
`196`	`196`	`new_stream.push(comma);`
`197`	`197`	`new_stream.extend_from_slice(parts.1);`
`198`		`- return Some((TokenStream::concat(new_stream), sp));`
	`198`	`+ return Some((TokenStream::new(new_stream), sp));`
`199`	`199`	`}`
`200`	`200`	`}`
`201`	`201`	`None`
`@@ -216,7 +216,7 @@ impl From<Token> for TokenStream {`
`216`	`216`
`217`	`217`	`impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {`
`218`	`218`	`fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {`
`219`		`- TokenStream::concat(iter.into_iter().map(Into::into).collect::<Vec<_>>())`
	`219`	`+ TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<_>>())`
`220`	`220`	`}`
`221`	`221`	`}`
`222`	`222`
`@@ -265,7 +265,7 @@ impl Extend<TokenStream> for TokenStream {`
`265`	`265`	`// Build the resulting token stream. If it contains more than one token,`
`266`	`266`	`// preserve capacity in the vector in anticipation of the caller`
`267`	`267`	`// performing additional calls to extend.`
`268`		`- *self = TokenStream::concat(builder.0);`
	`268`	`+ *self = TokenStream::new(builder.0);`
`269`	`269`	`}`
`270`	`270`	`}`
`271`	`271`
`@@ -297,18 +297,14 @@ impl TokenStream {`
`297`	`297`	`}`
`298`	`298`	`}`
`299`	`299`
`300`		`- pub fn concat(mut streams: Vec<TokenStream>) -> TokenStream {`
	`300`	`+ pub fn new(mut streams: Vec<TokenStream>) -> TokenStream {`
`301`	`301`	`match streams.len() {`
`302`	`302`	`0 => TokenStream::empty(),`
`303`	`303`	`1 => streams.pop().unwrap(),`
`304`		`- _ => TokenStream::concat_rc_vec(Lrc::new(streams)),`
	`304`	`+ _ => TokenStream::Stream(Lrc::new(streams)),`
`305`	`305`	`}`
`306`	`306`	`}`
`307`	`307`
`308`		`- fn concat_rc_vec(streams: Lrc<Vec<TokenStream>>) -> TokenStream {`
`309`		`- TokenStream::Stream(streams)`
`310`		`- }`
`311`		`-`
`312`	`308`	`pub fn trees(&self) -> Cursor {`
`313`	`309`	`self.clone().into_trees()`
`314`	`310`	`}`
`@@ -389,7 +385,7 @@ impl TokenStream {`
`389`	`385`	`});`
`390`	`386`	`i += 1;`
`391`	`387`	`}`
`392`		`- TokenStream::concat(result)`
	`388`	`+ TokenStream::new(result)`
`393`	`389`	`}`
`394`	`390`
`395`	`391`	`pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {`
`@@ -402,7 +398,7 @@ impl TokenStream {`
`402`	`398`	`_ => unreachable!()`
`403`	`399`	`});`
`404`	`400`	`}`
`405`		`- TokenStream::concat(result)`
	`401`	`+ TokenStream::new(result)`
`406`	`402`	`}`
`407`	`403`
`408`	`404`	`fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {`
`@@ -461,7 +457,7 @@ impl TokenStreamBuilder {`
`461`	`457`	`}`
`462`	`458`
`463`	`459`	`pub fn build(self) -> TokenStream {`
`464`		`- TokenStream::concat(self.0)`
	`460`	`+ TokenStream::new(self.0)`
`465`	`461`	`}`
`466`	`462`
`467`	`463`	`fn push_all_but_last_tree(&mut self, stream: &TokenStream) {`
`@@ -470,7 +466,7 @@ impl TokenStreamBuilder {`
`470`	`466`	`match len {`
`471`	`467`	`1 => {}`
`472`	`468`	`2 => self.0.push(streams[0].clone().into()),`
`473`		`- _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())),`
	`469`	`+ _ => self.0.push(TokenStream::new(streams[0 .. len - 1].to_vec())),`
`474`	`470`	`}`
`475`	`471`	`self.push_all_but_last_tree(&streams[len - 1])`
`476`	`472`	`}`
`@@ -482,7 +478,7 @@ impl TokenStreamBuilder {`
`482`	`478`	`match len {`
`483`	`479`	`1 => {}`
`484`	`480`	`2 => self.0.push(streams[1].clone().into()),`
`485`		`- _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())),`
	`481`	`+ _ => self.0.push(TokenStream::new(streams[1 .. len].to_vec())),`
`486`	`482`	`}`
`487`	`483`	`self.push_all_but_first_tree(&streams[0])`
`488`	`484`	`}`
`@@ -577,7 +573,7 @@ impl Cursor {`
`577`	`573`	`_ if stream.is_empty() => return,`
`578`	`574`	`CursorKind::Empty => *self = stream.trees(),`
`579`	`575`	`CursorKind::Tree(_, consumed) \| CursorKind::JointTree(_, consumed) => {`
`580`		`- *self = TokenStream::concat(vec![self.original_stream(), stream]).trees();`
	`576`	`+ *self = TokenStream::new(vec![self.original_stream(), stream]).trees();`
`581`	`577`	`if consumed {`
`582`	`578`	`self.next();`
`583`	`579`	`}`
`@@ -593,10 +589,10 @@ impl Cursor {`
`593`	`589`	`CursorKind::Empty => TokenStream::empty(),`
`594`	`590`	`CursorKind::Tree(ref tree, _) => tree.clone().into(),`
`595`	`591`	`CursorKind::JointTree(ref tree, _) => tree.clone().joint(),`
`596`		`- CursorKind::Stream(ref cursor) => TokenStream::concat_rc_vec({`
	`592`	`+ CursorKind::Stream(ref cursor) => TokenStream::Stream(`
`597`	`593`	`cursor.stack.get(0).cloned().map(\|(stream, _)\| stream)`
`598`	`594`	`.unwrap_or_else(\|\| cursor.stream.clone())`
`599`		`- }),`
	`595`	`+ ),`
`600`	`596`	`}`
`601`	`597`	`}`
`602`	`598`
`@@ -664,7 +660,7 @@ impl From<TokenStream> for ThinTokenStream {`
`664`	`660`
`665`	`661`	`impl From<ThinTokenStream> for TokenStream {`
`666`	`662`	`fn from(stream: ThinTokenStream) -> TokenStream {`
`667`		`- stream.0.map(TokenStream::concat_rc_vec).unwrap_or_else(TokenStream::empty)`
	`663`	`+ stream.0.map(TokenStream::Stream).unwrap_or_else(TokenStream::empty)`
`668`	`664`	`}`
`669`	`665`	`}`
`670`	`666`
`@@ -763,7 +759,7 @@ mod tests {`
`763`	`759`	`let test_res = string_to_ts("foo::bar::baz");`
`764`	`760`	`let test_fst = string_to_ts("foo::bar");`
`765`	`761`	`let test_snd = string_to_ts("::baz");`
`766`		`- let eq_res = TokenStream::concat(vec![test_fst, test_snd]);`
	`762`	`+ let eq_res = TokenStream::new(vec![test_fst, test_snd]);`
`767`	`763`	`assert_eq!(test_res.trees().count(), 5);`
`768`	`764`	`assert_eq!(eq_res.trees().count(), 5);`
`769`	`765`	`assert_eq!(test_res.eq_unspanned(&eq_res), true);`