Ignore whitespace tokens when re-computing spans in save_analysis

Author: Nick Cameron

src/librustc/middle/save/span_utils.rs

@@ -92,7 +92,7 @@ impl<'a> SpanUtils<'a> {
         let mut toks = self.retokenise_span(span);
         let mut bracket_count = 0u;
         loop {
-            let ts = toks.next_token();
+            let ts = toks.real_token();
             if ts.tok == token::Eof {
                 return self.make_sub_span(span, result)
             }
@@ -115,7 +115,7 @@ impl<'a> SpanUtils<'a> {
         let mut toks = self.retokenise_span(span);
         let mut bracket_count = 0u;
         loop {
-            let ts = toks.next_token();
+            let ts = toks.real_token();
             if ts.tok == token::Eof {
                 return None;
             }
@@ -137,13 +137,13 @@ impl<'a> SpanUtils<'a> {
     // any brackets, or the last span.
     pub fn sub_span_for_meth_name(&self, span: Span) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
-        let mut prev = toks.next_token();
+        let mut prev = toks.real_token();
         let mut result = None;
         let mut bracket_count = 0u;
         let mut last_span = None;
         while prev.tok != token::Eof {
             last_span = None;
-            let mut next = toks.next_token();
+            let mut next = toks.real_token();
 
             if (next.tok == token::OpenDelim(token::Paren) ||
                 next.tok == token::Lt) &&
@@ -156,7 +156,7 @@ impl<'a> SpanUtils<'a> {
                 next.tok == token::ModSep {
                 let old = prev;
                 prev = next;
-                next = toks.next_token();
+                next = toks.real_token();
                 if next.tok == token::Lt &&
                    old.tok.is_ident() {
                     result = Some(old.sp);
@@ -185,11 +185,11 @@ impl<'a> SpanUtils<'a> {
     // brackets, or the last span.
     pub fn sub_span_for_type_name(&self, span: Span) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
-        let mut prev = toks.next_token();
+        let mut prev = toks.real_token();
         let mut result = None;
         let mut bracket_count = 0u;
         loop {
-            let next = toks.next_token();
+            let next = toks.real_token();
 
             if (next.tok == token::Lt ||
                 next.tok == token::Colon) &&
@@ -234,7 +234,7 @@ impl<'a> SpanUtils<'a> {
         // We keep track of how many brackets we're nested in
         let mut bracket_count = 0i;
         loop {
-            let ts = toks.next_token();
+            let ts = toks.real_token();
             if ts.tok == token::Eof {
                 if bracket_count != 0 {
                     let loc = self.sess.codemap().lookup_char_pos(span.lo);
@@ -263,12 +263,12 @@ impl<'a> SpanUtils<'a> {
 
     pub fn sub_span_before_token(&self, span: Span, tok: Token) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
-        let mut prev = toks.next_token();
+        let mut prev = toks.real_token();
         loop {
             if prev.tok == token::Eof {
                 return None;
             }
-            let next = toks.next_token();
+            let next = toks.real_token();
             if next.tok == tok {
                 return self.make_sub_span(span, Some(prev.sp));
             }
@@ -281,15 +281,16 @@ impl<'a> SpanUtils<'a> {
                               keyword: keywords::Keyword) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
         loop {
-            let ts = toks.next_token();
+            let ts = toks.real_token();
             if ts.tok == token::Eof {
                 return None;
             }
             if ts.tok.is_keyword(keyword) {
-                let ts = toks.next_token();
+                let ts = toks.real_token();
                 if ts.tok == token::Eof {
                     return None
                 } else {
+                    println!("found keyword: {} at {}", ts, ts.sp);
                     return self.make_sub_span(span, Some(ts.sp));
                 }
             }

src/libsyntax/parse/lexer/mod.rs

@@ -35,6 +35,19 @@ pub trait Reader {
     /// Report a non-fatal error with the current span.
     fn err(&self, &str);
     fn peek(&self) -> TokenAndSpan;
+    /// Get a token the parser cares about.
+    fn real_token(&mut self) -> TokenAndSpan {
+        let mut t = self.next_token();
+        loop {
+            match t.tok {
+                token::Whitespace | token::Comment | token::Shebang(_) => {
+                    t = self.next_token();
+                },
+                _ => break
+            }
+        }
+        t
+    }
 }
 
 #[deriving(Clone, PartialEq, Eq, Show)]

src/libsyntax/parse/parser.rs

@@ -338,27 +338,13 @@ fn is_plain_ident_or_underscore(t: &token::Token) -> bool {
     t.is_plain_ident() || *t == token::Underscore
 }
 
-/// Get a token the parser cares about
-fn real_token(rdr: &mut Reader) -> TokenAndSpan {
-    let mut t = rdr.next_token();
-    loop {
-        match t.tok {
-            token::Whitespace | token::Comment | token::Shebang(_) => {
-                t = rdr.next_token();
-            },
-            _ => break
-        }
-    }
-    t
-}
-
 impl<'a> Parser<'a> {
     pub fn new(sess: &'a ParseSess,
                cfg: ast::CrateConfig,
                mut rdr: Box<Reader+'a>)
                -> Parser<'a>
     {
-        let tok0 = real_token(&mut *rdr);
+        let tok0 = rdr.real_token();
         let span = tok0.sp;
         let placeholder = TokenAndSpan {
             tok: token::Underscore,
@@ -898,7 +884,7 @@ impl<'a> Parser<'a> {
             None
         };
         let next = if self.buffer_start == self.buffer_end {
-            real_token(&mut *self.reader)
+            self.reader.real_token()
         } else {
             // Avoid token copies with `replace`.
             let buffer_start = self.buffer_start as uint;
@@ -942,7 +928,7 @@ impl<'a> Parser<'a> {
                       -> R {
         let dist = distance as int;
         while self.buffer_length() < dist {
-            self.buffer[self.buffer_end as uint] = real_token(&mut *self.reader);
+            self.buffer[self.buffer_end as uint] = self.reader.real_token();
             self.buffer_end = (self.buffer_end + 1) & 3;
         }
         f(&self.buffer[((self.buffer_start + dist - 1) & 3) as uint].tok)