Auto merge of #177 - servo:line-counting, r=emilio

Count line-numbers during tokenization

Before this PR, the `source_location` and `current_source_location` iterated (again) through input bytes separately from tokenization in order to count newline characters and determine the line number of some piece of a stylesheet.

This PR makes this counting happen during tokenization instead, where we already have a pass looking at every bytes.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/rust-cssparser/177)
<!-- Reviewable:end -->

Author: bors-servo

Cargo.toml

@@ -1,7 +1,6 @@
 [package]
-
 name = "cssparser"
-version = "0.18.2"
+version = "0.19.0"
 authors = [ "Simon Sapin <simon.sapin@exyr.org>" ]
 
 description = "Rust implementation of CSS Syntax Level 3"

src/lib.rs

@@ -80,16 +80,16 @@ fn parse_border_spacing(_context: &ParserContext, input: &mut Parser)
 
 pub use cssparser_macros::*;
 
-pub use tokenizer::{Token, SourceLocation};
+pub use tokenizer::{Token, SourcePosition, SourceLocation};
 pub use rules_and_declarations::{parse_important};
 pub use rules_and_declarations::{DeclarationParser, DeclarationListParser, parse_one_declaration};
-pub use rules_and_declarations::{RuleListParser, parse_one_rule};
+pub use rules_and_declarations::{RuleListParser, parse_one_rule, PreciseParseError};
 pub use rules_and_declarations::{AtRuleType, QualifiedRuleParser, AtRuleParser};
 pub use from_bytes::{stylesheet_encoding, EncodingSupport};
 pub use color::{RGBA, Color, parse_color_keyword};
 pub use nth::parse_nth;
 pub use serializer::{ToCss, CssStringWriter, serialize_identifier, serialize_string, TokenSerializationType};
-pub use parser::{Parser, Delimiter, Delimiters, SourcePosition, ParseError, BasicParseError, ParserInput};
+pub use parser::{Parser, Delimiter, Delimiters, ParserState, ParseError, BasicParseError, ParserInput};
 pub use unicode_range::UnicodeRange;
 pub use cow_rc_str::CowRcStr;
 

src/nth.rs

@@ -69,13 +69,13 @@ pub fn parse_nth<'i, 't>(input: &mut Parser<'i, 't>) -> Result<(i32, i32), Basic
 
 
 fn parse_b<'i, 't>(input: &mut Parser<'i, 't>, a: i32) -> Result<(i32, i32), BasicParseError<'i>> {
-    let start_position = input.position();
+    let start = input.state();
     match input.next() {
         Ok(&Token::Delim('+')) => parse_signless_b(input, a, 1),
         Ok(&Token::Delim('-')) => parse_signless_b(input, a, -1),
         Ok(&Token::Number { has_sign: true, int_value: Some(b), .. }) => Ok((a, b)),
         _ => {
-            input.reset(start_position);
+            input.reset(&start);
             Ok((a, 0))
         }
     }

src/parser.rs

@@ -6,20 +6,40 @@ use cow_rc_str::CowRcStr;
 use std::ops::Range;
 use std::ascii::AsciiExt;
 use std::ops::BitOr;
-use tokenizer::{self, Token, Tokenizer, SourceLocation};
+use tokenizer::{Token, Tokenizer, SourcePosition, SourceLocation};
 
 
 /// A capture of the internal state of a `Parser` (including the position within the input),
 /// obtained from the `Parser::position` method.
 ///
 /// Can be used with the `Parser::reset` method to restore that state.
 /// Should only be used with the `Parser` instance it came from.
-#[derive(PartialEq, Eq, Debug, Clone, Copy)]
-pub struct SourcePosition {
-    position: tokenizer::SourcePosition,
-    at_start_of: Option<BlockType>,
+#[derive(Debug, Clone)]
+pub struct ParserState {
+    pub(crate) position: usize,
+    pub(crate) current_line_start_position: usize,
+    pub(crate) current_line_number: u32,
+    pub(crate) at_start_of: Option<BlockType>,
 }
 
+impl ParserState {
+    /// The position from the start of the input, counted in UTF-8 bytes.
+    #[inline]
+    pub fn position(&self) -> SourcePosition {
+        SourcePosition(self.position)
+    }
+
+    /// The line number and column number
+    #[inline]
+    pub fn source_location(&self) -> SourceLocation {
+        SourceLocation {
+            line: self.current_line_number,
+            column: (self.position - self.current_line_start_position) as u32,
+        }
+    }
+}
+
+
 /// The funamental parsing errors that can be triggered by built-in parsing routines.
 #[derive(Clone, Debug, PartialEq)]
 pub enum BasicParseError<'a> {
@@ -68,8 +88,8 @@ pub struct ParserInput<'i> {
 
 struct CachedToken<'i> {
     token: Token<'i>,
-    start_position: tokenizer::SourcePosition,
-    end_position: tokenizer::SourcePosition,
+    start_position: SourcePosition,
+    end_state: ParserState,
 }
 
 impl<'i> ParserInput<'i> {
@@ -100,7 +120,7 @@ pub struct Parser<'i: 't, 't> {
 
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
-enum BlockType {
+pub(crate) enum BlockType {
     Parenthesis,
     SquareBracket,
     CurlyBracket,
@@ -224,24 +244,38 @@ impl<'i: 't, 't> Parser<'i, 't> {
     /// This ignores whitespace and comments.
     #[inline]
     pub fn expect_exhausted(&mut self) -> Result<(), BasicParseError<'i>> {
-        let start_position = self.position();
+        let start = self.state();
         let result = match self.next() {
             Err(BasicParseError::EndOfInput) => Ok(()),
             Err(e) => unreachable!("Unexpected error encountered: {:?}", e),
             Ok(t) => Err(BasicParseError::UnexpectedToken(t.clone())),
         };
-        self.reset(start_position);
+        self.reset(&start);
         result
     }
 
+    /// Return the current position within the input.
+    ///
+    /// This can be used with the `Parser::slice` and `slice_from` methods.
+    #[inline]
+    pub fn position(&self) -> SourcePosition {
+        self.input.tokenizer.position()
+    }
+
+    /// The current line number and column number.
+    #[inline]
+    pub fn current_source_location(&self) -> SourceLocation {
+        self.input.tokenizer.current_source_location()
+    }
+
     /// Return the current internal state of the parser (including position within the input).
     ///
     /// This state can later be restored with the `Parser::reset` method.
     #[inline]
-    pub fn position(&self) -> SourcePosition {
-        SourcePosition {
-            position: self.input.tokenizer.position(),
+    pub fn state(&self) -> ParserState {
+        ParserState {
             at_start_of: self.at_start_of,
+            .. self.input.tokenizer.state()
         }
     }
 
@@ -250,9 +284,9 @@ impl<'i: 't, 't> Parser<'i, 't> {
     ///
     /// Should only be used with `SourcePosition` values from the same `Parser` instance.
     #[inline]
-    pub fn reset(&mut self, new_position: SourcePosition) {
-        self.input.tokenizer.reset(new_position.position);
-        self.at_start_of = new_position.at_start_of;
+    pub fn reset(&mut self, state: &ParserState) {
+        self.input.tokenizer.reset(state);
+        self.at_start_of = state.at_start_of;
     }
 
     /// Start looking for `var()` functions. (See the `.seen_var_functions()` method.)
@@ -289,36 +323,24 @@ impl<'i: 't, 't> Parser<'i, 't> {
     #[inline]
     pub fn try<F, T, E>(&mut self, thing: F) -> Result<T, E>
     where F: FnOnce(&mut Parser<'i, 't>) -> Result<T, E> {
-        let start_position = self.position();
+        let start = self.state();
         let result = thing(self);
         if result.is_err() {
-            self.reset(start_position)
+            self.reset(&start)
         }
         result
     }
 
     /// Return a slice of the CSS input
     #[inline]
     pub fn slice(&self, range: Range<SourcePosition>) -> &'i str {
-        self.input.tokenizer.slice(range.start.position..range.end.position)
+        self.input.tokenizer.slice(range)
     }
 
     /// Return a slice of the CSS input, from the given position to the current one.
     #[inline]
     pub fn slice_from(&self, start_position: SourcePosition) -> &'i str {
-        self.input.tokenizer.slice_from(start_position.position)
-    }
-
-    /// Return the line and column number within the input for the current position.
-    #[inline]
-    pub fn current_source_location(&self) -> SourceLocation {
-        self.input.tokenizer.current_source_location()
-    }
-
-    /// Return the line and column number within the input for the given position.
-    #[inline]
-    pub fn source_location(&self, target: SourcePosition) -> SourceLocation {
-        self.input.tokenizer.source_location(target.position)
+        self.input.tokenizer.slice_from(start_position)
     }
 
     /// Return the next token in the input that is neither whitespace or a comment,
@@ -374,8 +396,9 @@ impl<'i: 't, 't> Parser<'i, 't> {
         let token_start_position = self.input.tokenizer.position();
         let token;
         match self.input.cached_token {
-            Some(ref cached_token) if cached_token.start_position == token_start_position => {
-                self.input.tokenizer.reset(cached_token.end_position);
+            Some(ref cached_token)
+            if cached_token.start_position == token_start_position => {
+                self.input.tokenizer.reset(&cached_token.end_state);
                 match cached_token.token {
                     Token::Dimension { ref unit, .. } => self.input.tokenizer.see_dimension(unit),
                     Token::Function(ref name) => self.input.tokenizer.see_function(name),
@@ -388,7 +411,7 @@ impl<'i: 't, 't> Parser<'i, 't> {
                 self.input.cached_token = Some(CachedToken {
                     token: new_token,
                     start_position: token_start_position,
-                    end_position: self.input.tokenizer.position(),
+                    end_state: self.input.tokenizer.state(),
                 });
                 token = self.input.cached_token_ref()
             }