Move buffer_queue and smallcharset to markup5ever

Author: Ygg01

html5ever/src/driver.rs

@@ -9,8 +9,8 @@
 
 //! High-level interface to the parser.
 
-use tokenizer::buffer_queue::BufferQueue;
 use tokenizer::{Tokenizer, TokenizerOpts, TokenizerResult};
+use markup5ever::util::buffer_queue::BufferQueue;
 use tree_builder::{TreeBuilderOpts, TreeBuilder, TreeSink};
 
 use std::borrow::Cow;

html5ever/src/tokenizer/char_ref/mod.rs

@@ -8,7 +8,7 @@
 // except according to those terms.
 
 use super::{Tokenizer, TokenSink};
-use super::buffer_queue::BufferQueue;
+use markup5ever::util::buffer_queue::BufferQueue;
 
 use util::str::{is_ascii_alnum};
 

html5ever/src/tokenizer/mod.rs

@@ -21,8 +21,6 @@ use self::states::{DoctypeIdKind, Public, System};
 
 use self::char_ref::{CharRef, CharRefTokenizer};
 
-use self::buffer_queue::{BufferQueue, SetResult, FromSet, NotFromSet};
-
 use util::str::lower_ascii_letter;
 
 use std::ascii::AsciiExt;
@@ -33,11 +31,10 @@ use std::collections::BTreeMap;
 
 use {LocalName, QualName};
 use tendril::StrTendril;
-use markup5ever::SmallCharSet;
+use markup5ever::{SmallCharSet};
 use markup5ever::interface::{Attribute};
 pub use markup5ever::util::buffer_queue::{BufferQueue, SetResult, FromSet, NotFromSet};
 
-pub mod buffer_queue;
 pub mod states;
 mod interface;
 mod char_ref;
@@ -1418,7 +1415,7 @@ mod test {
     use super::interface::{CharacterTokens, NullCharacterToken, EOFToken, ParseError};
     use super::interface::{TagKind, StartTag, EndTag, Tag};
 
-    use super::buffer_queue::{BufferQueue};
+    use markup5ever::util::buffer_queue::{BufferQueue};
     use std::mem::replace;
 
     use {LocalName};

html5ever/tests/tokenizer.rs

@@ -8,7 +8,6 @@
 // except according to those terms.
 
 extern crate rustc_serialize;
-extern crate tendril;
 extern crate test;
 extern crate html5ever;
 
@@ -31,14 +30,15 @@ use std::collections::BTreeMap;
 use std::borrow::Cow::Borrowed;
 
 use html5ever::{LocalName, QualName};
-use html5ever::tokenizer::{Doctype, Attribute, StartTag, EndTag, Tag};
+use html5ever::tokenizer::{Doctype, StartTag, EndTag, Tag};
 use html5ever::tokenizer::{Token, DoctypeToken, TagToken, CommentToken};
 use html5ever::tokenizer::{CharacterTokens, NullCharacterToken, EOFToken, ParseError};
 use html5ever::tokenizer::{TokenSink, Tokenizer, TokenizerOpts, TokenSinkResult};
-use html5ever::tokenizer::buffer_queue::BufferQueue;
+use html5ever::tokenizer::{BufferQueue};
 use html5ever::tokenizer::states::{Plaintext, RawData, Rcdata, Rawtext};
+use html5ever::tendril::*;
+use html5ever::{Attribute};
 
-use tendril::{StrTendril, SliceExt};
 
 // Return all ways of splitting the string into at most n
 // possibly-empty pieces.

markup5ever/lib.rs

@@ -37,6 +37,10 @@ include!(concat!(env!("OUT_DIR"), "/generated.rs"));
 
 pub mod data;
 pub mod interface;
+pub mod util {
+    pub mod buffer_queue;
+    pub mod smallcharset;
 }
 
 pub use interface::{QualName, Attribute};
+pub use util::smallcharset::SmallCharSet;

html5ever/src/tokenizer/buffer_queue.rs renamed to markup5ever/util/buffer_queue.rs

@@ -8,12 +8,13 @@
 // except according to those terms.
 
 
+use std::ascii::AsciiExt;
 use std::collections::VecDeque;
 
 use tendril::StrTendril;
-use markup5ever::SmallCharSet;
 
 pub use self::SetResult::{FromSet, NotFromSet};
+use util::smallcharset::SmallCharSet;
 
 /// Result from `pop_except_from`.
 #[derive(PartialEq, Eq, Debug)]

markup5ever/util/smallcharset.rs

@@ -0,0 +1,64 @@
+// Copyright 2014 The html5ever Project Developers. See the
+// COPYRIGHT file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[macro_export]
+macro_rules! small_char_set ( ($($e:expr)+) => (
+    ::markup5ever::SmallCharSet {
+        bits: $( (1 << ($e as usize)) )|+
+    }
+));
+
+/// Represents a set of "small characters", those with Unicode scalar
+/// values less than 64.
+pub struct SmallCharSet {
+    pub bits: u64,
+}
+
+impl SmallCharSet {
+    #[inline]
+    fn contains(&self, n: u8) -> bool {
+        0 != (self.bits & (1 << (n as usize)))
+    }
+
+    /// Count the number of bytes of characters at the beginning
+    /// of `buf` which are not in the set.
+    /// See `tokenizer::buffer_queue::pop_except_from`.
+    pub fn nonmember_prefix_len(&self, buf: &str) -> u32 {
+        let mut n = 0;
+        for b in buf.bytes() {
+            if b >= 64 || !self.contains(b) {
+                n += 1;
+            } else {
+                break;
+            }
+        }
+        n
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::iter::repeat;
+
+    #[test]
+    fn nonmember_prefix() {
+        for &c in ['&', '\0'].iter() {
+            for x in 0 .. 48u32 {
+                for y in 0 .. 48u32 {
+                    let mut s = repeat("x").take(x as usize).collect::<String>();
+                    s.push(c);
+                    s.push_str(&repeat("x").take(y as usize).collect::<String>());
+                    let set = small_char_set!('&' '\0');
+
+                    assert_eq!(x, set.nonmember_prefix_len(&s));
+                }
+            }
+        }
+    }
+}