Auto merge of #272 - servo:utf8, r=nox

Remove broken BytesParser

It cannot be the future API for handling `<meta charset>` since it does not support interrupting for scripts.

Update Tendril to a version that doesn’t depend on rust-encoding (unless a Cargo feature is enabled).

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/html5ever/272)
<!-- Reviewable:end -->

Author: bors-servo

html5ever/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 
 name = "html5ever"
-version = "0.16.0"
+version = "0.17.0"
 authors = [ "The html5ever Project Developers" ]
 license = "MIT / Apache-2.0"
 repository = "https://github.com/servo/html5ever"
@@ -30,13 +30,12 @@ name = "tokenizer"
 harness = false
 
 [features]
-unstable = ["tendril/unstable"]
+unstable = ["markup5ever/unstable"]
 heap_size = ["markup5ever/heap_size"]
 
 [dependencies]
-log = "0"
-mac = "0"
-tendril = "0.2.2"
+log = "0.3"
+mac = "0.1"
 markup5ever = { version = "0.1", path = "../markup5ever" }
 
 [dev-dependencies]

html5ever/examples/html2html.rs

@@ -15,18 +15,17 @@
 //!
 //! where htmlparser-1.4.jar comes from http://about.validator.nu/htmlparser/
 
-extern crate tendril;
 extern crate html5ever;
 
 use std::io::{self, Write};
 use std::default::Default;
 
-use tendril::TendrilSink;
 
-use html5ever::driver::ParseOpts;
-use html5ever::tree_builder::TreeBuilderOpts;
 use html5ever::{parse_document, serialize};
+use html5ever::driver::ParseOpts;
 use html5ever::rcdom::RcDom;
+use html5ever::tendril::TendrilSink;
+use html5ever::tree_builder::TreeBuilderOpts;
 
 fn main() {
     let opts = ParseOpts {

html5ever/examples/print-rcdom.rs

@@ -8,16 +8,15 @@
 // except according to those terms.
 
 #[macro_use] extern crate html5ever;
-extern crate tendril;
 
 use std::io;
 use std::iter::repeat;
 use std::default::Default;
 use std::string::String;
 
-use tendril::TendrilSink;
 use html5ever::parse_document;
 use html5ever::rcdom::{NodeData, RcDom, Handle};
+use html5ever::tendril::TendrilSink;
 
 // This is not proper HTML serialization, of course.
 

html5ever/src/driver.rs

@@ -15,12 +15,10 @@ use tokenizer::{Tokenizer, TokenizerOpts, TokenizerResult};
 use tree_builder::{TreeBuilderOpts, TreeBuilder, TreeSink, create_element};
 
 use std::borrow::Cow;
-use std::mem;
 
-use encoding::{self, EncodingRef};
 use tendril;
-use tendril::{StrTendril, ByteTendril};
-use tendril::stream::{TendrilSink, Utf8LossyDecoder, LossyDecoder};
+use tendril::StrTendril;
+use tendril::stream::{TendrilSink, Utf8LossyDecoder};
 
 /// All-encompassing options struct for the parser.
 #[derive(Clone, Default)]
@@ -92,7 +90,7 @@ impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> {
 
     // FIXME: Is it too noisy to report every character decoding error?
     fn error(&mut self, desc: Cow<'static, str>) {
-        self.tokenizer.sink_mut().sink_mut().parse_error(desc)
+        self.tokenizer.sink.sink.parse_error(desc)
     }
 
     type Output = Sink::Output;
@@ -102,7 +100,7 @@ impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> {
         while let TokenizerResult::Script(_) = self.tokenizer.feed(&mut self.input_buffer) {}
         assert!(self.input_buffer.is_empty());
         self.tokenizer.end();
-        self.tokenizer.unwrap().unwrap().finish()
+        self.tokenizer.sink.sink.finish()
     }
 }
 
@@ -114,220 +112,20 @@ impl<Sink: TreeSink> Parser<Sink> {
     pub fn from_utf8(self) -> Utf8LossyDecoder<Self> {
         Utf8LossyDecoder::new(self)
     }
-
-    /// Wrap this parser into a `TendrilSink` that accepts bytes
-    /// and tries to detect the correct character encoding.
-    ///
-    /// Currently this looks for a Byte Order Mark,
-    /// then uses `BytesOpts::transport_layer_encoding`,
-    /// then falls back to UTF-8.
-    ///
-    /// FIXME(https://github.com/servo/html5ever/issues/18): this should look for `<meta>` elements
-    /// and other data per
-    /// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
-    pub fn from_bytes(self, opts: BytesOpts) -> BytesParser<Sink> {
-        BytesParser {
-            state: BytesParserState::Initial { parser: self },
-            opts: opts,
-        }
-    }
-}
-
-/// Options for choosing a character encoding
-#[derive(Clone, Default)]
-pub struct BytesOpts {
-    /// The character encoding specified by the transport layer, if any.
-    /// In HTTP for example, this is the `charset` parameter of the `Content-Type` response header.
-    pub transport_layer_encoding: Option<EncodingRef>,
-}
-
-/// An HTML parser,
-/// ready to receive bytes input through the `tendril::TendrilSink` trait’s methods.
-///
-/// See `Parser::from_bytes`.
-pub struct BytesParser<Sink> where Sink: TreeSink {
-    state: BytesParserState<Sink>,
-    opts: BytesOpts,
-}
-
-enum BytesParserState<Sink> where Sink: TreeSink {
-    Initial {
-        parser: Parser<Sink>,
-    },
-    Buffering {
-        parser: Parser<Sink>,
-        buffer: ByteTendril
-    },
-    Parsing {
-        decoder: LossyDecoder<Parser<Sink>>,
-    },
-    Transient
-}
-
-impl<Sink: TreeSink> BytesParser<Sink> {
-    /// Access the underlying Parser
-    pub fn str_parser(&self) -> &Parser<Sink> {
-        match self.state {
-            BytesParserState::Initial { ref parser } => parser,
-            BytesParserState::Buffering { ref parser, .. } => parser,
-            BytesParserState::Parsing { ref decoder } => decoder.inner_sink(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    /// Access the underlying Parser
-    pub fn str_parser_mut(&mut self) -> &mut Parser<Sink> {
-        match self.state {
-            BytesParserState::Initial { ref mut parser } => parser,
-            BytesParserState::Buffering { ref mut parser, .. } => parser,
-            BytesParserState::Parsing { ref mut decoder } => decoder.inner_sink_mut(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    /// Insert a Unicode chunk in the middle of the byte stream.
-    ///
-    /// This is e.g. for supporting `document.write`.
-    pub fn process_unicode(&mut self, t: StrTendril) {
-        if t.is_empty() {
-            return  // Don’t prevent buffering/encoding detection
-        }
-        if let BytesParserState::Parsing { ref mut decoder } = self.state {
-            decoder.inner_sink_mut().process(t)
-        } else {
-            match mem::replace(&mut self.state, BytesParserState::Transient) {
-                BytesParserState::Initial { mut parser } => {
-                    parser.process(t);
-                    self.start_parsing(parser, ByteTendril::new())
-                }
-                BytesParserState::Buffering { parser, buffer } => {
-                    self.start_parsing(parser, buffer);
-                    if let BytesParserState::Parsing { ref mut decoder } = self.state {
-                        decoder.inner_sink_mut().process(t)
-                    } else {
-                        unreachable!()
-                    }
-                }
-                BytesParserState::Parsing { .. } | BytesParserState::Transient => unreachable!(),
-            }
-        }
-    }
-
-    fn start_parsing(&mut self, parser: Parser<Sink>, buffer: ByteTendril) {
-        let encoding = detect_encoding(&buffer, &self.opts);
-        let mut decoder = LossyDecoder::new(encoding, parser);
-        decoder.process(buffer);
-        self.state = BytesParserState::Parsing { decoder: decoder }
-    }
-}
-
-impl<Sink: TreeSink> TendrilSink<tendril::fmt::Bytes> for BytesParser<Sink> {
-    fn process(&mut self, t: ByteTendril) {
-        if let &mut BytesParserState::Parsing { ref mut decoder } = &mut self.state {
-            return decoder.process(t)
-        }
-        let (parser, buffer) = match mem::replace(&mut self.state, BytesParserState::Transient) {
-            BytesParserState::Initial{ parser } => (parser, t),
-            BytesParserState::Buffering { parser, mut buffer } => {
-                buffer.push_tendril(&t);
-                (parser, buffer)
-            }
-            BytesParserState::Parsing { .. } | BytesParserState::Transient => unreachable!(),
-        };
-        if buffer.len32() >= PRESCAN_BYTES {
-            self.start_parsing(parser, buffer)
-        } else {
-            self.state = BytesParserState::Buffering {
-                parser: parser,
-                buffer: buffer,
-            }
-        }
-    }
-
-    fn error(&mut self, desc: Cow<'static, str>) {
-        match self.state {
-            BytesParserState::Initial { ref mut parser } => parser.error(desc),
-            BytesParserState::Buffering { ref mut parser, .. } => parser.error(desc),
-            BytesParserState::Parsing { ref mut decoder } => decoder.error(desc),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    type Output = Sink::Output;
-
-    fn finish(self) -> Self::Output {
-        match self.state {
-            BytesParserState::Initial { parser } => parser.finish(),
-            BytesParserState::Buffering { parser, buffer } => {
-                let encoding = detect_encoding(&buffer, &self.opts);
-                let mut decoder = LossyDecoder::new(encoding, parser);
-                decoder.process(buffer);
-                decoder.finish()
-            },
-            BytesParserState::Parsing { decoder } => decoder.finish(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-}
-
-/// How many bytes does detect_encoding() need
-// FIXME(#18): should be 1024 for <meta> elements.
-const PRESCAN_BYTES: u32 = 3;
-
-/// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
-fn detect_encoding(bytes: &ByteTendril, opts: &BytesOpts) -> EncodingRef {
-    if bytes.starts_with(b"\xEF\xBB\xBF") {
-        return encoding::all::UTF_8
-    }
-    if bytes.starts_with(b"\xFE\xFF") {
-        return encoding::all::UTF_16BE
-    }
-    if bytes.starts_with(b"\xFF\xFE") {
-        return encoding::all::UTF_16LE
-    }
-    if let Some(encoding) = opts.transport_layer_encoding {
-        return encoding
-    }
-    // FIXME(#18): <meta> etc.
-    return encoding::all::UTF_8
 }
 
 #[cfg(test)]
 mod tests {
     use rcdom::RcDom;
     use serialize::serialize;
-    use std::iter::repeat;
     use tendril::TendrilSink;
     use super::*;
 
     #[test]
     fn from_utf8() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_utf8()
-                .one("<title>Test".as_bytes()));
-    }
-
-    #[test]
-    fn from_bytes_one() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_bytes(BytesOpts::default())
-                .one("<title>Test".as_bytes()));
-    }
-
-    #[test]
-    fn from_bytes_iter() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_bytes(BytesOpts::default())
-                .from_iter([
-                    "<title>Test".as_bytes(),
-                    repeat(' ').take(1200).collect::<String>().as_bytes(),
-                ].iter().cloned()));
-    }
-
-    fn assert_serialization(dom: RcDom) {
+        let dom = parse_document(RcDom::default(), ParseOpts::default())
+            .from_utf8()
+            .one("<title>Test".as_bytes());
         let mut serialized = Vec::new();
         serialize(&mut serialized, &dom.document, Default::default()).unwrap();
         assert_eq!(String::from_utf8(serialized).unwrap().replace(" ", ""),

html5ever/src/lib.rs

@@ -33,12 +33,3 @@ pub mod serialize;
 pub mod tokenizer;
 pub mod tree_builder;
 pub mod driver;
-
-/// Re-export the tendril crate.
-pub mod tendril {
-    extern crate tendril;
-    pub use self::tendril::*;
-}
-
-/// Re-export the encoding crate.
-pub use tendril::encoding;