Remove broken BytesParser

It cannot be the future API for handling `<meta charset>`
since it does not support interupting for scripts.

Author: SimonSapin

html5ever/src/driver.rs

@@ -15,12 +15,10 @@ use tokenizer::{Tokenizer, TokenizerOpts, TokenizerResult};
 use tree_builder::{TreeBuilderOpts, TreeBuilder, TreeSink, create_element};
 
 use std::borrow::Cow;
-use std::mem;
 
-use encoding::{self, EncodingRef};
 use tendril;
-use tendril::{StrTendril, ByteTendril};
-use tendril::stream::{TendrilSink, Utf8LossyDecoder, LossyDecoder};
+use tendril::StrTendril;
+use tendril::stream::{TendrilSink, Utf8LossyDecoder};
 
 /// All-encompassing options struct for the parser.
 #[derive(Clone, Default)]
@@ -114,220 +112,20 @@ impl<Sink: TreeSink> Parser<Sink> {
     pub fn from_utf8(self) -> Utf8LossyDecoder<Self> {
         Utf8LossyDecoder::new(self)
     }
-
-    /// Wrap this parser into a `TendrilSink` that accepts bytes
-    /// and tries to detect the correct character encoding.
-    ///
-    /// Currently this looks for a Byte Order Mark,
-    /// then uses `BytesOpts::transport_layer_encoding`,
-    /// then falls back to UTF-8.
-    ///
-    /// FIXME(https://github.com/servo/html5ever/issues/18): this should look for `<meta>` elements
-    /// and other data per
-    /// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
-    pub fn from_bytes(self, opts: BytesOpts) -> BytesParser<Sink> {
-        BytesParser {
-            state: BytesParserState::Initial { parser: self },
-            opts: opts,
-        }
-    }
-}
-
-/// Options for choosing a character encoding
-#[derive(Clone, Default)]
-pub struct BytesOpts {
-    /// The character encoding specified by the transport layer, if any.
-    /// In HTTP for example, this is the `charset` parameter of the `Content-Type` response header.
-    pub transport_layer_encoding: Option<EncodingRef>,
-}
-
-/// An HTML parser,
-/// ready to receive bytes input through the `tendril::TendrilSink` trait’s methods.
-///
-/// See `Parser::from_bytes`.
-pub struct BytesParser<Sink> where Sink: TreeSink {
-    state: BytesParserState<Sink>,
-    opts: BytesOpts,
-}
-
-enum BytesParserState<Sink> where Sink: TreeSink {
-    Initial {
-        parser: Parser<Sink>,
-    },
-    Buffering {
-        parser: Parser<Sink>,
-        buffer: ByteTendril
-    },
-    Parsing {
-        decoder: LossyDecoder<Parser<Sink>>,
-    },
-    Transient
-}
-
-impl<Sink: TreeSink> BytesParser<Sink> {
-    /// Access the underlying Parser
-    pub fn str_parser(&self) -> &Parser<Sink> {
-        match self.state {
-            BytesParserState::Initial { ref parser } => parser,
-            BytesParserState::Buffering { ref parser, .. } => parser,
-            BytesParserState::Parsing { ref decoder } => decoder.inner_sink(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    /// Access the underlying Parser
-    pub fn str_parser_mut(&mut self) -> &mut Parser<Sink> {
-        match self.state {
-            BytesParserState::Initial { ref mut parser } => parser,
-            BytesParserState::Buffering { ref mut parser, .. } => parser,
-            BytesParserState::Parsing { ref mut decoder } => decoder.inner_sink_mut(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    /// Insert a Unicode chunk in the middle of the byte stream.
-    ///
-    /// This is e.g. for supporting `document.write`.
-    pub fn process_unicode(&mut self, t: StrTendril) {
-        if t.is_empty() {
-            return  // Don’t prevent buffering/encoding detection
-        }
-        if let BytesParserState::Parsing { ref mut decoder } = self.state {
-            decoder.inner_sink_mut().process(t)
-        } else {
-            match mem::replace(&mut self.state, BytesParserState::Transient) {
-                BytesParserState::Initial { mut parser } => {
-                    parser.process(t);
-                    self.start_parsing(parser, ByteTendril::new())
-                }
-                BytesParserState::Buffering { parser, buffer } => {
-                    self.start_parsing(parser, buffer);
-                    if let BytesParserState::Parsing { ref mut decoder } = self.state {
-                        decoder.inner_sink_mut().process(t)
-                    } else {
-                        unreachable!()
-                    }
-                }
-                BytesParserState::Parsing { .. } | BytesParserState::Transient => unreachable!(),
-            }
-        }
-    }
-
-    fn start_parsing(&mut self, parser: Parser<Sink>, buffer: ByteTendril) {
-        let encoding = detect_encoding(&buffer, &self.opts);
-        let mut decoder = LossyDecoder::new(encoding, parser);
-        decoder.process(buffer);
-        self.state = BytesParserState::Parsing { decoder: decoder }
-    }
-}
-
-impl<Sink: TreeSink> TendrilSink<tendril::fmt::Bytes> for BytesParser<Sink> {
-    fn process(&mut self, t: ByteTendril) {
-        if let &mut BytesParserState::Parsing { ref mut decoder } = &mut self.state {
-            return decoder.process(t)
-        }
-        let (parser, buffer) = match mem::replace(&mut self.state, BytesParserState::Transient) {
-            BytesParserState::Initial{ parser } => (parser, t),
-            BytesParserState::Buffering { parser, mut buffer } => {
-                buffer.push_tendril(&t);
-                (parser, buffer)
-            }
-            BytesParserState::Parsing { .. } | BytesParserState::Transient => unreachable!(),
-        };
-        if buffer.len32() >= PRESCAN_BYTES {
-            self.start_parsing(parser, buffer)
-        } else {
-            self.state = BytesParserState::Buffering {
-                parser: parser,
-                buffer: buffer,
-            }
-        }
-    }
-
-    fn error(&mut self, desc: Cow<'static, str>) {
-        match self.state {
-            BytesParserState::Initial { ref mut parser } => parser.error(desc),
-            BytesParserState::Buffering { ref mut parser, .. } => parser.error(desc),
-            BytesParserState::Parsing { ref mut decoder } => decoder.error(desc),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-
-    type Output = Sink::Output;
-
-    fn finish(self) -> Self::Output {
-        match self.state {
-            BytesParserState::Initial { parser } => parser.finish(),
-            BytesParserState::Buffering { parser, buffer } => {
-                let encoding = detect_encoding(&buffer, &self.opts);
-                let mut decoder = LossyDecoder::new(encoding, parser);
-                decoder.process(buffer);
-                decoder.finish()
-            },
-            BytesParserState::Parsing { decoder } => decoder.finish(),
-            BytesParserState::Transient => unreachable!(),
-        }
-    }
-}
-
-/// How many bytes does detect_encoding() need
-// FIXME(#18): should be 1024 for <meta> elements.
-const PRESCAN_BYTES: u32 = 3;
-
-/// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
-fn detect_encoding(bytes: &ByteTendril, opts: &BytesOpts) -> EncodingRef {
-    if bytes.starts_with(b"\xEF\xBB\xBF") {
-        return encoding::all::UTF_8
-    }
-    if bytes.starts_with(b"\xFE\xFF") {
-        return encoding::all::UTF_16BE
-    }
-    if bytes.starts_with(b"\xFF\xFE") {
-        return encoding::all::UTF_16LE
-    }
-    if let Some(encoding) = opts.transport_layer_encoding {
-        return encoding
-    }
-    // FIXME(#18): <meta> etc.
-    return encoding::all::UTF_8
 }
 
 #[cfg(test)]
 mod tests {
     use rcdom::RcDom;
     use serialize::serialize;
-    use std::iter::repeat;
     use tendril::TendrilSink;
     use super::*;
 
     #[test]
     fn from_utf8() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_utf8()
-                .one("<title>Test".as_bytes()));
-    }
-
-    #[test]
-    fn from_bytes_one() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_bytes(BytesOpts::default())
-                .one("<title>Test".as_bytes()));
-    }
-
-    #[test]
-    fn from_bytes_iter() {
-        assert_serialization(
-            parse_document(RcDom::default(), ParseOpts::default())
-                .from_bytes(BytesOpts::default())
-                .from_iter([
-                    "<title>Test".as_bytes(),
-                    repeat(' ').take(1200).collect::<String>().as_bytes(),
-                ].iter().cloned()));
-    }
-
-    fn assert_serialization(dom: RcDom) {
+        let dom = parse_document(RcDom::default(), ParseOpts::default())
+            .from_utf8()
+            .one("<title>Test".as_bytes());
         let mut serialized = Vec::new();
         serialize(&mut serialized, &dom.document, Default::default()).unwrap();
         assert_eq!(String::from_utf8(serialized).unwrap().replace(" ", ""),

html5ever/src/lib.rs

@@ -33,6 +33,3 @@ pub mod serialize;
 pub mod tokenizer;
 pub mod tree_builder;
 pub mod driver;
-
-/// Re-export the encoding crate.
-pub use tendril::encoding;

html5ever/src/tree_builder/mod.rs

@@ -500,7 +500,7 @@ mod test {
     use ExpandedName;
     use QualName;
     use tendril::StrTendril;
-    use tendril::stream::{TendrilSink, Utf8LossyDecoder, LossyDecoder};
+    use tendril::stream::{TendrilSink, Utf8LossyDecoder};
 
     use tokenizer;
     use tokenizer::{Tokenizer, TokenizerOpts};

xml5ever/examples/hello_xml.rs

@@ -13,19 +13,15 @@ extern crate xml5ever;
 use std::default::Default;
 
 use xml5ever::tendril::TendrilSink;
-use xml5ever::driver::{parse_document, BytesOpts};
+use xml5ever::driver::parse_document;
 use xml5ever::tree_builder::{TreeSink};
 use xml5ever::rcdom::{RcDom, NodeData};
 
 fn main() {
     // To parse a string into a tree of nodes, we need to invoke
     // `parse_document` and supply it with a TreeSink implementation (RcDom).
-    //
-    // Since this is a string, it's best to use `from_bytes` to create a
-    // BytesParser for given string.
     let dom: RcDom = parse_document(RcDom::default(), Default::default())
-        .from_bytes(BytesOpts::default())
-        .one("<hello>XML</hello>".as_bytes());
+        .one("<hello>XML</hello>");
 
     // Do some processing
     let doc = &dom.document;