Merge pull request #24 from sujayakar/quick-ss

Implement Stream-Safe QuickCheck variant

Author: sujayakar

benches/bench.rs

@@ -6,52 +6,83 @@ extern crate test;
 use test::Bencher;
 use unicode_normalization::UnicodeNormalization;
 
+const ASCII: &'static str = "all types of normalized";
+const NFC: &'static str = "Introducci\u{00f3}n a Unicode.pdf";
+const NFD: &'static str = "Introduccio\u{0301}n a Unicode.pdf";
+
 #[bench]
 fn bench_is_nfc_ascii(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfc("all types of normalized"));
+    b.iter(|| unicode_normalization::is_nfc(ASCII));
 }
 
 #[bench]
 fn bench_is_nfc_normalized(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfc("Introducci\u{00f3}n a Unicode.pdf"));
+    b.iter(|| unicode_normalization::is_nfc(NFC));
 }
 
 #[bench]
 fn bench_is_nfc_not_normalized(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfc("Introduccio\u{0301}n a Unicode.pdf"));
+    b.iter(|| unicode_normalization::is_nfc(NFD));
 }
 
 #[bench]
 fn bench_is_nfd_ascii(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfd("an easy string to check"));
+    b.iter(|| unicode_normalization::is_nfd(ASCII));
 }
 
 #[bench]
 fn bench_is_nfd_normalized(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfd("Introduccio\u{0301}n a Unicode.pdf"));
+    b.iter(|| unicode_normalization::is_nfd(NFD));
 }
 
 #[bench]
 fn bench_is_nfd_not_normalized(b: &mut Bencher) {
-    b.iter(|| unicode_normalization::is_nfd("Introducci\u{00f3}n a Unicode.pdf"));
+    b.iter(|| unicode_normalization::is_nfd(NFC));
+}
+
+#[bench]
+fn bench_is_nfc_stream_safe_ascii(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfc_stream_safe(ASCII));
+}
+
+#[bench]
+fn bench_is_nfc_stream_safe_normalized(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfc_stream_safe(NFC));
+}
+
+#[bench]
+fn bench_is_nfc_stream_safe_not_normalized(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfc_stream_safe(NFD));
+}
+
+#[bench]
+fn bench_is_nfd_stream_safe_ascii(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfd_stream_safe(ASCII));
+}
+
+#[bench]
+fn bench_is_nfd_stream_safe_normalized(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfd_stream_safe(NFD));
+}
+
+#[bench]
+fn bench_is_nfd_stream_safe_not_normalized(b: &mut Bencher) {
+    b.iter(|| unicode_normalization::is_nfd_stream_safe(NFC));
 }
 
 #[bench]
 fn bench_nfc_ascii(b: &mut Bencher) {
-    let s = "normalize me please";
-    b.iter(|| s.nfc().count());
+    b.iter(|| ASCII.nfc().count());
 }
 
 #[bench]
 fn bench_nfd_ascii(b: &mut Bencher) {
-    let s = "decompose me entirely";
-    b.iter(|| s.nfd().count());
+    b.iter(|| ASCII.nfd().count());
 }
 
 #[bench]
 fn bench_streamsafe_ascii(b: &mut Bencher) {
-    let s = "quite nonthreatening";
-    b.iter(|| s.stream_safe().count());
+    b.iter(|| ASCII.stream_safe().count());
 }
 
 #[bench]

src/lib.rs

@@ -47,8 +47,12 @@ pub use quick_check::{
     IsNormalized,
     is_nfc,
     is_nfc_quick,
+    is_nfc_stream_safe,
+    is_nfc_stream_safe_quick,
     is_nfd,
     is_nfd_quick,
+    is_nfd_stream_safe,
+    is_nfd_stream_safe_quick,
 };
 pub use recompose::Recompositions;
 pub use stream_safe::StreamSafe;

src/quick_check.rs

@@ -1,9 +1,11 @@
 use UnicodeNormalization;
+use stream_safe;
 use tables;
 
 /// The QuickCheck algorithm can quickly determine if a text is or isn't
 /// normalized without any allocations in many cases, but it has to be able to
 /// return `Maybe` when a full decomposition and recomposition is necessary.
+#[derive(Debug, Eq, PartialEq)]
 pub enum IsNormalized {
     /// The text is definitely normalized.
     Yes,
@@ -15,17 +17,20 @@ pub enum IsNormalized {
 
 // https://unicode.org/reports/tr15/#Detecting_Normalization_Forms
 #[inline]
-fn quick_check<F, I>(s: I, is_allowed: F) -> IsNormalized
+fn quick_check<F, I>(s: I, is_allowed: F, stream_safe: bool) -> IsNormalized
     where I: Iterator<Item=char>, F: Fn(char) -> IsNormalized
 {
     let mut last_cc = 0u8;
+    let mut nonstarter_count = 0;
     let mut result = IsNormalized::Yes;
     for ch in s {
         // For ASCII we know it's always allowed and a starter
         if ch <= '\x7f' {
             last_cc = 0;
+            nonstarter_count = 0;
             continue;
         }
+
         // Otherwise, lookup the combining class and QC property
         let cc = tables::canonical_combining_class(ch);
         if last_cc > cc && cc != 0 {
@@ -38,6 +43,20 @@ fn quick_check<F, I>(s: I, is_allowed: F) -> IsNormalized
                 result = IsNormalized::Maybe;
             },
         }
+        if stream_safe {
+            let decomp = stream_safe::classify_nonstarters(ch);
+
+            // If we're above `MAX_NONSTARTERS`, we're definitely *not*
+            // stream-safe normalized.
+            if nonstarter_count + decomp.leading_nonstarters > stream_safe::MAX_NONSTARTERS {
+                return IsNormalized::No;
+            }
+            if decomp.leading_nonstarters == decomp.decomposition_len {
+                nonstarter_count += decomp.decomposition_len;
+            } else {
+                nonstarter_count = decomp.trailing_nonstarters;
+            }
+        }
         last_cc = cc;
     }
     result
@@ -48,16 +67,29 @@ fn quick_check<F, I>(s: I, is_allowed: F) -> IsNormalized
 /// like `s.chars().nfc().eq(s.chars())` should suffice.
 #[inline]
 pub fn is_nfc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
-    quick_check(s, tables::qc_nfc)
+    quick_check(s, tables::qc_nfc, false)
 }
 
 /// Quickly check if a string is in NFD.
 #[inline]
 pub fn is_nfd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
-    quick_check(s, tables::qc_nfd)
+    quick_check(s, tables::qc_nfd, false)
+}
+
+/// Quickly check if a string is Stream-Safe NFC.
+#[inline]
+pub fn is_nfc_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
+    quick_check(s, tables::qc_nfc, true)
+}
+
+/// Quickly check if a string is Stream-Safe NFD.
+#[inline]
+pub fn is_nfd_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
+    quick_check(s, tables::qc_nfd, true)
 }
 
 /// Authoritatively check if a string is in NFC.
+#[inline]
 pub fn is_nfc(s: &str) -> bool {
     match is_nfc_quick(s.chars()) {
         IsNormalized::Yes => true,
@@ -67,10 +99,58 @@ pub fn is_nfc(s: &str) -> bool {
 }
 
 /// Authoritatively check if a string is in NFD.
+#[inline]
 pub fn is_nfd(s: &str) -> bool {
     match is_nfd_quick(s.chars()) {
         IsNormalized::Yes => true,
         IsNormalized::No => false,
         IsNormalized::Maybe => s.chars().eq(s.chars().nfd()),
     }
 }
+
+/// Authoritatively check if a string is Stream-Safe NFC.
+#[inline]
+pub fn is_nfc_stream_safe(s: &str) -> bool {
+    match is_nfc_stream_safe_quick(s.chars()) {
+        IsNormalized::Yes => true,
+        IsNormalized::No => false,
+        IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfc()),
+    }
+}
+
+/// Authoritatively check if a string is Stream-Safe NFD.
+#[inline]
+pub fn is_nfd_stream_safe(s: &str) -> bool {
+    match is_nfd_stream_safe_quick(s.chars()) {
+        IsNormalized::Yes => true,
+        IsNormalized::No => false,
+        IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfd()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        IsNormalized,
+        is_nfc_stream_safe_quick,
+        is_nfd_stream_safe_quick,
+    };
+
+    #[test]
+    fn test_stream_safe_nfd() {
+        let okay = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone";
+        assert_eq!(is_nfd_stream_safe_quick(okay.chars()), IsNormalized::Yes);
+
+        let too_much = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone";
+        assert_eq!(is_nfd_stream_safe_quick(too_much.chars()), IsNormalized::No);
+    }
+
+    #[test]
+    fn test_stream_safe_nfc() {
+        let okay = "ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y";
+        assert_eq!(is_nfc_stream_safe_quick(okay.chars()), IsNormalized::Maybe);
+
+        let too_much = "not ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y";
+        assert_eq!(is_nfc_stream_safe_quick(too_much.chars()), IsNormalized::No);
+    }
+}

src/stream_safe.rs

@@ -4,7 +4,7 @@ use normalize::{
 };
 use tables;
 
-const MAX_NONSTARTERS: usize = 30;
+pub(crate) const MAX_NONSTARTERS: usize = 30;
 const COMBINING_GRAPHEME_JOINER: char = '\u{034F}';
 
 /// UAX15-D4: This iterator keeps track of how many non-starters there have been
@@ -54,14 +54,14 @@ impl<I: Iterator<Item=char>> Iterator for StreamSafe<I> {
 }
 
 #[derive(Debug)]
-struct Decomposition {
-    leading_nonstarters: usize,
-    trailing_nonstarters: usize,
-    decomposition_len: usize,
+pub(crate) struct Decomposition {
+    pub(crate) leading_nonstarters: usize,
+    pub(crate) trailing_nonstarters: usize,
+    pub(crate) decomposition_len: usize,
 }
 
 #[inline]
-fn classify_nonstarters(c: char) -> Decomposition {
+pub(crate) fn classify_nonstarters(c: char) -> Decomposition {
     // As usual, fast path for ASCII (which is always a starter)
     if c <= '\x7f' {
         return Decomposition {