Move code out of tables

The code has been moved out of the tables module into perfect_hash, and
there is a bit more explanation in comments.

Author: raphlinus

scripts/unicode.py

@@ -286,67 +286,46 @@ def _compute_stream_safe_tables(self):
 
 hexify = lambda c: '{:04X}'.format(c)
 
-def gen_combining_class(combining_classes, out):
-    (salt, keys) = minimal_perfect_hash(combining_classes)
-    out.write("pub fn canonical_combining_class(c: char) -> u8 {\n")
-    out.write("    mph_lookup(c.into(), &[\n")
+def gen_mph_data(name, d, kv_type, kv_callback):
+    (salt, keys) = minimal_perfect_hash(d)
+    out.write("pub(crate) const %s_SALT: &[u16] = &[\n" % name.upper())
     for s in salt:
-        out.write("        0x{:x},\n".format(s))
-    out.write("    ],\n")
-    out.write("    &[\n")
+        out.write("    0x{:x},\n".format(s))
+    out.write("];\n")
+    out.write("pub(crate) const {}_KV: &[{}] = &[\n".format(name.upper(), kv_type))
     for k in keys:
-        kv = int(combining_classes[k]) | (k << 8)
-        out.write("        0x{:x},\n".format(kv))
-    out.write("    ],\n")
-    out.write("    u8_lookup_fk, u8_lookup_fv, 0)\n")
-    out.write("}\n")
+        out.write("    {},\n".format(kv_callback(k)))
+    out.write("];\n\n")
+
+def gen_combining_class(combining_classes, out):
+    gen_mph_data('canonical_combining_class', combining_classes, 'u32',
+        lambda k: "0x{:X}".format(int(combining_classes[k]) | (k << 8)))
 
 def gen_composition_table(canon_comp, out):
     table = {}
     for (c1, c2), c3 in canon_comp.items():
         if c1 < 0x10000 and c2 < 0x10000:
             table[(c1 << 16) | c2] = c3
     (salt, keys) = minimal_perfect_hash(table)
-    out.write("pub fn composition_table(c1: char, c2: char) -> Option<char> {\n")
-    out.write("    if c1 < '\\u{10000}' && c2 < '\\u{10000}' {\n")
-    out.write("        mph_lookup((c1 as u32) << 16 | (c2 as u32), &[\n")
-    for s in salt:
-        out.write("            0x{:x},\n".format(s))
-    out.write("        ],\n")
-    out.write("        &[\n")
-    for k in keys:
-        out.write("            (0x%s, '\\u{%s}'),\n" % (hexify(k), hexify(table[k])))
-    out.write("        ],\n")
-    out.write("        pair_lookup_fk, pair_lookup_fv_opt, None)\n")
-    out.write("    } else {\n")
-    out.write("        match (c1, c2) {\n")
+    gen_mph_data('COMPOSITION_TABLE', table, '(u32, char)',
+        lambda k: "(0x%s, '\\u{%s}')" % (hexify(k), hexify(table[k])))
 
+    out.write("pub(crate) fn composition_table_astral(c1: char, c2: char) -> Option<char> {\n")
+    out.write("    match (c1, c2) {\n")
     for (c1, c2), c3 in sorted(canon_comp.items()):
         if c1 >= 0x10000 and c2 >= 0x10000:
-            out.write("            ('\\u{%s}', '\\u{%s}') => Some('\\u{%s}'),\n" % (hexify(c1), hexify(c2), hexify(c3)))
+            out.write("        ('\\u{%s}', '\\u{%s}') => Some('\\u{%s}'),\n" % (hexify(c1), hexify(c2), hexify(c3)))
 
-    out.write("            _ => None,\n")
-    out.write("        }\n")
+    out.write("        _ => None,\n")
     out.write("    }\n")
     out.write("}\n")
 
 def gen_decomposition_tables(canon_decomp, compat_decomp, out):
     tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility')]
     for table, name in tables:
-        (salt, keys) = minimal_perfect_hash(table)
-        out.write("const {}_DECOMPOSED_KV: &[(u32, &'static [char])] = &[\n".format(name.upper()))
-        for char in keys:
-            d = ", ".join("'\\u{%s}'" % hexify(c) for c in table[char])
-            out.write("    (0x{:x}, &[{}]),\n".format(char, d))
-        out.write("];\n")
-        out.write("pub fn %s_fully_decomposed(c: char) -> Option<&'static [char]> {\n" % name)
-        out.write("    mph_lookup(c.into(), &[\n")
-        for s in salt:
-            out.write("        0x{:x},\n".format(s))
-        out.write("    ],\n")
-        out.write("    {}_DECOMPOSED_KV,\n".format(name.upper()))
-        out.write("    pair_lookup_fk, pair_lookup_fv_opt, None)\n")
-        out.write("}\n")
+        gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])",
+            lambda k: "(0x{:x}, &[{}])".format(k,
+                ", ".join("'\\u{%s}'" % hexify(c) for c in table[k])))
 
 def gen_qc_match(prop_table, out):
     out.write("    match c {\n")
@@ -388,18 +367,8 @@ def gen_nfkd_qc(prop_tables, out):
     out.write("}\n")
 
 def gen_combining_mark(general_category_mark, out):
-    (salt, keys) = minimal_perfect_hash(general_category_mark)
-    out.write("pub fn is_combining_mark(c: char) -> bool {\n")
-    out.write("    mph_lookup(c.into(), &[\n")
-    for s in salt:
-        out.write("        0x{:x},\n".format(s))
-    out.write("    ],\n")
-    out.write("    &[\n")
-    for k in keys:
-        out.write("        0x{:x},\n".format(k))
-    out.write("    ],\n")
-    out.write("    bool_lookup_fk, bool_lookup_fv, false)\n")
-    out.write("}\n")
+    gen_mph_data('combining_mark', general_category_mark, 'u32',
+        lambda k: '0x{:04x}'.format(k))
 
 def gen_stream_safe(leading, trailing, out):
     # This could be done as a hash but the table is very small.
@@ -415,19 +384,8 @@ def gen_stream_safe(leading, trailing, out):
     out.write("}\n")
     out.write("\n")
 
-    (salt, keys) = minimal_perfect_hash(trailing)
-    out.write("pub fn stream_safe_trailing_nonstarters(c: char) -> usize {\n")
-    out.write("    mph_lookup(c.into(), &[\n")
-    for s in salt:
-        out.write("        0x{:x},\n".format(s))
-    out.write("    ],\n")
-    out.write("    &[\n")
-    for k in keys:
-        kv = int(trailing[k]) | (k << 8)
-        out.write("        0x{:x},\n".format(kv))
-    out.write("    ],\n")
-    out.write("    u8_lookup_fk, u8_lookup_fv, 0) as usize\n")
-    out.write("}\n")
+    gen_mph_data('trailing_nonstarters', trailing, 'u32',
+        lambda k: "0x{:X}".format(int(trailing[k]) | (k << 8)))
 
 def gen_tests(tests, out):
     out.write("""#[derive(Debug)]
@@ -463,7 +421,7 @@ def my_hash(x, salt, n):
     return (y * n) >> 32
 
 # Compute minimal perfect hash function, d can be either a dict or list of keys.
-def minimal_perfect_hash(d, singleton_buckets = False):
+def minimal_perfect_hash(d):
     n = len(d)
     buckets = dict((h, []) for h in range(n))
     for key in d:
@@ -475,18 +433,16 @@ def minimal_perfect_hash(d, singleton_buckets = False):
     salts = [0] * n
     keys = [0] * n
     for (bucket_size, h) in bsorted:
+        # Note: the traditional perfect hashing approach would also special-case
+        # bucket_size == 1 here and assign any empty slot, rather than iterating
+        # until rehash finds an empty slot. But we're not doing that so we can
+        # avoid the branch.
         if bucket_size == 0:
             break
-        elif singleton_buckets and bucket_size == 1:
-            for i in range(n):
-                if not claimed[i]:
-                    salts[h] = -(i + 1)
-                    claimed[i] = True
-                    keys[i] = buckets[h][0]
-                    break
         else:
             for salt in range(1, 32768):
                 rehashes = [my_hash(key, salt, n) for key in buckets[h]]
+                # Make sure there are no rehash collisions within this bucket.
                 if all(not claimed[hash] for hash in rehashes):
                     if len(set(rehashes)) < bucket_size:
                         continue
@@ -498,6 +454,17 @@ def minimal_perfect_hash(d, singleton_buckets = False):
                     break
             if salts[h] == 0:
                 print("minimal perfect hashing failed")
+                # Note: if this happens (because of unfortunate data), then there are
+                # a few things that could be done. First, the hash function could be
+                # tweaked. Second, the bucket order could be scrambled (especially the
+                # singletons). Right now, the buckets are sorted, which has the advantage
+                # of being deterministic.
+                #
+                # As a more extreme approach, the singleton bucket optimization could be
+                # applied (give the direct address for singleton buckets, rather than
+                # relying on a rehash). That is definitely the more standard approach in
+                # the minimal perfect hashing literature, but in testing the branch was a
+                # significant slowdown.
                 exit(1)
     return (salts, keys)
 
@@ -508,8 +475,6 @@ def minimal_perfect_hash(d, singleton_buckets = False):
         out.write("use quick_check::IsNormalized;\n")
         out.write("use quick_check::IsNormalized::*;\n")
         out.write("\n")
-        out.write("use perfect_hash::*;\n")
-        out.write("\n")
 
         version = "(%s, %s, %s)" % tuple(UNICODE_VERSION.split("."))
         out.write("#[allow(unused)]\n")

src/lib.rs

@@ -81,11 +81,7 @@ mod normalization_tests;
 pub mod char {
     pub use normalize::{decompose_canonical, decompose_compatible, compose};
 
-    /// Look up the canonical combining class of a character.
-    pub use tables::canonical_combining_class;
-
-    /// Return whether the given character is a combining mark (`General_Category=Mark`)
-    pub use tables::is_combining_mark;
+    pub use perfect_hash::{canonical_combining_class, is_combining_mark};
 }
 
 

src/normalize.rs

@@ -11,23 +11,23 @@
 //! Functions for computing canonical and compatible decompositions for Unicode characters.
 use std::char;
 use std::ops::FnMut;
-use tables;
+use perfect_hash::{canonical_fully_decomposed, composition_table, compatibility_fully_decomposed};
 
 /// Compute canonical Unicode decomposition for character.
 /// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
 /// for more information.
 #[inline]
 pub fn decompose_canonical<F>(c: char, emit_char: F) where F: FnMut(char) {
-    decompose(c, tables::canonical_fully_decomposed, emit_char)
+    decompose(c, canonical_fully_decomposed, emit_char)
 }
 
 /// Compute canonical or compatible Unicode decomposition for character.
 /// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
 /// for more information.
 #[inline]
 pub fn decompose_compatible<F: FnMut(char)>(c: char, emit_char: F) {
-    let decompose_char = |c| tables::compatibility_fully_decomposed(c)
-        .or_else(|| tables::canonical_fully_decomposed(c));
+    let decompose_char = |c| compatibility_fully_decomposed(c)
+        .or_else(|| canonical_fully_decomposed(c));
     decompose(c, decompose_char, emit_char)
 }
 
@@ -62,7 +62,7 @@ fn decompose<D, F>(c: char, decompose_char: D, mut emit_char: F)
 /// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
 /// for more information.
 pub fn compose(a: char, b: char) -> Option<char> {
-    compose_hangul(a, b).or_else(|| tables::composition_table(a, b))
+    compose_hangul(a, b).or_else(|| composition_table(a, b))
 }
 
 // Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior

src/perfect_hash.rs

@@ -10,6 +10,8 @@
 
 //! Support for lookups based on minimal perfect hashing.
 
+use tables::*;
+
 // This function is based on multiplication being fast and is "good enough". Also
 // it can share some work between the unsalted and salted versions.
 #[inline]
@@ -27,7 +29,7 @@ fn my_hash(key: u32, salt: u32, n: usize) -> usize {
 /// The hash function doesn't have to be very good, just good enough that the
 /// resulting map is unique.
 #[inline]
-pub(crate) fn mph_lookup<KV, V, FK, FV>(x: u32, salt: &[u16], kv: &[KV], fk: FK, fv: FV,
+fn mph_lookup<KV, V, FK, FV>(x: u32, salt: &[u16], kv: &[KV], fk: FK, fv: FV,
     default: V) -> V
     where KV: Copy, FK: Fn(KV) -> u32, FV: Fn(KV) -> V
 {
@@ -42,37 +44,75 @@ pub(crate) fn mph_lookup<KV, V, FK, FV>(x: u32, salt: &[u16], kv: &[KV], fk: FK,
 
 /// Extract the key in a 24 bit key and 8 bit value packed in a u32.
 #[inline]
-pub(crate) fn u8_lookup_fk(kv: u32) -> u32 {
+fn u8_lookup_fk(kv: u32) -> u32 {
     kv >> 8
 }
 
 /// Extract the value in a 24 bit key and 8 bit value packed in a u32.
 #[inline]
-pub(crate) fn u8_lookup_fv(kv: u32) -> u8 {
+fn u8_lookup_fv(kv: u32) -> u8 {
     (kv & 0xff) as u8
 }
 
 /// Extract the key for a boolean lookup.
 #[inline]
-pub(crate) fn bool_lookup_fk(kv: u32) -> u32 {
+fn bool_lookup_fk(kv: u32) -> u32 {
     kv
 }
 
 /// Extract the value for a boolean lookup.
 #[inline]
-pub(crate) fn bool_lookup_fv(_kv: u32) -> bool {
+fn bool_lookup_fv(_kv: u32) -> bool {
     true
 }
 
 /// Extract the key in a pair.
 #[inline]
-pub(crate) fn pair_lookup_fk<T>(kv: (u32, T)) -> u32 {
+fn pair_lookup_fk<T>(kv: (u32, T)) -> u32 {
     kv.0
 }
 
 /// Extract the value in a pair, returning an option.
 #[inline]
-pub(crate) fn pair_lookup_fv_opt<T>(kv: (u32, T)) -> Option<T> {
+fn pair_lookup_fv_opt<T>(kv: (u32, T)) -> Option<T> {
     Some(kv.1)
 }
 
+/// Look up the canonical combining class for a codepoint.
+/// 
+/// The value returned is as defined in the Unicode Character Database.
+pub fn canonical_combining_class(c: char) -> u8 {
+    mph_lookup(c.into(), CANONICAL_COMBINING_CLASS_SALT, CANONICAL_COMBINING_CLASS_KV,
+        u8_lookup_fk, u8_lookup_fv, 0)
+}
+
+pub(crate) fn composition_table(c1: char, c2: char) -> Option<char> {
+    if c1 < '\u{10000}' && c2 < '\u{10000}' {
+        mph_lookup((c1 as u32) << 16 | (c2 as u32),
+        COMPOSITION_TABLE_SALT, COMPOSITION_TABLE_KV,
+        pair_lookup_fk, pair_lookup_fv_opt, None)
+    } else {
+        composition_table_astral(c1, c2)
+    }
+}
+
+pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> {
+    mph_lookup(c.into(), CANONICAL_DECOMPOSED_SALT, CANONICAL_DECOMPOSED_KV,
+        pair_lookup_fk, pair_lookup_fv_opt, None)
+}
+
+pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {
+    mph_lookup(c.into(), COMPATIBILITY_DECOMPOSED_SALT, COMPATIBILITY_DECOMPOSED_KV,
+        pair_lookup_fk, pair_lookup_fv_opt, None)
+}
+
+/// Return whether the given character is a combining mark (`General_Category=Mark`)
+pub fn is_combining_mark(c: char) -> bool {
+    mph_lookup(c.into(), COMBINING_MARK_SALT, COMBINING_MARK_KV,
+        bool_lookup_fk, bool_lookup_fv, false)
+}
+
+pub fn stream_safe_trailing_nonstarters(c: char) -> usize {
+    mph_lookup(c.into(), TRAILING_NONSTARTERS_SALT, TRAILING_NONSTARTERS_KV,
+        u8_lookup_fk, u8_lookup_fv, 0) as usize
+}

src/quick_check.rs

@@ -1,4 +1,5 @@
 use UnicodeNormalization;
+use perfect_hash::canonical_combining_class;
 use stream_safe;
 use tables;
 
@@ -32,7 +33,7 @@ fn quick_check<F, I>(s: I, is_allowed: F, stream_safe: bool) -> IsNormalized
         }
 
         // Otherwise, lookup the combining class and QC property
-        let cc = tables::canonical_combining_class(ch);
+        let cc = canonical_combining_class(ch);
         if last_cc > cc && cc != 0 {
             return IsNormalized::No;
         }