Merge pull request #86 from glandium/tables

Avoid slices in entries of decomposition tables

Author: Manishearth

scripts/unicode.py

@@ -391,9 +391,19 @@ def gen_composition_table(canon_comp, out):
 def gen_decomposition_tables(canon_decomp, compat_decomp, cjk_compat_variants_decomp, out):
     tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (cjk_compat_variants_decomp, 'cjk_compat_variants')]
     for table, name in tables:
-        gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])",
-            lambda k: "(0x{:x}, &[{}])".format(k,
-                ", ".join("'\\u{%s}'" % hexify(c) for c in table[k])))
+        offsets = {}
+        offset = 0
+        out.write("pub(crate) const %s_DECOMPOSED_CHARS: &[char] = &[\n" % name.upper())
+        for k, v in table.items():
+            offsets[k] = offset
+            offset += len(v)
+            for c in v:
+                out.write("    '\\u{%s}',\n" % hexify(c))
+        # The largest offset must fit in a u16.
+        assert offset < 65536
+        out.write("];\n")
+        gen_mph_data(name + '_decomposed', table, "(u32, (u16, u16))",
+            lambda k: "(0x{:x}, ({}, {}))".format(k, offsets[k], len(table[k])))
 
 def gen_qc_match(prop_table, out):
     out.write("    match c {\n")

src/lookups.rs

@@ -51,6 +51,7 @@ pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> {
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &CANONICAL_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {
@@ -62,6 +63,7 @@ pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]>
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &COMPATIBILITY_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [char]> {
@@ -73,6 +75,7 @@ pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &CJK_COMPAT_VARIANTS_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 /// Return whether the given character is a combining mark (`General_Category=Mark`)