Merge pull request #86 from glandium/tables

Manishearth · web-flow · commit e9dc93ae8ea4 · 2022-06-23T15:20:43.000-07:00
Avoid slices in entries of decomposition tables
diff --git a/scripts/unicode.py b/scripts/unicode.py
@@ -391,9 +391,19 @@ def gen_composition_table(canon_comp, out):
 def gen_decomposition_tables(canon_decomp, compat_decomp, cjk_compat_variants_decomp, out):
     tables = [(canon_decomp, 'canonical'), (compat_decomp, 'compatibility'), (cjk_compat_variants_decomp, 'cjk_compat_variants')]
     for table, name in tables:
-        gen_mph_data(name + '_decomposed', table, "(u32, &'static [char])",
-            lambda k: "(0x{:x}, &[{}])".format(k,
-                ", ".join("'\\u{%s}'" % hexify(c) for c in table[k])))
+        offsets = {}
+        offset = 0
+        out.write("pub(crate) const %s_DECOMPOSED_CHARS: &[char] = &[\n" % name.upper())
+        for k, v in table.items():
+            offsets[k] = offset
+            offset += len(v)
+            for c in v:
+                out.write("    '\\u{%s}',\n" % hexify(c))
+        # The largest offset must fit in a u16.
+        assert offset < 65536
+        out.write("];\n")
+        gen_mph_data(name + '_decomposed', table, "(u32, (u16, u16))",
+            lambda k: "(0x{:x}, ({}, {}))".format(k, offsets[k], len(table[k])))
 
 def gen_qc_match(prop_table, out):
     out.write("    match c {\n")
diff --git a/src/lookups.rs b/src/lookups.rs
@@ -51,6 +51,7 @@ pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> {
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &CANONICAL_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {
@@ -62,6 +63,7 @@ pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]>
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &COMPATIBILITY_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [char]> {
@@ -73,6 +75,7 @@ pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [
         pair_lookup_fv_opt,
         None,
     )
+    .map(|(start, len)| &CJK_COMPAT_VARIANTS_DECOMPOSED_CHARS[start as usize..][..len as usize])
 }
 
 /// Return whether the given character is a combining mark (`General_Category=Mark`)
diff --git a/src/tables.rs b/src/tables.rs

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ pub(crate) fn canonical_fully_decomposed(c: char) -> Option<&'static [char]> {`
`51`	`51`	`pair_lookup_fv_opt,`
`52`	`52`	`None,`
`53`	`53`	`)`
	`54`	`+ .map(\|(start, len)\| &CANONICAL_DECOMPOSED_CHARS[start as usize..][..len as usize])`
`54`	`55`	`}`
`55`	`56`
`56`	`57`	`pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]> {`
`@@ -62,6 +63,7 @@ pub(crate) fn compatibility_fully_decomposed(c: char) -> Option<&'static [char]>`
`62`	`63`	`pair_lookup_fv_opt,`
`63`	`64`	`None,`
`64`	`65`	`)`
	`66`	`+ .map(\|(start, len)\| &COMPATIBILITY_DECOMPOSED_CHARS[start as usize..][..len as usize])`
`65`	`67`	`}`
`66`	`68`
`67`	`69`	`pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [char]> {`
`@@ -73,6 +75,7 @@ pub(crate) fn cjk_compat_variants_fully_decomposed(c: char) -> Option<&'static [`
`73`	`75`	`pair_lookup_fv_opt,`
`74`	`76`	`None,`
`75`	`77`	`)`
	`78`	`+ .map(\|(start, len)\| &CJK_COMPAT_VARIANTS_DECOMPOSED_CHARS[start as usize..][..len as usize])`
`76`	`79`	`}`
`77`	`80`
`78`	`81`	/// Return whether the given character is a combining mark (`General_Category=Mark`)