Unicode 16: Initial support

Jules-Bertholet · Jules-Bertholet · commit 6ab41d793acd · 2025-01-14T21:50:02.000-05:00
Includes Kirat Rai normalization behavior.
diff --git a/scripts/unicode.py b/scripts/unicode.py
@@ -43,7 +43,7 @@
 from itertools import batched
 from typing import Callable, Iterable
 
-UNICODE_VERSION = "15.1.0"
+UNICODE_VERSION = "16.0.0"
 """The version of the Unicode data files to download."""
 
 NUM_CODEPOINTS = 0x110000
@@ -264,6 +264,12 @@ class WidthState(enum.IntEnum):
     TAG_A6_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1110
     "(\\uE0061..=\\uE007A){6} \\uE007F \\u200D `Emoji_Presentation`"
 
+    # Kirat Rai
+    KIRAT_RAI_VOWEL_SIGN_E = 0b0000_0000_0010_0000
+    "\\u16D67 (\\u16D67 \\u16D67)+ and canonical equivalents"
+    KIRAT_RAI_VOWEL_SIGN_AI = 0b0000_0000_0010_0001
+    "(\\u16D68)+ and canonical equivalents"
+
     # VARIATION SELECTORS
 
     # Text presentation sequences (not CJK)
@@ -639,6 +645,8 @@ def load_width_maps() -> tuple[list[WidthState], list[WidthState]]:
         ([0xA4FD], WidthState.LISU_TONE_LETTER_MYA_NA_JEU),
         ([0xFE0F], WidthState.VARIATION_SELECTOR_16),
         ([0x10C03], WidthState.OLD_TURKIC_LETTER_ORKHON_I),
+        ([0x16D67], WidthState.KIRAT_RAI_VOWEL_SIGN_E),
+        ([0x16D68], WidthState.KIRAT_RAI_VOWEL_SIGN_AI),
         (emoji_presentation, WidthState.EMOJI_PRESENTATION),
         (emoji_modifiers, WidthState.EMOJI_MODIFIER),
         (regional_indicators, WidthState.REGIONAL_INDICATOR),
@@ -1496,6 +1504,22 @@ def lookup_fns(
                     return (0, WidthInfo::EMOJI_PRESENTATION)
                 }}
 
+                (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D63}}') => {{
+                    return (0, WidthInfo::DEFAULT);
+                }}
+                (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D67}}') => {{
+                    return (0, WidthInfo::KIRAT_RAI_VOWEL_SIGN_AI);
+                }}
+                (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D68}}') => {{
+                    return (1, WidthInfo::KIRAT_RAI_VOWEL_SIGN_E);
+                }}
+                (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D69}}') => {{
+                    return (0, WidthInfo::DEFAULT);
+                }}
+                (WidthInfo::KIRAT_RAI_VOWEL_SIGN_AI, '\\u{{16D63}}') => {{
+                    return (0, WidthInfo::DEFAULT);
+                }}
+
                 // Fallback
                 _ => {{}}
             }}
diff --git a/src/lib.rs b/src/lib.rs
@@ -65,7 +65,7 @@
 //!    - Script-specific ligatures:
 //!      - For all the following ligatures, the insertion of any number of [default-ignorable][`Default_Ignorable_Code_Point`]
 //!        [combining marks] anywhere in the sequence will not change the total width. In addition, for all non-Arabic
-//!        ligatures, the insertion of any number of [`'\u{200D}'` ZERO WIDTH JOINER](https://www.unicode.org/versions/Unicode15.0.0/ch23.pdf#G23126)s
+//!        ligatures, the insertion of any number of [`'\u{200D}'` ZERO WIDTH JOINER](https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-23/#G23126)s
 //!        will not affect the width.
 //!      - **[Arabic]**: A character sequence consisting of one character with [`Joining_Group`]`=Lam`,
 //!        followed by any number of characters with [`Joining_Type`]`=Transparent`, followed by one character
@@ -75,6 +75,7 @@
 //!      - **[Khmer]**: Coeng signs consisting of `'\u{17D2}'` followed by a character in
 //!        `'\u{1780}'..='\u{1782}' | '\u{1784}'..='\u{1787}' | '\u{1789}'..='\u{178C}' | '\u{178E}'..='\u{1793}' | '\u{1795}'..='\u{1798}' | '\u{179B}'..='\u{179D}' | '\u{17A0}' | '\u{17A2}'  | '\u{17A7}' | '\u{17AB}'..='\u{17AC}' | '\u{17AF}'`
 //!        have width 0.
+//!      - **[Kirat Rai]**: Any sequence canonically equivalent to `\u{16D68}`, `\u{16D69}`, or `\u{16D6A}` has total width 1.
 //!      - **[Lisu]**: Tone letter combinations consisting of a character in the range `'\u{A4F8}'..='\u{A4FB}'`
 //!        followed by a character in the range `'\u{A4FC}'..='\u{A4FD}'` have width 1. For example: `ꓹꓼ`
 //!      - **[Old Turkic]**: `"\u{10C32}\u{200D}\u{10C03}"` (`𐰲‍𐰃`) has total width 1.
@@ -130,18 +131,18 @@
 //! [`'\u{0338}'` COMBINING LONG SOLIDUS OVERLAY]: https://util.unicode.org/UnicodeJsps/character.jsp?a=0338
 //! [`'\u{2D7F}'` TIFINAGH CONSONANT JOINER]: https://util.unicode.org/UnicodeJsps/character.jsp?a=2D7F
 //!
-//! [`Canonical_Combining_Class`]: https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf#G50313
-//! [`Default_Ignorable_Code_Point`]: https://www.unicode.org/versions/Unicode15.0.0/ch05.pdf#G40095
+//! [`Canonical_Combining_Class`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G50313
+//! [`Default_Ignorable_Code_Point`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-5/#G40095
 //! [`East_Asian_Width`]: https://www.unicode.org/reports/tr11/#ED1
 //! [`Emoji_Presentation`]: https://unicode.org/reports/tr51/#def_emoji_presentation
-//! [`General_Category`]: https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G124142
+//! [`General_Category`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-4/#G124142
 //! [`Grapheme_Extend=Prepend`]: https://www.unicode.org/reports/tr29/#Prepend
-//! [`Grapheme_Extend`]: https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf#G52443
-//! [`Hangul_Syllable_Type`]: https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf#G45593
+//! [`Grapheme_Extend`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G52443
+//! [`Hangul_Syllable_Type`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G45593
 //! [`Joining_Group`]: https://www.unicode.org/versions/Unicode14.0.0/ch09.pdf#G36862
-//! [`Joining_Type`]: http://www.unicode.org/versions/Unicode15.0.0/ch09.pdf#G50009
+//! [`Joining_Type`]: http://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-9/#G50009
 //! [`Line_Break`]: https://www.unicode.org/reports/tr14/#LD5
-//! [`Prepended_Concatenation_Mark`]: https://www.unicode.org/versions/Unicode15.0.0/ch23.pdf#G37908
+//! [`Prepended_Concatenation_Mark`]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-23/#G37908
 //! [`Script`]: https://www.unicode.org/reports/tr24/#Script
 //!
 //! [`Fullwidth`]: https://www.unicode.org/reports/tr11/#ED2
@@ -150,7 +151,7 @@
 //!
 //! [`AI`]: https://www.unicode.org/reports/tr14/#AI
 //!
-//! [combining marks]: https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf#G30602
+//! [combining marks]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G30602
 //!
 //! [emoji ZWJ sequences]: https://www.unicode.org/reports/tr51/#def_emoji_sequence
 //! [Emoji modifier sequences]: https://www.unicode.org/reports/tr51/#def_emoji_modifier_sequence
@@ -159,13 +160,14 @@
 //!
 //! [Enclosed Ideographic Supplement]: https://unicode.org/charts/nameslist/n_1F200.html
 //!
-//! [Arabic]: https://www.unicode.org/versions/Unicode15.0.0/ch09.pdf#G7480
-//! [Buginese]: https://www.unicode.org/versions/Unicode15.0.0/ch17.pdf#G26743
-//! [Hebrew]: https://www.unicode.org/versions/Unicode15.0.0/ch09.pdf#G6528
-//! [Khmer]: https://www.unicode.org/versions/Unicode15.0.0/ch16.pdf#G64642
-//! [Lisu]: https://www.unicode.org/versions/Unicode15.0.0/ch18.pdf#G44587
-//! [Old Turkic]: https://www.unicode.org/versions/Unicode15.0.0/ch14.pdf#G41975
-//! [Tifinagh]: http://www.unicode.org/versions/Unicode15.0.0/ch19.pdf#G43184
+//! [Arabic]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-9/#G7480
+//! [Buginese]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-17/#G26743
+//! [Hebrew]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-9/#G6528
+//! [Khmer]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-16/#G64642
+//! [Kirat Rai]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-13/#G746409
+//! [Lisu]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-18/#G44587
+//! [Old Turkic]: https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-14/#G41975
+//! [Tifinagh]: http://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-19/#G43184
 //!
 //!
 //! ## Canonical equivalence
diff --git a/src/tables.rs b/src/tables.rs
