8000 Reset the stream-safe position when a starter is seen. · emilio/unicode-normalization@91a8493 · GitHub
[go: up one dir, main page]
Skip to content

Commit 91a8493

Browse files
sunfishcodesunfishcode
committed
Reset the stream-safe position when a starter is seen.
This avoids inserting too many CGJs.
1 parent 5967481 commit 91a8493

File tree

1 file changed

+7
-2
lines changed

1 file changed

+7
-2
lines changed

‎src/stream_safe.rs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ impl<I: Iterator<Item = char>> Iterator for StreamSafe<I> {
4242
let d = classify_nonstarters(next_ch);
4343
if self.nonstarter_count + d.leading_nonstarters > MAX_NONSTARTERS {
4444
self.buffer = Some(next_ch);
45-
self.nonstarter_count += d.decomposition_len;
45+
self.nonstarter_count = d.decomposition_len;
4646
return Some(COMBINING_GRAPHEME_JOINER);
4747
}
4848

@@ -123,7 +123,12 @@ mod tests {
123123
assert_eq!(stream_safe(technically_okay), technically_okay);
124124

125125
let too_much = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone";
126-
assert_ne!(stream_safe(too_much), too_much);
126+
let fixed_it = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{034f}\u{032e}ngerzone";
127+
assert_eq!(stream_safe(too_much), fixed_it);
128+
129+
let woah_nelly = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone";
130+
let its_cool = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{034f}\u{032e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{034f}\u{031d}\u{032e}ngerzone";
131+
assert_eq!(stream_safe(woah_nelly), its_cool);
127132
}
128133

129134
#[test]

0 commit comments

Comments
 (0)
0