Merge pull request #62 from unicode-rs/streamsafe-reset

Manishearth · web-flow · commit c243feeb7a57 · 2020-11-17T19:43:35.000-08:00
Correctly reset streamsafe iterator
diff --git a/src/stream_safe.rs b/src/stream_safe.rs
@@ -42,7 +42,7 @@ impl<I: Iterator<Item = char>> Iterator for StreamSafe<I> {
         let d = classify_nonstarters(next_ch);
         if self.nonstarter_count + d.leading_nonstarters > MAX_NONSTARTERS {
             self.buffer = Some(next_ch);
-            self.nonstarter_count = 0;
+            self.nonstarter_count += d.decomposition_len;
             return Some(COMBINING_GRAPHEME_JOINER);
         }
 
diff --git a/tests/test_streamsafe_regression.rs b/tests/test_streamsafe_regression.rs
@@ -0,0 +1,14 @@
+use unicode_normalization::{
+    char::canonical_combining_class, is_nfc, is_nfc_stream_safe, UnicodeNormalization,
+};
+
+#[test]
+fn test_streamsafe_regression(){
+    let input = "\u{342}".repeat(55) + &"\u{344}".repeat(3);
+    let nfc_ss = input.chars().nfc().stream_safe().collect::<String>();
+
+    // The result should be NFC:
+    assert!(is_nfc(&nfc_ss));
+    // and should be stream-safe:
+    assert!(is_nfc_stream_safe(&nfc_ss))
+}

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ impl<I: Iterator<Item = char>> Iterator for StreamSafe<I> {`
`42`	`42`	`let d = classify_nonstarters(next_ch);`
`43`	`43`	`if self.nonstarter_count + d.leading_nonstarters > MAX_NONSTARTERS {`
`44`	`44`	`self.buffer = Some(next_ch);`
`45`		`- self.nonstarter_count = 0;`
	`45`	`+ self.nonstarter_count += d.decomposition_len;`
`46`	`46`	`return Some(COMBINING_GRAPHEME_JOINER);`
`47`	`47`	`}`
`48`	`48`