gh-133541: Handle SyntaxError raised by the tokenizer on user input (#133606)

ambv · web-flow · commit b2fabce6abb2 · 2025-05-08T21:14:38.000+01:00
diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py
@@ -102,6 +102,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
         for color in gen_colors_from_token_stream(gen, line_lengths):
             yield color
             last_emitted = color
+    except SyntaxError:
+        return
     except tokenize.TokenError as te:
         yield from recover_unterminated_string(
             te, line_lengths, last_emitted, buffer
diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py
@@ -497,6 +497,26 @@ def unfinished_function():
         self.assert_screen_equal(reader, code, clean=True)
         self.assert_screen_equal(reader, expected)
 
+    def test_syntax_highlighting_indentation_error(self):
+        code = dedent(
+            """\
+            def unfinished_function():
+                var = 1
+               oops
+            """
+        )
+        expected = dedent(
+            """\
+            {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z}
+                var {o}={z} {n}1{z}
+               oops
+            """
+        ).format(**colors)
+        events = code_to_events(code)
+        reader, _ = handle_all_events(events)
+        self.assert_screen_equal(reader, code, clean=True)
+        self.assert_screen_equal(reader, expected)
+
     def test_control_characters(self):
         code = 'flag = "🏳️‍🌈"'
         events = code_to_events(code)
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst
@@ -0,0 +1,2 @@
+Inconsistent indentation in user input crashed the new REPL when syntax
+highlighting was active. This is now fixed.

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Inconsistent indentation in user input crashed the new REPL when syntax`
	`2`	`+highlighting was active. This is now fixed.`