From 431b9280387329c17c6aeb38a0ab57699d345862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 7 May 2025 21:14:40 +0200 Subject: [PATCH 1/2] gh-133541: Handle SyntaxError raised by the tokenizer on user input --- Lib/_pyrepl/utils.py | 2 ++ Lib/test/test_pyrepl/test_reader.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 38cf6b5a08e892..752049ac05acdf 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -102,6 +102,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]: for color in gen_colors_from_token_stream(gen, line_lengths): yield color last_emitted = color + except SyntaxError: + return except tokenize.TokenError as te: yield from recover_unterminated_string( te, line_lengths, last_emitted, buffer diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 4ee320a5a4dabb..57526f88f9384b 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -497,6 +497,26 @@ def unfinished_function(): self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected) + def test_syntax_highlighting_indentation_error(self): + code = dedent( + """\ + def unfinished_function(): + var = 1 + oops + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + var {o}={z} {n}1{z} + oops + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + def test_control_characters(self): code = 'flag = "🏳️‍🌈"' events = code_to_events(code) From d2cd3c6a155d53c4473b19ff9b0d9ec7cb46198a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 7 May 2025 23:26:57 +0200 Subject: [PATCH 2/2] Add blurb --- .../2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst new file mode 100644 index 00000000000000..4f4cd847fa51fb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst @@ -0,0 +1,2 @@ +Inconsistent indentation in user input crashed the new REPL when syntax +highlighting was active. This is now fixed.