8000 gh-105042: Disable unmatched parens syntax error in python tokenize (… · python/cpython@70f315c · GitHub
[go: up one dir, main page]
Skip to content

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 70f315c

Browse files
lysnikolaoulysnikolaou
authored
gh-105042: Disable unmatched parens syntax error in python tokenize (#105061)
1 parent 9216e69 commit 70f315c

File tree

5 files changed

+49
-34
lines changed

5 files changed

+49
-34
lines changed

‎Lib/test/inspect_fodder.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,8 @@ async def asyncf(self):
113113
# after asyncf - line 113
114114
# end of WhichComments - line 114
115115
# after WhichComments - line 115
116+
117+
# Test that getsource works on a line that includes
118+
# a closing parenthesis with the opening paren being in another line
119+
(
120+
); after_closing = lambda: 1

‎Lib/test/test_inspect.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,8 @@ def test_getclasses(self):
557557

558558
def test_getfunctions(self):
559559
functions = inspect.getmembers(mod, inspect.isfunction)
560-
self.assertEqual(functions, [('eggs', mod.eggs),
560+
self.assertEqual(functions, [('after_closing', mod.after_closing),
561+
('eggs', mod.eggs),
561562
('lobbest', mod.lobbest),
562563
('spam', mod.spam)])
563564

@@ -641,6 +642,7 @@ def test_getsource(self):
641642
self.assertSourceEqual(git.abuse, 29, 39)
642643
self.assertSourceEqual(mod.StupidGit, 21, 51)
643644
self.assertSourceEqual(mod.lobbest, 75, 76)
645+
self.assertSourceEqual(mod.after_closing, 120, 120)
644646

645647
def test_getsourcefile(self):
646648
self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile)

‎Lib/test/test_tokenize.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,6 +1119,13 @@ def test_newline_after_parenthesized_block_with_comment(self):
11191119
NEWLINE '\\n' (4, 1) (4, 2)
11201120
""")
11211121

1122+
def test_closing_parenthesis_from_different_line(self):
1123+
self.check_tokenize("); x", """\
1124+
OP ')' (1, 0) (1, 1)
1125+
OP ';' (1, 1) (1, 2)
1126+
NAME 'x' (1, 3) (1, 4)
1127+
""")
1128+
11221129
class GenerateTokensTest(TokenizeTest):
11231130
def check_tokenize(self, s, expected):
11241131
# Format the tokens in s in a table format.

‎Parser/tokenizer.c‎

Lines changed: 33 additions & 32 deletions
Original file line numberDi 7440 ff line numberDiff line change
@@ -2626,41 +2626,42 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
26262626
case ')':
26272627
case ']':
26282628
case '}':
2629-
if (!tok->level) {
2630-
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
2631-
return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
2632-
}
2629+
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
2630+
return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
2631+
}
2632+
if (!tok->tok_extra_tokens && !tok->level) {
26332633
return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c));
26342634
}
2635-
tok->level--;
2636-
int opening = tok->parenstack[tok->level];
2637-
if (!((opening == '(' && c == ')') ||
2638-
(opening == '[' && c == ']') ||
2639 ED4F -
(opening == '{' && c == '}')))
2640-
{
2641-
/* If the opening bracket belongs to an f-string's expression
2642-
part (e.g. f"{)}") and the closing bracket is an arbitrary
2643-
nested expression, then instead of matching a different
2644-
syntactical construct with it; we'll throw an unmatched
2645-
parentheses error. */
2646-
if (INSIDE_FSTRING(tok) && opening == '{') {
2647-
assert(current_tok->curly_bracket_depth >= 0);
2648-
int previous_bracket = current_tok->curly_bracket_depth - 1;
2649-
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
2650-
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
2635+
if (tok->level > 0) {
2636+
tok->level--;
2637+
int opening = tok->parenstack[tok->level];
2638+
if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
2639+
(opening == '[' && c == ']') ||
2640+
(opening == '{' && c == '}'))) {
2641+
/* If the opening bracket belongs to an f-string's expression
2642+
part (e.g. f"{)}") and the closing bracket is an arbitrary
2643+
nested expression, then instead of matching a different
2644+
syntactical construct with it; we'll throw an unmatched
2645+
parentheses error. */
2646+
if (INSIDE_FSTRING(tok) && opening == '{') {
2647+
assert(current_tok->curly_bracket_depth >= 0);
2648+
int previous_bracket = current_tok->curly_bracket_depth - 1;
2649+
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
2650+
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
2651+
}
2652+
}
2653+
if (tok->parenlinenostack[tok->level] != tok->lineno) {
2654+
return MAKE_TOKEN(syntaxerror(tok,
2655+
"closing parenthesis '%c' does not match "
2656+
"opening parenthesis '%c' on line %d",
2657+
c, opening, tok->parenlinenostack[tok->level]));
2658+
}
2659+
else {
2660+
return MAKE_TOKEN(syntaxerror(tok,
2661+
"closing parenthesis '%c' does not match "
2662+
"opening parenthesis '%c'",
2663+
c, opening));
26512664
}
2652-
}
2653-
if (tok->parenlinenostack[tok->level] != tok->lineno) {
2654-
return MAKE_TOKEN(syntaxerror(tok,
2655-
"closing parenthesis '%c' does not match "
2656-
"opening parenthesis '%c' on line %d",
2657-
c, opening, tok->parenlinenostack[tok->level]));
2658-
}
2659-
else {
2660-
return MAKE_TOKEN(syntaxerror(tok,
2661-
"closing parenthesis '%c' does not match "
2662-
"opening parenthesis '%c'",
2663-
c, opening));
26642665
}
26652666
}
26662667

‎Python/Python-tokenize.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ _tokenizer_error(struct tok_state *tok)
8484
msg = "invalid token";
8585
break;
8686
case E_EOF:
87-
if (tok->level) {
87+
if (tok->level > 0) {
8888
PyErr_Format(PyExc_SyntaxError,
8989
"parenthesis '%c' was never closed",
9090
tok->parenstack[tok->level-1]);

0 commit comments

Comments
 (0)
0