8000 gh-105042: Disable unmatched parens syntax error in python tokenize (… · python/cpython@70f315c · GitHub
[go: up one dir, main page]
Skip to content

Commit 70f315c

Browse files
lysnikolaoulysnikolaou
authored
gh-105042: Disable unmatched parens syntax error in python tokenize (#105061)
1 parent 9216e69 commit 70f315c

File tree

5 files changed

+49
-34
lines changed
  • test_tokenize.py
  • Parser
  • Python

    • 5 files changed

    +49
    -34
    lines changed

    ‎Lib/test/inspect_fodder.py

    Lines changed: 5 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -113,3 +113,8 @@ async def asyncf(self):
    113113
    # after asyncf - line 113
    114114
    # end of WhichComments - line 114
    115115
    # after WhichComments - line 115
    116+
    117+
    # Test that getsource works on a line that includes
    118+
    # a closing parenthesis with the opening paren being in another line
    119+
    (
    120+
    ); after_closing = lambda: 1

    ‎Lib/test/test_inspect.py

    Lines changed: 3 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -557,7 +557,8 @@ def test_getclasses(self):
    557557

    558558
    def test_getfunctions(self):
    559559
    functions = inspect.getmembers(mod, inspect.isfunction)
    560-
    self.assertEqual(functions, [('eggs', mod.eggs),
    560+
    self.assertEqual(functions, [('after_closing', mod.after_closing),
    561+
    ('eggs', mod.eggs),
    561562
    ('lobbest', mod.lobbest),
    562563
    ('spam', mod.spam)])
    563564

    @@ -641,6 +642,7 @@ def test_getsource(self):
    641642
    self.assertSourceEqual(git.abuse, 29, 39)
    642643
    self.assertSourceEqual(mod.StupidGit, 21, 51)
    643644
    self.assertSourceEqual(mod.lobbest, 75, 76)
    645+
    self.assertSourceEqual(mod.after_closing, 120, 120)
    644646

    645647
    def test_getsourcefile(self):
    646648
    self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile)

    ‎Lib/test/test_tokenize.py

    Lines changed: 7 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -1119,6 +1119,13 @@ def test_newline_after_parenthesized_block_with_comment(self):
    11191119
    NEWLINE '\\n' (4, 1) (4, 2)
    11201120
    """)
    11211121

    1122+
    def test_closing_parenthesis_from_different_line(self):
    1123+
    self.check_tokenize("); x", """\
    1124+
    OP ')' (1, 0) (1, 1)
    1125+
    OP ';' (1, 1) (1, 2)
    1126+
    NAME 'x' (1, 3) (1, 4)
    1127+
    """)
    1128+
    11221129
    class GenerateTokensTest(TokenizeTest):
    11231130
    def check_tokenize(self, s, expected):
    11241131
    # Format the tokens in s in a table format.

    ‎Parser/tokenizer.c

    Lines changed: 33 additions & 32 deletions
    Original file line numberDiff line numberDiff line change
    @@ -2626,41 +2626,42 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
    26262626
    case ')':
    26272627
    case ']':
    26282628
    case '}':
    2629-
    if (!tok->level) {
    2630-
    if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
    2631-
    return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
    2632-
    }
    2629+
    if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
    2630+
    return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
    2631+
    }
    2632+
    if (!tok->tok_extra_tokens && !tok->level) {
    26332633
    return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c));
    26342634
    }
    2635-
    tok->level--;
    2636-
    int opening = tok->parenstack[tok->level];
    2637-
    if (!((opening == '(' && c == ')') ||
    2638-
    (opening == '[' && c == ']') ||
    2639-
    (opening == '{' && c == '}')))
    2640-
    {
    2641-
    /* If the opening bracket belongs to an f-string's expression
    2642-
    part (e.g. f"{)}") and the closing bracket is an arbitrary
    2643-
    nested expression, then instead of matching a different
    2644-
    syntactical construct with it; we'll throw an unmatched
    2645-
    parentheses error. */
    2646-
    if (INSIDE_FSTRING(tok) && opening == '{') {
    2647-
    assert(current_tok->curly_bracket_depth >= 0);
    2648-
    int previous_bracket = current_tok->curly_bracket_depth - 1;
    2649-
    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
    2650-
    return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
    2635+
    if (tok->level > 0) {
    2636+
    tok->level--;
    2637+
    int opening = tok->parenstack[tok->level];
    2638+
    if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
    2639+
    (opening == '[' && c == ']') ||
    2640+
    (opening == '{' && c == '}'))) {
    2641+
    /* If the opening bracket belongs to an f-string's expression
    2642+
    part (e.g. f"{)}") and the closing bracket is an arbitrary
    2643+
    nested expression, then instead of matching a different
    2644+
    syntactical construct with it; we'll throw an unmatched
    2645+
    parentheses error. */
    2646+
    if (INSIDE_FSTRING(tok) && opening == '{') {
    2647+
    assert(current_tok->curly_bracket_depth >= 0);
    2648+
    int previous_bracket = current_tok->curly_bracket_depth - 1;
    2649+
    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
    2650+
    return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
    2651+
    }
    2652+
    }
    2653+
    if (tok->parenlinenostack[tok->level] != tok->lineno) {
    2654+
    return MAKE_TOKEN(syntaxerror(tok,
    2655+
    "closing parenthesis '%c' does not match "
    2656+
    "opening parenthesis '%c' on line %d",
    2657+
    c, opening, tok->parenlinenostack[tok->level]));
    2658+
    }
    2659+
    else {
    2660+
    return MAKE_TOKEN(syntaxerror(tok,
    2661+
    "closing parenthesis '%c' does not match "
    2662+
    "opening parenthesis '%c'",
    2663+
    c, opening));
    26512664
    }
    2652-
    }
    2653-
    if (tok->parenlinenostack[tok->level] != tok->lineno) {
    2654-
    return MAKE_TOKEN(syntaxerror(tok,
    2655-
    "closing parenthesis '%c' does not match "
    2656-
    "opening parenthesis '%c' on line %d",
    2657-
    c, opening, tok->parenlinenostack[tok->level]));
    2658-
    }
    2659-
    else {
    2660-
    return MAKE_TOKEN(syntaxerror(tok,
    2661-
    "closing parenthesis '%c' does not match "
    2662-
    "opening parenthesis '%c'",
    2663-
    c, opening));
    26642665
    }
    26652666
    }
    26662667

    ‎Python/Python-tokenize.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -84,7 +84,7 @@ _tokenizer_error(struct tok_state *tok)
    8484
    msg = "invalid token";
    8585
    break;
    8686
    case E_EOF:
    87-
    if (tok->level) {
    87+
    if (tok->level > 0) {
    8888
    PyErr_Format(PyExc_SyntaxError,
    8989
    "parenthesis '%c' was never closed",
    9090
    tok->parenstack[tok->level-1]);

    0 commit comments

    Comments
     (0)
    0