8000 gh-118235: Move RAISE_SYNTAX_ERROR actions to invalid rules and make … · python/cpython@48f21b3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 48f21b3

Browse files
encukouencukou
authored
gh-118235: Move RAISE_SYNTAX_ERROR actions to invalid rules and make sure they stay there (GH-119731)
The Full Grammar specification in the docs omits rule actions, so grammar rules that raise a syntax error looked like valid syntax. This was solved in ef940de by hiding those rules in the custom syntax highlighter. This moves all syntax-error alternatives to invalid rules, adds a validator th 8000 at ensures that actions containing RAISE_SYNTAX_ERROR are in invalid rules, and reverts the syntax highlighter hack.
1 parent a5fef80 commit 48f21b3

File tree

5 files changed

+1584
-1434
lines changed

5 files changed

+1584
-1434
lines changed

‎Doc/tools/extensions/peg_highlight.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ class PEGLexer(RegexLexer):
1616
- Rule types
1717
- Rule options
1818
- Rules named `invalid_*` or `incorrect_*`
19-
- Rules with `RAISE_SYNTAX_ERROR`
2019
"""
2120

2221
name = "PEG"
@@ -60,7 +59,6 @@ class PEGLexer(RegexLexer):
6059
(r"^(\s+\|\s+.*invalid_\w+.*\n)", bygroups(None)),
6160
(r"^(\s+\|\s+.*incorrect_\w+.*\n)", bygroups(None)),
6261
(r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),),
63-
(r"^(\s+\|\s+.*\{[^}]*RAISE_SYNTAX_ERROR[^}]*\})\n", bygroups(None)),
6462
],
6563
"root": [
6664
include("invalids"),

‎Grammar/python.gram

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -650,17 +650,8 @@ type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ ['
650650

651651
type_param[type_param_ty] (memo):
652652
| a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) }
653-
| '*' a=NAME colon=':' e=expression {
654-
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
655-
? "cannot use constraints with TypeVarTuple"
656-
: "cannot use bound with TypeVarTuple")
657-
}
653+
| invalid_type_param
658654
| '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) }
659-
| '**' a=NAME colon=':' e=expression {
660-
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
661-
? "cannot use constraints with ParamSpec"
662-
: "cannot use bound with ParamSpec")
663-
}
664655
| '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) }
665656

666657
type_param_bound[expr_ty]: ':' e=expression { e }
@@ -979,8 +970,7 @@ for_if_clause[comprehension_ty]:
979970
CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) }
980971
| 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
981972
_PyAST_comprehension(a, b, c, 0, p->arena) }
982-
| 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' {
983-
RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") }
973+
| invalid_for_if_clause
984974
| invalid_for_target
985975

986976
listcomp[expr_ty]:
@@ -1020,9 +1010,9 @@ kwargs[asdl_seq*]:
10201010
| ','.kwarg_or_double_starred+
10211011

10221012
starred_expression[expr_ty]:
1023-
| invalid_starred_expression
1013+
| invalid_starred_expression_unpacking
10241014
| '*' a=expression { _PyAST_Starred(a, Load, EXTRA) }
1025-
| '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
1015+
| invalid_starred_expression
10261016

10271017
kwarg_or_starred[KeywordOrStarred*]:
10281018
| invalid_kwarg
@@ -1176,6 +1166,18 @@ invalid_legacy_expression:
11761166
_PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b,
11771167
"Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL}
11781168

1169+
invalid_type_param:
1170+
| '*' a=NAME colon=':' e=expression {
1171+
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
1172+
? "cannot use constraints with TypeVarTuple"
1173+
: "cannot use bound with TypeVarTuple")
1174+
}
1175+
| '**' a=NAME colon=':' e=expression {
1176+
RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
1177+
? "cannot use constraints with ParamSpec"
1178+
: "cannot use bound with ParamSpec")
1179+
}
1180+
11791181
invalid_expression:
11801182
# !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
11811183
# Soft keywords need to also be ignored because they can be parsed as NAME NAME
@@ -1296,6 +1298,10 @@ invalid_with_item:
12961298
| expression 'as' a=expression &(',' | ')' | ':') {
12971299
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
12981300

1301+
invalid_for_if_clause:
1302+
| 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' {
1303+
RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") }
1304+
12991305
invalid_for_target:
13001306
| 'async'? 'for' a=star_expressions {
13011307
RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) }
@@ -1409,8 +1415,10 @@ invalid_kvpair:
14091415
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") }
14101416
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") }
14111417
| expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
1412-
invalid_starred_expression:
1418+
invalid_starred_expression_unpacking:
14131419
| a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") }
1420+
invalid_starred_expression:
1421+
| '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
14141422

14151423
invalid_replacement_field:
14161424
| '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") }

‎Lib/test/test_peg_generator/test_grammar_validator.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
test_tools.skip_if_missing("peg_generator")
55
with test_tools.imports_under_tool("peg_generator"):
66
from pegen.grammar_parser import GeneratedParser as GrammarParser
7-
from pegen.validator import SubRuleValidator, ValidationError
7+
from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator
88
from pegen.testutil import parse_string
99
from pegen.grammar import Grammar
1010

@@ -49,3 +49,13 @@ def test_rule_with_collision_after_some_other_rules(self) -> None:
4949
with self.assertRaises(ValidationError):
5050
for rule_name, rule in grammar.rules.items():
5151
validator.validate_rule(rule_name, rule)
52+
53+
def test_raising_valid_rule(self) -> None:
54+
grammar_source = """
55+
start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") }
56+
"""
57+
grammar: Grammar = parse_string(grammar_source, GrammarParser)
58+
validator = RaiseRuleValidator(grammar)
59+
with self.assertRaises(ValidationError):
60+
for rule_name, rule in grammar.rules.items():
61+
validator.validate_rule(rule_name, rule)

0 commit comments

Comments
 (0)
0