diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 1dcbaad1c38870..92d396865b643f 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -176,7 +176,7 @@ def main() -> None: if __name__ == "__main__": - if sys.version_info < (3, 8): - print("ERROR: using pegen requires at least Python 3.8!", file=sys.stderr) + if sys.version_info < (3, 6): + print("ERROR: using pegen requires at least Python 3.6!", file=sys.stderr) sys.exit(1) main() diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index 67039d5a032abd..dbb232a46f20b5 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from abc import abstractmethod from typing import ( AbstractSet, @@ -45,7 +43,7 @@ def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> None: class Grammar: - def __init__(self, rules: Iterable[Rule], metas: Iterable[Tuple[str, Optional[str]]]): + def __init__(self, rules: Iterable['Rule'], metas: Iterable[Tuple[str, Optional[str]]]): self.rules = {rule.name: rule for rule in rules} self.metas = dict(metas) @@ -62,7 +60,7 @@ def __repr__(self) -> str: lines.append(")") return "\n".join(lines) - def __iter__(self) -> Iterator[Rule]: + def __iter__(self) -> Iterator['Rule']: yield from self.rules.values() @@ -71,7 +69,7 @@ def __iter__(self) -> Iterator[Rule]: class Rule: - def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[object] = None): + def __init__(self, name: str, type: Optional[str], rhs: 'Rhs', memo: Optional[object] = None): self.name = name self.type = type self.rhs = rhs @@ -101,10 +99,10 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Rule({self.name!r}, {self.type!r}, {self.rhs!r})" - def __iter__(self) -> Iterator[Rhs]: + def __iter__(self) -> Iterator['Rhs']: yield self.rhs - def nullable_visit(self, rules: Dict[str, Rule]) -> bool: + def nullable_visit(self, rules: Dict[str, 'Rule']) -> bool: if self.visited: # A left-recursive rule is considered non-nullable. return False @@ -115,7 +113,7 @@ def nullable_visit(self, rules: Dict[str, Rule]) -> bool: def initial_names(self) -> AbstractSet[str]: return self.rhs.initial_names() - def flatten(self) -> Rhs: + def flatten(self) -> 'Rhs': # If it's a single parenthesized group, flatten it. rhs = self.rhs if ( @@ -127,7 +125,7 @@ def flatten(self) -> Rhs: rhs = rhs.alts[0].items[0].item.rhs return rhs - def collect_todo(self, gen: ParserGenerator) -> None: + def collect_todo(self, gen: 'ParserGenerator') -> None: rhs = self.flatten() rhs.collect_todo(gen) @@ -188,7 +186,7 @@ def initial_names(self) -> AbstractSet[str]: class Rhs: - def __init__(self, alts: List[Alt]): + def __init__(self, alts: List['Alt']): self.alts = alts self.memo: Optional[Tuple[Optional[str], str]] = None @@ -198,7 +196,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Rhs({self.alts!r})" - def __iter__(self) -> Iterator[List[Alt]]: + def __iter__(self) -> Iterator[List['Alt']]: yield self.alts def nullable_visit(self, rules: Dict[str, Rule]) -> bool: @@ -213,13 +211,13 @@ def initial_names(self) -> AbstractSet[str]: names |= alt.initial_names() return names - def collect_todo(self, gen: ParserGenerator) -> None: + def collect_todo(self, gen: 'ParserGenerator') -> None: for alt in self.alts: alt.collect_todo(gen) class Alt: - def __init__(self, items: List[NamedItem], *, icut: int = -1, action: Optional[str] = None): + def __init__(self, items: List['NamedItem'], *, icut: int = -1, action: Optional[str] = None): self.items = items self.icut = icut self.action = action @@ -239,7 +237,7 @@ def __repr__(self) -> str: args.append(f"action={self.action!r}") return f"Alt({', '.join(args)})" - def __iter__(self) -> Iterator[List[NamedItem]]: + def __iter__(self) -> Iterator[List['NamedItem']]: yield self.items def nullable_visit(self, rules: Dict[str, Rule]) -> bool: @@ -256,13 +254,13 @@ def initial_names(self) -> AbstractSet[str]: break return names - def collect_todo(self, gen: ParserGenerator) -> None: + def collect_todo(self, gen: 'ParserGenerator') -> None: for item in self.items: item.collect_todo(gen) class NamedItem: - def __init__(self, name: Optional[str], item: Item): + def __init__(self, name: Optional[str], item: 'Item'): self.name = name self.item = item self.nullable = False @@ -276,7 +274,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"NamedItem({self.name!r}, {self.item!r})" - def __iter__(self) -> Iterator[Item]: + def __iter__(self) -> Iterator['Item']: yield self.item def nullable_visit(self, rules: Dict[str, Rule]) -> bool: @@ -286,19 +284,19 @@ def nullable_visit(self, rules: Dict[str, Rule]) -> bool: def initial_names(self) -> AbstractSet[str]: return self.item.initial_names() - def collect_todo(self, gen: ParserGenerator) -> None: + def collect_todo(self, gen: 'ParserGenerator') -> None: gen.callmakervisitor.visit(self.item) class Lookahead: - def __init__(self, node: Plain, sign: str): + def __init__(self, node: 'Plain', sign: str): self.node = node self.sign = sign def __str__(self) -> str: return f"{self.sign}{self.node}" - def __iter__(self) -> Iterator[Plain]: + def __iter__(self) -> Iterator['Plain']: yield self.node def nullable_visit(self, rules: Dict[str, Rule]) -> bool: @@ -309,7 +307,7 @@ def initial_names(self) -> AbstractSet[str]: class PositiveLookahead(Lookahead): - def __init__(self, node: Plain): + def __init__(self, node: 'Plain'): super().__init__(node, "&") def __repr__(self) -> str: @@ -317,7 +315,7 @@ def __repr__(self) -> str: class NegativeLookahead(Lookahead): - def __init__(self, node: Plain): + def __init__(self, node: 'Plain'): super().__init__(node, "!") def __repr__(self) -> str: @@ -325,7 +323,7 @@ def __repr__(self) -> str: class Opt: - def __init__(self, node: Item): + def __init__(self, node: 'Item'): self.node = node def __str__(self) -> str: @@ -339,7 +337,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Opt({self.node!r})" - def __iter__(self) -> Iterator[Item]: + def __iter__(self) -> Iterator['Item']: yield self.node def nullable_visit(self, rules: Dict[str, Rule]) -> bool: @@ -352,7 +350,7 @@ def initial_names(self) -> AbstractSet[str]: class Repeat: """Shared base class for x* and x+.""" - def __init__(self, node: Plain): + def __init__(self, node: 'Plain'): self.node = node self.memo: Optional[Tuple[Optional[str], str]] = None @@ -360,7 +358,7 @@ def __init__(self, node: Plain): def nullable_visit(self, rules: Dict[str, Rule]) -> bool: raise NotImplementedError - def __iter__(self) -> Iterator[Plain]: + def __iter__(self) -> Iterator['Plain']: yield self.node def initial_names(self) -> AbstractSet[str]: @@ -400,7 +398,7 @@ def nullable_visit(self, rules: Dict[str, Rule]) -> bool: class Gather(Repeat): - def __init__(self, separator: Plain, node: Plain): + def __init__(self, separator: 'Plain', node: 'Plain'): self.separator = separator self.node = node diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index 0e206ee9cd5e44..15d6bf2b727541 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -44,12 +44,11 @@ def start(self) -> Optional[Grammar]: # start: grammar $ mark = self.mark() cut = False - if ( - (grammar := self.grammar()) - and - (endmarker := self.expect('ENDMARKER')) - ): - return grammar + grammar = self.grammar() + if grammar: + endmarker = self.expect('ENDMARKER') + if endmarker: + return grammar self.reset(mark) if cut: return None return None @@ -59,18 +58,16 @@ def grammar(self) -> Optional[Grammar]: # grammar: metas rules | rules mark = self.mark() cut = False - if ( - (metas := self.metas()) - and - (rules := self.rules()) - ): - return Grammar ( rules , metas ) + metas = self.metas() + if metas: + rules = self.rules() + if rules: + return Grammar ( rules , metas ) self.reset(mark) if cut: return None cut = False - if ( - (rules := self.rules()) - ): + rules = self.rules() + if rules: return Grammar ( rules , [ ] ) self.reset(mark) if cut: return None @@ -81,18 +78,16 @@ def metas(self) -> Optional[MetaList]: # metas: meta metas | meta mark = self.mark() cut = False - if ( - (meta := self.meta()) - and - (metas := self.metas()) - ): - return [ meta ] + metas + meta = self.meta() + if meta: + metas = self.metas() + if metas: + return [ meta ] + metas self.reset(mark) if cut: return None cut = False - if ( - (meta := self.meta()) - ): + meta = self.meta() + if meta: return [ meta ] self.reset(mark) if cut: return None @@ -103,40 +98,37 @@ def meta(self) -> Optional[MetaTuple]: # meta: "@" NAME NEWLINE | "@" NAME NAME NEWLINE | "@" NAME STRING NEWLINE mark = self.mark() cut = False - if ( - (literal := self.expect("@")) - and - (name := self.name()) - and - (newline := self.expect('NEWLINE')) - ): - return ( name . string , None ) + literal = self.expect("@") + if literal: + name = self.name() + if name: + newline = self.expect('NEWLINE') + if newline: + return ( name . string , None ) self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect("@")) - and - (a := self.name()) - and - (b := self.name()) - and - (newline := self.expect('NEWLINE')) - ): - return ( a . string , b . string ) + literal = self.expect("@") + if literal: + a = self.name() + if a: + b = self.name() + if b: + newline = self.expect('NEWLINE') + if newline: + return ( a . string , b . string ) self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect("@")) - and - (name := self.name()) - and - (string := self.string()) - and - (newline := self.expect('NEWLINE')) - ): - return ( name . string , literal_eval ( string . string ) ) + literal = self.expect("@") + if literal: + name = self.name() + if name: + string = self.string() + if string: + newline = self.expect('NEWLINE') + if newline: + return ( name . string , literal_eval ( string . string ) ) self.reset(mark) if cut: return None return None @@ -146,18 +138,16 @@ def rules(self) -> Optional[RuleList]: # rules: rule rules | rule mark = self.mark() cut = False - if ( - (rule := self.rule()) - and - (rules := self.rules()) - ): - return [ rule ] + rules + rule = self.rule() + if rule: + rules = self.rules() + if rules: + return [ rule ] + rules self.reset(mark) if cut: return None cut = False - if ( - (rule := self.rule()) - ): + rule = self.rule() + if rule: return [ rule ] self.reset(mark) if cut: return None @@ -168,58 +158,52 @@ def rule(self) -> Optional[Rule]: # rule: rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" alts NEWLINE mark = self.mark() cut = False - if ( - (rulename := self.rulename()) - and - (opt := self.memoflag(),) - and - (literal := self.expect(":")) - and - (alts := self.alts()) - and - (newline := self.expect('NEWLINE')) - and - (indent := self.expect('INDENT')) - and - (more_alts := self.more_alts()) - and - (dedent := self.expect('DEDENT')) - ): - return Rule ( rulename [ 0 ] , rulename [ 1 ] , Rhs ( alts . alts + more_alts . alts ) , memo = opt ) - self.reset(mark) - if cut: return None - cut = False - if ( - (rulename := self.rulename()) - and - (opt := self.memoflag(),) - and - (literal := self.expect(":")) - and - (newline := self.expect('NEWLINE')) - and - (indent := self.expect('INDENT')) - and - (more_alts := self.more_alts()) - and - (dedent := self.expect('DEDENT')) - ): - return Rule ( rulename [ 0 ] , rulename [ 1 ] , more_alts , memo = opt ) - self.reset(mark) - if cut: return None - cut = False - if ( - (rulename := self.rulename()) - and - (opt := self.memoflag(),) - and - (literal := self.expect(":")) - and - (alts := self.alts()) - and - (newline := self.expect('NEWLINE')) - ): - return Rule ( rulename [ 0 ] , rulename [ 1 ] , alts , memo = opt ) + rulename = self.rulename() + if rulename: + opt = self.memoflag() + literal = self.expect(":") + if literal: + alts = self.alts() + if alts: + newline = self.expect('NEWLINE') + if newline: + indent = self.expect('INDENT') + if indent: + more_alts = self.more_alts() + if more_alts: + dedent = self.expect('DEDENT') + if dedent: + return Rule ( rulename [ 0 ] , rulename [ 1 ] , Rhs ( alts . alts + more_alts . alts ) , memo = opt ) + self.reset(mark) + if cut: return None + cut = False + rulename = self.rulename() + if rulename: + opt = self.memoflag() + literal = self.expect(":") + if literal: + newline = self.expect('NEWLINE') + if newline: + indent = self.expect('INDENT') + if indent: + more_alts = self.more_alts() + if more_alts: + dedent = self.expect('DEDENT') + if dedent: + return Rule ( rulename [ 0 ] , rulename [ 1 ] , more_alts , memo = opt ) + self.reset(mark) + if cut: return None + cut = False + rulename = self.rulename() + if rulename: + opt = self.memoflag() + literal = self.expect(":") + if literal: + alts = self.alts() + if alts: + newline = self.expect('NEWLINE') + if newline: + return Rule ( rulename [ 0 ] , rulename [ 1 ] , alts , memo = opt ) self.reset(mark) if cut: return None return None @@ -229,37 +213,34 @@ def rulename(self) -> Optional[RuleName]: # rulename: NAME '[' NAME '*' ']' | NAME '[' NAME ']' | NAME mark = self.mark() cut = False - if ( - (name := self.name()) - and - (literal := self.expect('[')) - and - (type := self.name()) - and - (literal_1 := self.expect('*')) - and - (literal_2 := self.expect(']')) - ): - return ( name . string , type . string + "*" ) - self.reset(mark) - if cut: return None - cut = False - if ( - (name := self.name()) - and - (literal := self.expect('[')) - and - (type := self.name()) - and - (literal_1 := self.expect(']')) - ): - return ( name . string , type . string ) - self.reset(mark) - if cut: return None - cut = False - if ( - (name := self.name()) - ): + name = self.name() + if name: + literal = self.expect('[') + if literal: + type = self.name() + if type: + literal_1 = self.expect('*') + if literal_1: + literal_2 = self.expect(']') + if literal_2: + return ( name . string , type . string + "*" ) + self.reset(mark) + if cut: return None + cut = False + name = self.name() + if name: + literal = self.expect('[') + if literal: + type = self.name() + if type: + literal_1 = self.expect(']') + if literal_1: + return ( name . string , type . string ) + self.reset(mark) + if cut: return None + cut = False + name = self.name() + if name: return ( name . string , None ) self.reset(mark) if cut: return None @@ -270,14 +251,13 @@ def memoflag(self) -> Optional[str]: # memoflag: '(' 'memo' ')' mark = self.mark() cut = False - if ( - (literal := self.expect('(')) - and - (literal_1 := self.expect('memo')) - and - (literal_2 := self.expect(')')) - ): - return "memo" + literal = self.expect('(') + if literal: + literal_1 = self.expect('memo') + if literal_1: + literal_2 = self.expect(')') + if literal_2: + return "memo" self.reset(mark) if cut: return None return None @@ -287,20 +267,18 @@ def alts(self) -> Optional[Rhs]: # alts: alt "|" alts | alt mark = self.mark() cut = False - if ( - (alt := self.alt()) - and - (literal := self.expect("|")) - and - (alts := self.alts()) - ): - return Rhs ( [ alt ] + alts . alts ) + alt = self.alt() + if alt: + literal = self.expect("|") + if literal: + alts = self.alts() + if alts: + return Rhs ( [ alt ] + alts . alts ) self.reset(mark) if cut: return None cut = False - if ( - (alt := self.alt()) - ): + alt = self.alt() + if alt: return Rhs ( [ alt ] ) self.reset(mark) if cut: return None @@ -311,27 +289,25 @@ def more_alts(self) -> Optional[Rhs]: # more_alts: "|" alts NEWLINE more_alts | "|" alts NEWLINE mark = self.mark() cut = False - if ( - (literal := self.expect("|")) - and - (alts := self.alts()) - and - (newline := self.expect('NEWLINE')) - and - (more_alts := self.more_alts()) - ): - return Rhs ( alts . alts + more_alts . alts ) + literal = self.expect("|") + if literal: + alts = self.alts() + if alts: + newline = self.expect('NEWLINE') + if newline: + more_alts = self.more_alts() + if more_alts: + return Rhs ( alts . alts + more_alts . alts ) self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect("|")) - and - (alts := self.alts()) - and - (newline := self.expect('NEWLINE')) - ): - return Rhs ( alts . alts ) + literal = self.expect("|") + if literal: + alts = self.alts() + if alts: + newline = self.expect('NEWLINE') + if newline: + return Rhs ( alts . alts ) self.reset(mark) if cut: return None return None @@ -341,38 +317,34 @@ def alt(self) -> Optional[Alt]: # alt: items '$' action | items '$' | items action | items mark = self.mark() cut = False - if ( - (items := self.items()) - and - (literal := self.expect('$')) - and - (action := self.action()) - ): - return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = action ) + items = self.items() + if items: + literal = self.expect('$') + if literal: + action = self.action() + if action: + return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = action ) self.reset(mark) if cut: return None cut = False - if ( - (items := self.items()) - and - (literal := self.expect('$')) - ): - return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = None ) + items = self.items() + if items: + literal = self.expect('$') + if literal: + return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = None ) self.reset(mark) if cut: return None cut = False - if ( - (items := self.items()) - and - (action := self.action()) - ): - return Alt ( items , action = action ) + items = self.items() + if items: + action = self.action() + if action: + return Alt ( items , action = action ) self.reset(mark) if cut: return None cut = False - if ( - (items := self.items()) - ): + items = self.items() + if items: return Alt ( items , action = None ) self.reset(mark) if cut: return None @@ -383,18 +355,16 @@ def items(self) -> Optional[NamedItemList]: # items: named_item items | named_item mark = self.mark() cut = False - if ( - (named_item := self.named_item()) - and - (items := self.items()) - ): - return [ named_item ] + items + named_item = self.named_item() + if named_item: + items = self.items() + if items: + return [ named_item ] + items self.reset(mark) if cut: return None cut = False - if ( - (named_item := self.named_item()) - ): + named_item = self.named_item() + if named_item: return [ named_item ] self.reset(mark) if cut: return None @@ -405,29 +375,25 @@ def named_item(self) -> Optional[NamedItem]: # named_item: NAME '=' ~ item | item | lookahead mark = self.mark() cut = False - if ( - (name := self.name()) - and - (literal := self.expect('=')) - and - (cut := True) - and - (item := self.item()) - ): - return NamedItem ( name . string , item ) + name = self.name() + if name: + literal = self.expect('=') + if literal: + cut = True + item = self.item() + if item: + return NamedItem ( name . string , item ) self.reset(mark) if cut: return None cut = False - if ( - (item := self.item()) - ): + item = self.item() + if item: return NamedItem ( None , item ) self.reset(mark) if cut: return None cut = False - if ( - (it := self.lookahead()) - ): + it = self.lookahead() + if it: return NamedItem ( None , it ) self.reset(mark) if cut: return None @@ -438,31 +404,26 @@ def lookahead(self) -> Optional[LookaheadOrCut]: # lookahead: '&' ~ atom | '!' ~ atom | '~' mark = self.mark() cut = False - if ( - (literal := self.expect('&')) - and - (cut := True) - and - (atom := self.atom()) - ): - return PositiveLookahead ( atom ) + literal = self.expect('&') + if literal: + cut = True + atom = self.atom() + if atom: + return PositiveLookahead ( atom ) self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect('!')) - and - (cut := True) - and - (atom := self.atom()) - ): - return NegativeLookahead ( atom ) + literal = self.expect('!') + if literal: + cut = True + atom = self.atom() + if atom: + return NegativeLookahead ( atom ) self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect('~')) - ): + literal = self.expect('~') + if literal: return Cut ( ) self.reset(mark) if cut: return None @@ -473,62 +434,55 @@ def item(self) -> Optional[Item]: # item: '[' ~ alts ']' | atom '?' | atom '*' | atom '+' | atom '.' atom '+' | atom mark = self.mark() cut = False - if ( - (literal := self.expect('[')) - and - (cut := True) - and - (alts := self.alts()) - and - (literal_1 := self.expect(']')) - ): - return Opt ( alts ) + literal = self.expect('[') + if literal: + cut = True + alts = self.alts() + if alts: + literal_1 = self.expect(']') + if literal_1: + return Opt ( alts ) self.reset(mark) if cut: return None cut = False - if ( - (atom := self.atom()) - and - (literal := self.expect('?')) - ): - return Opt ( atom ) + atom = self.atom() + if atom: + literal = self.expect('?') + if literal: + return Opt ( atom ) self.reset(mark) if cut: return None cut = False - if ( - (atom := self.atom()) - and - (literal := self.expect('*')) - ): - return Repeat0 ( atom ) + atom = self.atom() + if atom: + literal = self.expect('*') + if literal: + return Repeat0 ( atom ) self.reset(mark) if cut: return None cut = False - if ( - (atom := self.atom()) - and - (literal := self.expect('+')) - ): - return Repeat1 ( atom ) + atom = self.atom() + if atom: + literal = self.expect('+') + if literal: + return Repeat1 ( atom ) self.reset(mark) if cut: return None cut = False - if ( - (sep := self.atom()) - and - (literal := self.expect('.')) - and - (node := self.atom()) - and - (literal_1 := self.expect('+')) - ): - return Gather ( sep , node ) + sep = self.atom() + if sep: + literal = self.expect('.') + if literal: + node = self.atom() + if node: + literal_1 = self.expect('+') + if literal_1: + return Gather ( sep , node ) self.reset(mark) if cut: return None cut = False - if ( - (atom := self.atom()) - ): + atom = self.atom() + if atom: return atom self.reset(mark) if cut: return None @@ -539,29 +493,25 @@ def atom(self) -> Optional[Plain]: # atom: '(' ~ alts ')' | NAME | STRING mark = self.mark() cut = False - if ( - (literal := self.expect('(')) - and - (cut := True) - and - (alts := self.alts()) - and - (literal_1 := self.expect(')')) - ): - return Group ( alts ) + literal = self.expect('(') + if literal: + cut = True + alts = self.alts() + if alts: + literal_1 = self.expect(')') + if literal_1: + return Group ( alts ) self.reset(mark) if cut: return None cut = False - if ( - (name := self.name()) - ): + name = self.name() + if name: return NameLeaf ( name . string ) self.reset(mark) if cut: return None cut = False - if ( - (string := self.string()) - ): + string = self.string() + if string: return StringLeaf ( string . string ) self.reset(mark) if cut: return None @@ -572,16 +522,14 @@ def action(self) -> Optional[str]: # action: "{" ~ target_atoms "}" mark = self.mark() cut = False - if ( - (literal := self.expect("{")) - and - (cut := True) - and - (target_atoms := self.target_atoms()) - and - (literal_1 := self.expect("}")) - ): - return target_atoms + literal = self.expect("{") + if literal: + cut = True + target_atoms = self.target_atoms() + if target_atoms: + literal_1 = self.expect("}") + if literal_1: + return target_atoms self.reset(mark) if cut: return None return None @@ -591,18 +539,16 @@ def target_atoms(self) -> Optional[str]: # target_atoms: target_atom target_atoms | target_atom mark = self.mark() cut = False - if ( - (target_atom := self.target_atom()) - and - (target_atoms := self.target_atoms()) - ): - return target_atom + " " + target_atoms + target_atom = self.target_atom() + if target_atom: + target_atoms = self.target_atoms() + if target_atoms: + return target_atom + " " + target_atoms self.reset(mark) if cut: return None cut = False - if ( - (target_atom := self.target_atom()) - ): + target_atom = self.target_atom() + if target_atom: return target_atom self.reset(mark) if cut: return None @@ -613,60 +559,51 @@ def target_atom(self) -> Optional[str]: # target_atom: "{" ~ target_atoms "}" | NAME | NUMBER | STRING | "?" | ":" | !"}" OP mark = self.mark() cut = False - if ( - (literal := self.expect("{")) - and - (cut := True) - and - (target_atoms := self.target_atoms()) - and - (literal_1 := self.expect("}")) - ): - return "{" + target_atoms + "}" + literal = self.expect("{") + if literal: + cut = True + target_atoms = self.target_atoms() + if target_atoms: + literal_1 = self.expect("}") + if literal_1: + return "{" + target_atoms + "}" self.reset(mark) if cut: return None cut = False - if ( - (name := self.name()) - ): + name = self.name() + if name: return name . string self.reset(mark) if cut: return None cut = False - if ( - (number := self.number()) - ): + number = self.number() + if number: return number . string self.reset(mark) if cut: return None cut = False - if ( - (string := self.string()) - ): + string = self.string() + if string: return string . string self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect("?")) - ): + literal = self.expect("?") + if literal: return "?" self.reset(mark) if cut: return None cut = False - if ( - (literal := self.expect(":")) - ): + literal = self.expect(":") + if literal: return ":" self.reset(mark) if cut: return None cut = False - if ( - self.negative_lookahead(self.expect, "}") - and - (op := self.op()) - ): - return op . string + if self.negative_lookahead(self.expect, "}"): + op = self.op() + if op: + return op . string self.reset(mark) if cut: return None return None diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index bde27890c15a65..cda85121e4b1ef 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -45,81 +45,82 @@ class PythonCallMakerVisitor(GrammarVisitor): def __init__(self, parser_generator: ParserGenerator): self.gen = parser_generator - self.cache: Dict[Any, Any] = {} + self.cache: Dict[Any, Any, bool] = {} - def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str, bool]: name = node.value if name in ("NAME", "NUMBER", "STRING", "OP"): name = name.lower() - return name, f"self.{name}()" + return name, f"self.{name}()", False if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): - return name.lower(), f"self.expect({name!r})" - return name, f"self.{name}()" + return name.lower(), f"self.expect({name!r})", False + return name, f"self.{name}()", False - def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: - return "literal", f"self.expect({node.value})" + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str, bool]: + return "literal", f"self.expect({node.value})", False - def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: + def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str, bool]: if node in self.cache: return self.cache[node] if len(node.alts) == 1 and len(node.alts[0].items) == 1: self.cache[node] = self.visit(node.alts[0].items[0]) else: name = self.gen.name_node(node) - self.cache[node] = name, f"self.{name}()" + self.cache[node] = name, f"self.{name}()", False return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: - name, call = self.visit(node.item) + def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str, bool]: + name, call, opt = self.visit(node.item) if node.name: name = node.name - return name, call + return name, call, opt def lookahead_call_helper(self, node: Lookahead) -> Tuple[str, str]: - name, call = self.visit(node.node) + name, call, opt = self.visit(node.node) head, tail = call.split("(", 1) assert tail[-1] == ")" + assert not opt tail = tail[:-1] return head, tail - def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]: + def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str, bool]: head, tail = self.lookahead_call_helper(node) - return None, f"self.positive_lookahead({head}, {tail})" + return None, f"self.positive_lookahead({head}, {tail})", False - def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]: + def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str, bool]: head, tail = self.lookahead_call_helper(node) - return None, f"self.negative_lookahead({head}, {tail})" + return None, f"self.negative_lookahead({head}, {tail})", False - def visit_Opt(self, node: Opt) -> Tuple[str, str]: - name, call = self.visit(node.node) - return "opt", f"{call}," # Note trailing comma! + def visit_Opt(self, node: Opt) -> Tuple[str, str, bool]: + name, call, opt = self.visit(node.node) + return "opt", call, True - def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: + def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str, bool]: if node in self.cache: return self.cache[node] name = self.gen.name_loop(node.node, False) - self.cache[node] = name, f"self.{name}()," # Also a trailing comma! + self.cache[node] = name, f"self.{name}()", True return self.cache[node] - def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: + def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str, bool]: if node in self.cache: return self.cache[node] name = self.gen.name_loop(node.node, True) - self.cache[node] = name, f"self.{name}()" # But no trailing comma here! + self.cache[node] = name, f"self.{name}()", False return self.cache[node] - def visit_Gather(self, node: Gather) -> Tuple[str, str]: + def visit_Gather(self, node: Gather) -> Tuple[str, str, bool]: if node in self.cache: return self.cache[node] name = self.gen.name_gather(node) - self.cache[node] = name, f"self.{name}()" # No trailing comma here either! + self.cache[node] = name, f"self.{name}()", False return self.cache[node] - def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: + def visit_Group(self, node: Group) -> Tuple[Optional[str], str, bool]: return self.visit(node.rhs) - def visit_Cut(self, node: Cut) -> Tuple[str, str]: - return "cut", "True" + def visit_Cut(self, node: Cut) -> Tuple[str, str, bool]: + return "cut", "True", True class PythonParserGenerator(ParserGenerator, GrammarVisitor): @@ -174,15 +175,20 @@ def visit_Rule(self, node: Rule) -> None: self.print("return None") def visit_NamedItem(self, node: NamedItem) -> None: - name, call = self.callmakervisitor.visit(node.item) + name, call, opt = self.callmakervisitor.visit(node.item) if node.name: name = node.name if not name: - self.print(call) + if not opt: + self.print(f"if {call}:") + self.level += 1 else: if name != "cut": name = self.dedupe(name) - self.print(f"({name} := {call})") + self.print(f"{name} = {call}") + if not opt: + self.print(f"if {name}:") + self.level += 1 def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: if is_loop: @@ -194,33 +200,30 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: with self.local_variable_context(): self.print("cut = False") # TODO: Only if needed. if is_loop: - self.print("while (") - else: - self.print("if (") - with self.indent(): - first = True - for item in node.items: - if first: - first = False - else: - self.print("and") - self.visit(item) - self.print("):") - with self.indent(): - action = node.action - if not action: - if is_gather: - assert len(self.local_variable_names) == 2 - action = ( - f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}" - ) - else: - action = f"[{', '.join(self.local_variable_names)}]" - if is_loop: - self.print(f"children.append({action})") - self.print(f"mark = self.mark()") + self.print("while True:") + self.level += 1 + level = self.level + for item in node.items: + self.visit(item) + action = node.action + if not action: + if is_gather: + assert len(self.local_variable_names) == 2 + action = ( + f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}" + ) else: - self.print(f"return {action}") + action = f"[{', '.join(self.local_variable_names)}]" + if is_loop: + self.print(f"children.append({action})") + self.print(f"mark = self.mark()") + self.print("continue") + else: + self.print(f"return {action}") + self.level = level + if is_loop: + self.print("break") + self.level -= 1 self.print("self.reset(mark)") # Skip remaining alternatives if a cut was reached. self.print("if cut: return None") # TODO: Only if needed. diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 1f79d8f702fb1b..6b98590d72a750 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -7,7 +7,7 @@ import tokenize import token -from typing import Any, cast, Dict, IO, Type, Final +from typing import Any, cast, Dict, IO, Type from pegen.build import compile_c_extension from pegen.c_generator import CParserGenerator @@ -17,7 +17,7 @@ from pegen.python_generator import PythonParserGenerator from pegen.tokenizer import Tokenizer -EXACT_TOKENS = token.EXACT_TOKEN_TYPES # type: ignore +EXACT_TOKENS = tokenize.EXACT_TOKEN_TYPES # type: ignore NON_EXACT_TOKENS = { name for index, name in token.tok_name.items() if index not in EXACT_TOKENS.values() } @@ -102,7 +102,7 @@ def generate_parser_c_extension( def print_memstats() -> bool: - MiB: Final = 2 ** 20 + MiB = 2 ** 20 try: import psutil # type: ignore except ImportError: diff --git a/Tools/peg_generator/pegen/tokenizer.py b/Tools/peg_generator/pegen/tokenizer.py index 61a28efc84b62b..b6e61ff1bdaea1 100644 --- a/Tools/peg_generator/pegen/tokenizer.py +++ b/Tools/peg_generator/pegen/tokenizer.py @@ -4,7 +4,7 @@ Mark = int # NewType('Mark', int) -exact_token_types = token.EXACT_TOKEN_TYPES # type: ignore +exact_token_types = tokenize.EXACT_TOKEN_TYPES # type: ignore def shorttok(tok: tokenize.TokenInfo) -> str: diff --git a/Tools/peg_generator/scripts/grammar_grapher.py b/Tools/peg_generator/scripts/grammar_grapher.py index 3aa25466c70d47..f9c8d15f385566 100755 --- a/Tools/peg_generator/scripts/grammar_grapher.py +++ b/Tools/peg_generator/scripts/grammar_grapher.py @@ -92,10 +92,10 @@ def main() -> None: # Flatten the start node if has only a single reference root_node = "start" - if start := references["start"]: - if len(start) == 1: - root_node = list(start)[0] - del references["start"] + start = references["start"] + if start and len(start) == 1: + root_node = list(start)[0] + del references["start"] print("digraph g1 {") print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps