8000 bpo-41746: Add type information to asdl_seq objects by pablogsal · Pull Request #22223 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

bpo-41746: Add type information to asdl_seq objects #22223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Sep 16, 2020
Merged

bpo-41746: Add type information to asdl_seq objects #22223

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
19d05ae
Modify asdl_c to generate new typed sequences
pablogsal Sep 13, 2020
9df286b
Change metagrammar to accept typed variables
pablogsal Sep 13, 2020
2dc6092
Modify PEG parser files to use new typed sequences
pablogsal Sep 13, 2020
4db35be
Modify the PEG highlight to ignore typed variables as well
pablogsal Sep 13, 2020
496ad2c
[Automatic changes] Regenerate AST files and parser
pablogsal Sep 13, 2020
5fb9faf
Adapt the code base to use the new typed sequences
pablogsal Sep 13, 2020
9820428
Rename asdl_seq_SET_GENERIC to asdl_seq_SET_UNTYPED
pablogsal Sep 14, 2020
53bcfd7
[Address feedback] Type more elements in the PEG parser
pablogsal Sep 14, 2020
691e048
Type the argument to _PyPegen_slash_with_default
lysnikolaou Sep 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Change metagrammar to accept typed variables
  • Loading branch information
@pablogsal
pablogsal committed Sep 13, 2020
commit 9df286b8e23c481bfa6e71b85d3b0afcd1f37d19
15 changes: 11 additions & 4 deletions Tools/peg_generator/pegen/c_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ class FunctionCall:
function: str
arguments: List[Any] = field(default_factory=list)
assigned_variable: Optional[str] = None
assigned_variable_type: Optional[str] = None
return_type: Optional[str] = None
nodetype: Optional[NodeTypes] = None
force_true: bool = False
Expand All @@ -87,7 +88,10 @@ def __str__(self) -> str:
if self.force_true:
parts.append(", 1")
if self.assigned_variable:
parts = ["(", self.assigned_variable, " = ", *parts, ")"]
if self.assigned_variable_type:
parts = ["(", self.assigned_variable, " = ", '(', self.assigned_variable_type, ')', *parts, ")"]
else:
parts = ["(", self.assigned_variable, " = ", *parts, ")"]
if self.comment:
parts.append(f" // {self.comment}")
return "".join(parts)
Expand Down Expand Up @@ -210,6 +214,8 @@ def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
call = self.generate_call(node.item)
if node.name:
call.assigned_variable = node.name
if node.type:
call.assigned_variable_type = node.type
return call

def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
Expand Down Expand Up @@ -568,9 +574,9 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
self.print("PyMem_Free(_children);")
self.add_return("NULL")
self.print("}")
self.print("asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);")
self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);")
self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);")
self.print("for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);")
self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_GENERIC(_seq, i, _children[i]);")
self.print("PyMem_Free(_children);")
if node.name:
self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);")
Expand Down Expand Up @@ -782,4 +788,5 @@ def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]:
name = node.name if node.name else call.assigned_variable
if name is not None:
name = self.dedupe(name)
return name, call.return_type
return_type = call.return_type if node.type is None else node.type
return name, return_type
3 changes: 2 additions & 1 deletion Tools/peg_generator/pegen/grammar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,10 @@ def collect_todo(self, gen: ParserGenerator) -> None:


class NamedItem:
def __init__(self, name: Optional[str], item: Item):
def __init__(self, name: Optional[str], item: Item, type: Optional[str] = None):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

__str__ and __repr__ will have to be updated here as well, right?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(for the future) is there are reason to avoid using dataclasses for these meta-grammar items itself?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What would be the advantage? We use custom __repr__ and __str__ , we do have some transformations in the __init__ and we do not compare nodes.

self.name = name
self.item = item
self.type = type
self.nullable = False

def __str__(self) -> str:
Expand Down
2 changes: 2 additions & 0 deletions Tools/peg_generator/pegen/metagrammar.gram
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ items[NamedItemList]:
| named_item { [named_item] }

named_item[NamedItem]:
| NAME '[' type=NAME '*' ']' '=' ~ item {NamedItem(name.string, item, type.string+"*")}
| NAME '[' type=NAME ']' '=' ~ item {NamedItem(name.string, item, type.string)}
| NAME '=' ~ item {NamedItem(name.string, item)}
| item {NamedItem(None, item)}
| it=lookahead {NamedItem(None, it)}
Expand Down
0