8000 Refactor scripts in Tools/peg_generator/scripts (GH-20401) · python/cpython@18f1226 · GitHub
[go: up one dir, main page]
Skip to content

Commit 18f1226

Browse files
miss-islingtonlysnikolaou
miss-islington
and
lysnikolaou
authored
Refactor scripts in Tools/peg_generator/scripts (GH-20401)
(cherry picked from commit ba6fd87) Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
1 parent d5e7348 commit 18f1226

File tree

7 files changed

+143
-144
lines changed

7 files changed

+143
-144
lines changed

‎Modules/_peg_parser.c

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,15 @@ _Py_compile_string(PyObject *self, PyObject *args, PyObject *kwds)
8080
PyObject *
8181
_Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
8282
{
83-
static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL};
83+
static char *keywords[] = {"string", "filename", "mode", "oldparser", "ast", NULL};
8484
char *the_string;
8585
char *filename = "<string>";
8686
char *mode_str = "exec";
8787
int oldparser = 0;
88+
int ast = 1;
8889

89-
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords,
90-
&the_string, &filename, &mode_str, &oldparser)) {
90+
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sspp", keywords,
91+
&the_string, &filename, &mode_str, &oldparser, &ast)) {
9192
return NULL;
9293
}
9394

@@ -110,7 +111,14 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
110111
return NULL;
111112
}
112113

113-
PyObject *result = PyAST_mod2obj(mod);
114+
PyObject *result;
115+
if (ast) {
116+
result = PyAST_mod2obj(mod);
117+
}
118+
else {
119+
Py_INCREF(Py_None);
120+
result = Py_None;
121+
}
114122
PyArena_Free(arena);
115123
return result;
116124
}

‎Tools/peg_generator/Makefile

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -70,23 +70,21 @@ stats: peg_extension/parse.c data/xxl.py
7070
time: time_compile
7171

7272
time_compile: venv data/xxl.py
73-
$(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl compile
73+
$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl compile
7474

7575
time_parse: venv data/xxl.py
76-
$(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl parse
76+
$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl parse
7777

7878
time_old: time_old_compile
7979

8080
time_old_compile: venv data/xxl.py
81-
$(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl compile
81+
$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl compile
8282

8383
time_old_parse: venv data/xxl.py
84-
$(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl parse
84+
$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl parse
8585

8686
time_peg_dir: venv
8787
$(VENVPYTHON) scripts/test_parse_directory.py \
88-
--grammar-file $(GRAMMAR) \
89-
--tokens-file $(TOKENS) \
9088
-d $(TESTDIR) \
9189
$(TESTFLAGS) \
9290
--exclude "*/failset/*" \
@@ -95,12 +93,8 @@ time_peg_dir: venv
9593

9694
time_stdlib: $(CPYTHON) venv
9795
$(VENVPYTHON) scripts/test_parse_directory.py \
98-
--grammar-file $(GRAMMAR) \
99-
--tokens-file $(TOKENS) \
10096
-d $(CPYTHON) \
10197
$(TESTFLAGS) \
102-
--exclude "*/test2to3/*" \
103-
--exclude "*/test2to3/**/*" \
10498
--exclude "*/bad*" \
10599
--exclude "*/lib2to3/tests/data/*"
106100

‎Tools/peg_generator/scripts/benchmark.py

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
argparser.add_argument(
2525
"--parser",
2626
action="store",
27-
choices=["pegen", "cpython"],
27+
choices=["new", "old"],
2828
default="pegen",
2929
help="Which parser to benchmark (default is pegen)",
3030
)
@@ -40,7 +40,12 @@
4040
command_compile = subcommands.add_parser(
4141
"compile", help="Benchmark parsing and compiling to bytecode"
4242
)
43-
command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST")
43+
command_parse = subcommands.add_parser(
44+
"parse", help="Benchmark parsing and generating an ast.AST"
45+
)
46+
command_notree = subcommands.add_parser(
47+
"notree", help="Benchmark parsing and dumping the tree"
48+
)
4449

4550

4651
def benchmark(func):
@@ -62,7 +67,7 @@ def wrapper(*args):
6267

6368
@benchmark
6469
def time_compile(source, parser):
65-
if parser == "cpython":
70+
if parser == "old":
6671
return _peg_parser.compile_string(
6772
source,
6873
oldparser=True,
@@ -73,32 +78,40 @@ def time_compile(source, parser):
7378

7479
@benchmark
7580
def time_parse(source, parser):
76-
if parser == "cpython":
81+
if parser == "old":
7782
return _peg_parser.parse_string(source, oldparser=True)
7883
else:
7984
return _peg_parser.parse_string(source)
8085

8186

87+
@benchmark
88+
def time_notree(source, parser):
89+
if parser == "old":
90+
return _peg_parser.parse_string(source, oldparser=True, ast=False)
91+
else:
92+
return _peg_parser.parse_string(source, ast=False)
93+
94+
8295
def run_benchmark_xxl(subcommand, parser, source):
8396
if subcommand == "compile":
8497
time_compile(source, parser)
8598
elif subcommand == "parse":
8699
time_parse(source, parser)
100+
elif subcommand == "notree":
101+
time_notree(source, parser)
87102

88103

89104
def run_benchmark_stdlib(subcommand, parser):
105+
modes = {"compile": 2, "parse": 1, "notree": 0}
90106
for _ in range(3):
91107
parse_directory(
92108
"../../Lib",
93-
"../../Grammar/python.gram",
94-
"../../Grammar/Tokens",
95109
verbose=False,
96110
excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
97-
skip_actions=False,
98111
tree_arg=0,
99112
short=True,
100-
mode=2 if subcommand == "compile" else 1,
101-
parser=parser,
113+
mode=modes[subcommand],
114+
oldparser=(parser == "old"),
102115
)
103116

104117

‎Tools/peg_generator/scripts/grammar_grapher.py

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,13 @@
4242
)
4343

4444
argparser = argparse.ArgumentParser(prog="graph_grammar", description="Graph a grammar tree",)
45+
argparser.add_argument(
46+
"-s",
47+
"--start",
48+
choices=["exec", "eval", "single"],
49+
default="exec",
50+
help="Choose the grammar's start rule (exec, eval or single)",
51+
)
4552
argparser.add_argument("grammar_file", help="The grammar file to graph")
4653

4754

@@ -91,19 +98,15 @@ def main() -> None:
9198
references[name] = set(references_for_item(rule))
9299

93100
# Flatten the start node if has only a single reference
94-
root_node = "start"
95-
if start := references["start"]:
96-
if len(start) == 1:
97-
root_node = list(start)[0]
98-
del references["start"]
101+
root_node = {"exec": "file", "eval": "eval", "single": "interactive"}[args.start]
99102

100103
print("digraph g1 {")
101104
print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps
102105
print(f'\troot="{root_node}";')
103-
print(f"\t{root_node} [color=green, shape=circle]")
106+
print(f"\t{root_node} [color=green, shape=circle];")
104107
for name, refs in references.items():
105-
if refs: # Ignore empty sets
106-
print(f"\t{name} -> {','.join(refs)};")
108+
for ref in refs:
109+
print(f"\t{name} -> {ref};")
107110
print("}")
108111

109112

‎Tools/peg_generator/scripts/show_parse.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,13 @@
4141
parser.add_argument(
4242
"-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)"
4343
)
44-
parser.add_argument("-g", "--grammar-file", help="grammar to use (default: use the ast module)")
44+
parser.add_argument(
45+
"-p",
46+
"--parser",
47+
choices=["new", "old"],
48+
default="new",
49+
help="choose the parser to use"
50+
)
4551
parser.add_argument(
4652
"-m",
4753
"--multiline",
@@ -84,31 +90,30 @@ def print_parse(source: str, verbose: bool = False) -> None:
8490

8591
def main() -> None:
8692
args = parser.parse_args()
87-
if args.diff and not args.grammar_file:
88-
parser.error("-d/--diff requires -g/--grammar-file")
93+
new_parser = args.parser == "new"
8994
if args.multiline:
9095
sep = "\n"
9196
else:
9297
sep = " "
9398
program = sep.join(args.program)
94-
if args.grammar_file:
99+
if new_parser:
95100
tree = _peg_parser.parse_string(program)
96101

97102
if args.diff:
98-
a = tree
99-
b = _peg_parser.parse_string(program, oldparser=True)
103+
a = _peg_parser.parse_string(program, oldparser=True)
104+
b = tree
100105
diff = diff_trees(a, b, args.verbose)
101106
if diff:
102107
for line in diff:
103108
print(line)
104109
else:
105110
print("# Trees are the same")
106111
else:
107-
print(f"# Parsed using {args.grammar_file}")
112+
print("# Parsed using the new parser")
108113
print(format_tree(tree, args.verbose))
109114
else:
110115
tree = _peg_parser.parse_string(program, oldparser=True)
111-
print("# Parse using the old parser")
116+
print("# Parsed using the old parser")
112117
print(format_tree(tree, args.verbose))
113118

114119

0 commit comments

Comments
 (0)
0