8000 bpo-39411: pyclbr rewrite on AST (#18103) · python/cpython@fa476fe · GitHub
[go: up one dir, main page]
Skip to content

Commit fa476fe

Browse files
isidenticalisidentical
authored
bpo-39411: pyclbr rewrite on AST (#18103)
- Rewrite pyclbr using an AST processor - Add is_async to the pyclbr.Function
1 parent 95ce7cd commit fa476fe

File tree

4 files changed

+118
-211
lines changed

4 files changed

+118
-211
lines changed

‎Doc/library/pyclbr.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,13 @@ statements. They have the following attributes:
9797
.. versionadded:: 3.7
9898

9999

100+
.. attribute:: Function.is_async
101+
102+
``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise.
103+
104+
.. versionadded:: 3.10
105+
106+
100107
.. _pyclbr-class-objects:
101108

102109
Class Objects

‎Lib/pyclbr.py

Lines changed: 107 additions & 206 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
children -- nested objects contained in this object.
2626
The 'children' attribute is a dictionary mapping names to objects.
2727
28-
Instances of Function describe functions with the attributes from _Object.
28+
Instances of Function describe functions with the attributes from _Object,
29+
plus the following:
30+
is_async -- if a function is defined with an 'async' prefix
2931
3032
Instances of Class describe classes with the attributes from _Object,
3133
plus the fo 6D40 llowing:
@@ -38,11 +40,10 @@
3840
shouldn't happen often.
3941
"""
4042

41-
import io
43+
import ast
44+
import copy
4245
import sys
4346
import importlib.util
44-
import tokenize
45-
from token import NAME, DEDENT, OP
4647

4748
__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
4849

@@ -58,41 +59,33 @@ def __init__(self, module, name, file, lineno, parent):
5859
self.lineno = lineno
5960
self.parent = parent
6061
self.children = {}
61-
62-
def _addchild(self, name, obj):
63-
self.children[name] = obj
64-
62+
if parent is not None:
63+
parent.children[name] = self
6564

6665
class Function(_Object):
6766
"Information about a Python function, including methods."
68-
def __init__(self, module, name, file, lineno, parent=None):
69-
_Object.__init__(self, module, name, file, lineno, parent)
70-
67+
def __init__(self, module, name, file, lineno, parent=None, is_async=False):
68+
super().__init__(module, name, file, lineno, parent)
69+
self.is_async = is_async
70+
if isinstance(parent, Class):
71+
parent.methods[name] = lineno
7172

7273
class Class(_Object):
7374
"Information about a Python class."
74-
def __init__(self, module, name, super, file, lineno, parent=None):
75-
_Object.__init__(self, module, name, file, lineno, parent)
76-
self.super = [] if super is None else super
75+
def __init__(self, module, name, super_, file, lineno, parent=None):
76+
super().__init__(module, name, file, lineno, parent)
77+
self.super = super_ or []
7778
self.methods = {}
7879

79-
def _addmethod(self, name, lineno):
80-
self.methods[name] = lineno
81-
82-
83-
def _nest_function(ob, func_name, lineno):
80+
# These 2 functions are used in these tests
81+
# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py
82+
def _nest_function(ob, func_name, lineno, is_async=False):
8483
"Return a Function after nesting within ob."
85-
newfunc = Function(ob.module, func_name, ob.file, lineno, ob)
86-
ob._addchild(func_name, newfunc)
87-
if isinstance(ob, Class):
88-
ob._addmethod(func_name, lineno)
89-
return newfunc
84+
return Function(ob.module, func_name, ob.file, lineno, ob, is_async)
9085

9186
def _nest_class(ob, class_name, lineno, super=None):
9287
"Return a Class after nesting within ob."
93-
newclass = Class(ob.module, class_name, super, ob.file, lineno, ob)
94-
ob._addchild(class_name, newclass)
95-
return newclass
88+
return Class(ob.module, class_name, super, ob.file, lineno, ob)
9689

9790
def readmodule(module, path=None):
9891
"""Return Class objects for the top-level classes in module.
@@ -179,187 +172,95 @@ def _readmodule(module, path, inpackage=None):
179172
return _create_tree(fullmodule, path, fname, source, tree, inpackage)
180173

181174

182-
def _create_tree(fullmodule, path, fname, source, tree, inpackage):
183-
"""Return the tree for a particular module.
184-
185-
fullmodule (full module name), inpackage+module, becomes o.module.
186-
path is passed to recursive calls of _readmodule.
187-
fname becomes o.file.
188-
source is tokenized. Imports cause recursive calls to _readmodule.
189-
tree is {} or {'__path__': <submodule search locations>}.
190-
inpackage, None or string, is passed to recursive calls of _readmodule.
191-
192-
The effect of recursive calls is mutation of global _modules.
193-
"""
194-
f = io.StringIO(source)
175+
class _ModuleBrowser(ast.NodeVisitor):
176+
def __init__(self, module, path, file, tree, inpackage):
177+
self.path = path
178+
self.tree = tree
179+
self.file = file
180+
self.module = module
181+
self.inpackage = inpackage
182+
self.stack = []
183+
184+
def visit_ClassDef(self, node):
185+
bases = []
186+
for base in node.bases:
187+
name = ast.unparse(base)
188+
if name in self.tree:
189+
# We know this super class.
190+
bases.append(self.tree[name])
191+
elif len(names := name.split(".")) > 1:
192+
# Super class form is module.class:
193+
# look in module for class.
194+
*_, module, class_ = names
195+
if module in _modules:
196+
bases.append(_modules[module].get(class_, name))
197+
else:
198+
bases.append(name)
199+
200+
parent = self.stack[-1] if self.stack else None
201+
class_ = Class(
202+
self.module, node.name, bases, self.file, node.lineno, parent
203+
)
204+
if parent is None:
205+
self.tree[node.name] = class_
206+
self.stack.append(class_)
207+
self.generic_visit(node)
208+
self.stack.pop()
209+
210+
def visit_FunctionDef(self, node, *, is_async=False):
211+
parent = self.stack[-1] if self.stack else None
212+
function = Function(
213+
self.module, node.name, self.file, node.lineno, parent, is_async
214+
)
215+
if parent is None:
216+
self.tree[node.name] = function
217+
self.stack.append(function)
218+
self.generic_visit(node)
219+
self.stack.pop()
220+
221+
def visit_AsyncFunctionDef(self, node):
222+
self.visit_FunctionDef(node, is_async=True)
223+
224+
def visit_Import(self, node):
225+
if node.col_offset != 0:
226+
return
227+
228+
for module in node.names:
229+
try:
230+
try:
231+
_readmodule(module.name, self.path, self.inpackage)
232+
except ImportError:
233+
_readmodule(module.name, [])
234+
except (ImportError, SyntaxError):
235+
# If we can't find or parse the imported module,
236+
# too bad -- don't die here.
237+
continue
238+
239+
def visit_ImportFrom(self, node):
240+
if node.col_offset != 0:
241+
return
242+
try:
243+
module = "." * node.level
244+
if node.module:
245+
module += node.module
246+
module = _readmodule(module, self.path, self.inpackage)
247+
except (ImportError, SyntaxError):
248+
return
249+
250+
for name in node.names:
251+
if name.name in module:
252+
self.tree[name.asname or name.name] = module[name.name]
253+
elif name.name == "*":
254+
for import_name, import_value in module.items():
255+
if import_name.startswith("_"):
256+
continue
257+
self.tree[import_name] = import_value
195258

196-
stack = [] # Initialize stack of (class, indent) pairs.
197259

198-
g = tokenize.generate_tokens(f.readline)
199-
try:
200-
for tokentype, token, start, _end, _line in g:
201-
if tokentype == DEDENT:
202-
lineno, thisindent = start
203-
# Close previous nested classes and defs.
204-
while stack and stack[-1][1] >= thisindent:
205-
del stack[-1]
206-
elif token == 'def':
207-
lineno, thisindent = start
208-
# Close previous nested classes and defs.
209-
while stack and stack[-1][1] >= thisindent:
210-
del stack[-1]
211-
tokentype, func_name, start = next(g)[0:3]
212-
if tokentype != NAME:
213-
continue # Skip def with syntax error.
214-
cur_func = None
215-
if stack:
216-
cur_obj = stack[-1][0]
217-
cur_func = _nest_function(cur_obj, func_name, lineno)
218-
else:
219-
# It is just a function.
220-
cur_func = Function(fullmodule, func_name, fname, lineno)
221-
tree[func_name] = cur_func
222-
stack.append((cur_func, thisindent))
223-
elif token == 'class':
224-
lineno, thisindent = start
225-
# Close previous nested classes and defs.
226-
while stack and stack[-1][1] >= thisindent:
227-
del stack[-1]
228-
tokentype, class_name, start = next(g)[0:3]
229-
if tokentype != NAME:
230-
continue # Skip class with syntax error.
231-
# Parse what follows the class name.
232-
tokentype, token, start = next(g)[0:3]
233-
inherit = None
234-
if token == '(':
235-
names = [] # Initialize list of superclasses.
236-
level = 1
237-
super = [] # Tokens making up current superclass.
238-
while True:
239-
tokentype, token, start = next(g)[0:3]
240-
if token in (')', ',') and level == 1:
241-
n = "".join(super)
242-
if n in tree:
243-
# We know this super class.
244-
n = tree[n]
245-
else:
246-
c = n.split('.')
247-
if len(c) > 1:
248-
# Super class form is module.class:
249-
# look in module for class.
250-
m = c[-2]
251-
c = c[-1]
252-
if m in _modules:
253-
d = _modules[m]
254-
if c in d:
255-
n = d[c]
256-
names.append(n)
257-
super = []
258-
if token == '(':
259-
level += 1
260-
elif token == ')':
261-
level -= 1
262-
if level == 0:
263-
break
264-
elif token == ',' and level == 1:
265-
pass
266-
# Only use NAME and OP (== dot) tokens for type name.
267-
elif tokentype in (NAME, OP) and level == 1:
268-
super.append(token)
269-
# Expressions in the base list are not supported.
270-
inherit = names
271-
if stack:
272-
cur_obj = stack[-1][0]
273-
cur_class = _nest_class(
274-
cur_obj, class_name, lineno, inherit)
275-
else:
276-
cur_class = Class(fullmodule, class_name, inherit,
277-
fname, lineno)
278-
tree[class_name] = cur_class
279-
stack.append((cur_class, thisindent))
280-
elif token == 'import' and start[1] == 0:
281-
modules = _getnamelist(g)
282-
for mod, _mod2 in modules:
283-
try:
284-
# Recursively read the imported module.
285-
if inpackage is None:
286-
_readmodule(mod, path)
287-
else:
288-
try:
289-
_readmodule(mod, path, inpackage)
290-
except ImportError:
291-
_readmodule(mod, [])
292-
except:
293-
# If we can't find or parse the imported module,
294-
# too bad -- don't die here.
295-
pass
296-
elif token == 'from' and start[1] == 0:
297-
mod, token = _getname(g)
298-
if not mod or token != "import":
299-
continue
300-
names = _getnamelist(g)
301-
try:
302-
# Recursively read the imported module.
303-
d = _readmodule(mod, path, inpackage)
304-
except:
305-
# If we can't find or parse the imported module,
306-
# too bad -- don't die here.
307-
continue
308-
# Add any classes that were defined in the imported module
309-
# to our name space if they were mentioned in the list.
310-
for n, n2 in names:
311-
if n in d:
312-
tree[n2 or n] = d[n]
313-
elif n == '*':
314-
# Don't add names that start with _.
315-
for n in d:
316-
if n[0] != '_':
317-
tree[n] = d[n]
318-
except StopIteration:
319-
pass
320-
321-
f.close()
322-
return tree
323-
324-
325-
def _getnamelist(g):
326-
"""Return list of (dotted-name, as-name or None) tuples for token source g.
327-
328-
An as-name is the name that follows 'as' in an as clause.
329-
"""
330-
names = []
331-
while True:
332-
name, token = _getname(g)
333-
if not name:
334-
break
335-
if token == 'as':
336-
name2, token = _getname(g)
337-
else:
338-
name2 = None
339-
names.append((name, name2))
340-
while token != "," and "\n" not in token:
341-
token = next(g)[1]
342-
if token != ",":
343-
break
344-
return names
345-
346-
347-
def _getname(g):
348-
"Return (dotted-name or None, next-token) tuple for token source g."
349-
parts = []
350-
tokentype, token = next(g)[0:2]
351-
if tokentype != NAME and token != '*':
352-
return (None, token)
353-
parts.append(token)
354-
while True:
355-
tokentype, token = next(g)[0:2]
356-
if token != '.':
357-
break
358-
tokentype, token = next(g)[0:2]
359-
if tokentype != NAME:
360-
break
361-
parts.append(token)
362-
return (".".join(parts), token)
260+
def _create_tree(fullmodule, path, fname, source, tree, inpackage):
261+
mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage)
262+
mbrowser.visit(ast.parse(source))
263+
return mbrowser.tree
363264

364265

365266
def _main():

‎Lib/test/test_pyclbr.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,20 +150,17 @@ def test_easy(self):
150150
self.checkModule('difflib', ignore=("Match",))
151151

152152
def test_decorators(self):
153-
# XXX: See comment in pyclbr_input.py for a test that would fail
154-
# if it were not commented out.
155-
#
156153
self.checkModule('test.pyclbr_input', ignore=['om'])
157154

158155
def test_nested(self):
159156
mb = pyclbr
160157
# Set arguments for descriptor creation and _creat_tree call.
161158
m, p, f, t, i = 'test', '', 'test.py', {}, None
162159
source = dedent("""\
163-
def f0:
160+
def f0():
164161
def f1(a,b,c):
165162
def f2(a=1, b=2, c=3): pass
166-
return f1(a,b,d)
163+
return f1(a,b,d)
167164
class c1: pass
168165
class C0:
169166
"Test class."

‎Misc/NEWS.d/next/Library/2020-01-21-16-38-25.bpo-39411.9uHFqT.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add an ``is_async`` identifier to :mod:`pyclbr`'s ``Function`` objects.
2+
Patch by Batuhan Taskaya

0 commit comments

Comments
 (0)
0