25
25
children -- nested objects contained in this object.
26
26
The 'children' attribute is a dictionary mapping names to objects.
27
27
28
- Instances of Function describe functions with the attributes from _Object.
28
+ Instances of Function describe functions with the attributes from _Object,
29
+ plus the following:
30
+ is_async -- if a function is defined with an 'async' prefix
29
31
30
32
Instances of Class describe classes with the attributes from _Object,
31
33
plus the fo
6D40
llowing:
38
40
shouldn't happen often.
39
41
"""
40
42
41
- import io
43
+ import ast
44
+ import copy
42
45
import sys
43
46
import importlib .util
44
- import tokenize
45
- from token import NAME , DEDENT , OP
46
47
47
48
__all__ = ["readmodule" , "readmodule_ex" , "Class" , "Function" ]
48
49
@@ -58,41 +59,33 @@ def __init__(self, module, name, file, lineno, parent):
58
59
self .lineno = lineno
59
60
self .parent = parent
60
61
self .children = {}
61
-
62
- def _addchild (self , name , obj ):
63
- self .children [name ] = obj
64
-
62
+ if parent is not None :
63
+ parent .children [name ] = self
65
64
66
65
class Function (_Object ):
67
66
"Information about a Python function, including methods."
68
- def __init__ (self , module , name , file , lineno , parent = None ):
69
- _Object .__init__ (self , module , name , file , lineno , parent )
70
-
67
+ def __init__ (self , module , name , file , lineno , parent = None , is_async = False ):
68
+ super ().__init__ (module , name , file , lineno , parent )
69
+ self .is_async = is_async
70
+ if isinstance (parent , Class ):
71
+ parent .methods [name ] = lineno
71
72
72
73
class Class (_Object ):
73
74
"Information about a Python class."
74
- def __init__ (self , module , name , super , file , lineno , parent = None ):
75
- _Object .__init__ (self , module , name , file , lineno , parent )
76
- self .super = [] if super is None else super
75
+ def __init__ (self , module , name , super_ , file , lineno , parent = None ):
76
+ super () .__init__ (module , name , file , lineno , parent )
77
+ self .super = super_ or []
77
78
self .methods = {}
78
79
79
- def _addmethod (self , name , lineno ):
80
- self .methods [name ] = lineno
81
-
82
-
83
- def _nest_function (ob , func_name , lineno ):
80
+ # These 2 functions are used in these tests
81
+ # Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py
82
+ def _nest_function (ob , func_name , lineno , is_async = False ):
84
83
"Return a Function after nesting within ob."
85
- newfunc = Function (ob .module , func_name , ob .file , lineno , ob )
86
- ob ._addchild (func_name , newfunc )
87
- if isinstance (ob , Class ):
88
- ob ._addmethod (func_name , lineno )
89
- return newfunc
84
+ return Function (ob .module , func_name , ob .file , lineno , ob , is_async )
90
85
91
86
def _nest_class (ob , class_name , lineno , super = None ):
92
87
"Return a Class after nesting within ob."
93
- newclass = Class (ob .module , class_name , super , ob .file , lineno , ob )
94
- ob ._addchild (class_name , newclass )
95
- return newclass
88
+ return Class (ob .module , class_name , super , ob .file , lineno , ob )
96
89
97
90
def readmodule (module , path = None ):
98
91
"""Return Class objects for the top-level classes in module.
@@ -179,187 +172,95 @@ def _readmodule(module, path, inpackage=None):
179
172
return _create_tree (fullmodule , path , fname , source , tree , inpackage )
180
173
181
174
182
- def _create_tree (fullmodule , path , fname , source , tree , inpackage ):
183
- """Return the tree for a particular module.
184
-
185
- fullmodule (full module name), inpackage+module, becomes o.module.
186
- path is passed to recursive calls of _readmodule.
187
- fname becomes o.file.
188
- source is tokenized. Imports cause recursive calls to _readmodule.
189
- tree is {} or {'__path__': <submodule search locations>}.
190
- inpackage, None or string, is passed to recursive calls of _readmodule.
191
-
192
- The effect of recursive calls is mutation of global _modules.
193
- """
194
- f = io .StringIO (source )
175
+ class _ModuleBrowser (ast .NodeVisitor ):
176
+ def __init__ (self , module , path , file , tree , inpackage ):
177
+ self .path = path
178
+ self .tree = tree
179
+ self .file = file
180
+ self .module = module
181
+ self .inpackage = inpackage
182
+ self .stack = []
183
+
184
+ def visit_ClassDef (self , node ):
185
+ bases = []
186
+ for base in node .bases :
187
+ name = ast .unparse (base )
188
+ if name in self .tree :
189
+ # We know this super class.
190
+ bases .append (self .tree [name ])
191
+ elif len (names := name .split ("." )) > 1 :
192
+ # Super class form is module.class:
193
+ # look in module for class.
194
+ * _ , module , class_ = names
195
+ if module in _modules :
196
+ bases .append (_modules [module ].get (class_ , name ))
197
+ else :
198
+ bases .append (name )
199
+
200
+ parent = self .stack [- 1 ] if self .stack else None
201
+ class_ = Class (
202
+ self .module , node .name , bases , self .file , node .lineno , parent
203
+ )
204
+ if parent is None :
205
+ self .tree [node .name ] = class_
206
+ self .stack .append (class_ )
207
+ self .generic_visit (node )
208
+ self .stack .pop ()
209
+
210
+ def visit_FunctionDef (self , node , * , is_async = False ):
211
+ parent = self .stack [- 1 ] if self .stack else None
212
+ function = Function (
213
+ self .module , node .name , self .file , node .lineno , parent , is_async
214
+ )
215
+ if parent is None :
216
+ self .tree [node .name ] = function
217
+ self .stack .append (function )
218
+ self .generic_visit (node )
219
+ self .stack .pop ()
220
+
221
+ def visit_AsyncFunctionDef (self , node ):
222
+ self .visit_FunctionDef (node , is_async = True )
223
+
224
+ def visit_Import (self , node ):
225
+ if node .col_offset != 0 :
226
+ return
227
+
228
+ for module in node .names :
229
+ try :
230
+ try :
231
+ _readmodule (module .name , self .path , self .inpackage )
232
+ except ImportError :
233
+ _readmodule (module .name , [])
234
+ except (ImportError , SyntaxError ):
235
+ # If we can't find or parse the imported module,
236
+ # too bad -- don't die here.
237
+ continue
238
+
239
+ def visit_ImportFrom (self , node ):
240
+ if node .col_offset != 0 :
241
+ return
242
+ try :
243
+ module = "." * node .level
244
+ if node .module :
245
+ module += node .module
246
+ module = _readmodule (module , self .path , self .inpackage )
247
+ except (ImportError , SyntaxError ):
248
+ return
249
+
250
+ for name in node .names :
251
+ if name .name in module :
252
+ self .tree [name .asname or name .name ] = module [name .name ]
253
+ elif name .name == "*" :
254
+ for import_name , import_value in module .items ():
255
+ if import_name .startswith ("_" ):
256
+ continue
257
+ self .tree [import_name ] = import_value
195
258
196
- stack = [] # Initialize stack of (class, indent) pairs.
197
259
198
- g = tokenize .generate_tokens (f .readline )
199
- try :
200
- for tokentype , token , start , _end , _line in g :
201
- if tokentype == DEDENT :
202
- lineno , thisindent = start
203
- # Close previous nested classes and defs.
204
- while stack and stack [- 1 ][1 ] >= thisindent :
205
- del stack [- 1 ]
206
- elif token == 'def' :
207
- lineno , thisindent = start
208
- # Close previous nested classes and defs.
209
- while stack and stack [- 1 ][1 ] >= thisindent :
210
- del stack [- 1 ]
211
- tokentype , func_name , start = next (g )[0 :3 ]
212
- if tokentype != NAME :
213
- continue # Skip def with syntax error.
214
- cur_func = None
215
- if stack :
216
- cur_obj = stack [- 1 ][0 ]
217
- cur_func = _nest_function (cur_obj , func_name , lineno )
218
- else :
219
- # It is just a function.
220
- cur_func = Function (fullmodule , func_name , fname , lineno )
221
- tree [func_name ] = cur_func
222
- stack .append ((cur_func , thisindent ))
223
- elif token == 'class' :
224
- lineno , thisindent = start
225
- # Close previous nested classes and defs.
226
- while stack and stack [- 1 ][1 ] >= thisindent :
227
- del stack [- 1 ]
228
- tokentype , class_name , start = next (g )[0 :3 ]
229
- if tokentype != NAME :
230
- continue # Skip class with syntax error.
231
- # Parse what follows the class name.
232
- tokentype , token , start = next (g )[0 :3 ]
233
- inherit = None
234
- if token == '(' :
235
- names = [] # Initialize list of superclasses.
236
- level = 1
237
- super = [] # Tokens making up current superclass.
238
- while True :
239
- tokentype , token , start = next (g )[0 :3 ]
240
- if token in (')' , ',' ) and level == 1 :
241
- n = "" .join (super )
242
- if n in tree :
243
- # We know this super class.
244
- n = tree [n ]
245
- else :
246
- c = n .split ('.' )
247
- if len (c ) > 1 :
248
- # Super class form is module.class:
249
- # look in module for class.
250
- m = c [- 2 ]
251
- c = c [- 1 ]
252
- if m in _modules :
253
- d = _modules [m ]
254
- if c in d :
255
- n = d [c ]
256
- names .append (n )
257
- super = []
258
- if token == '(' :
259
- level += 1
260
- elif token == ')' :
261
- level -= 1
262
- if level == 0 :
263
- break
264
- elif token == ',' and level == 1 :
265
- pass
266
- # Only use NAME and OP (== dot) tokens for type name.
267
- elif tokentype in (NAME , OP ) and level == 1 :
268
- super .append (token )
269
- # Expressions in the base list are not supported.
270
- inherit = names
271
- if stack :
272
- cur_obj = stack [- 1 ][0 ]
273
- cur_class = _nest_class (
274
- cur_obj , class_name , lineno , inherit )
275
- else :
276
- cur_class = Class (fullmodule , class_name , inherit ,
277
- fname , lineno )
278
- tree [class_name ] = cur_class
279
- stack .append ((cur_class , thisindent ))
280
- elif token == 'import' and start [1 ] == 0 :
281
- modules = _getnamelist (g )
282
- for mod , _mod2 in modules :
283
- try :
284
- # Recursively read the imported module.
285
- if inpackage is None :
286
- _readmodule (mod , path )
287
- else :
288
- try :
289
- _readmodule (mod , path , inpackage )
290
- except ImportError :
291
- _readmodule (mod , [])
292
- except :
293
- # If we can't find or parse the imported module,
294
- # too bad -- don't die here.
295
- pass
296
- elif token == 'from' and start [1 ] == 0 :
297
- mod , token = _getname (g )
298
- if not mod or token != "import" :
299
- continue
300
- names = _getnamelist (g )
301
- try :
302
- # Recursively read the imported module.
303
- d = _readmodule (mod , path , inpackage )
304
- except :
305
- # If we can't find or parse the imported module,
306
- # too bad -- don't die here.
307
- continue
308
- # Add any classes that were defined in the imported module
309
- # to our name space if they were mentioned in the list.
310
- for n , n2 in names :
311
- if n in d :
312
- tree [n2 or n ] = d [n ]
313
- elif n == '*' :
314
- # Don't add names that start with _.
315
- for n in d :
316
- if n [0 ] != '_' :
317
- tree [n ] = d [n ]
318
- except StopIteration :
319
- pass
320
-
321
- f .close ()
322
- return tree
323
-
324
-
325
- def _getnamelist (g ):
326
- """Return list of (dotted-name, as-name or None) tuples for token source g.
327
-
328
- An as-name is the name that follows 'as' in an as clause.
329
- """
330
- names = []
331
- while True :
332
- name , token = _getname (g )
333
- if not name :
334
- break
335
- if token == 'as' :
336
- name2 , token = _getname (g )
337
- else :
338
- name2 = None
339
- names .append ((name , name2 ))
340
- while token != "," and "\n " not in token :
341
- token = next (g )[1 ]
342
- if token != "," :
343
- break
344
- return names
345
-
346
-
347
- def _getname (g ):
348
- "Return (dotted-name or None, next-token) tuple for token source g."
349
- parts = []
350
- tokentype , token = next (g )[0 :2 ]
351
- if tokentype != NAME and token != '*' :
352
- return (None , token )
353
- parts .append (token )
354
- while True :
355
- tokentype , token = next (g )[0 :2 ]
356
- if token != '.' :
357
- break
358
- tokentype , token = next (g )[0 :2 ]
359
- if tokentype != NAME :
360
- break
361
- parts .append (token )
362
- return ("." .join (parts ), token )
260
+ def _create_tree (fullmodule , path , fname , source , tree , inpackage ):
261
+ mbrowser = _ModuleBrowser (fullmodule , path , fname , tree , inpackage )
262
+ mbrowser .visit (ast .parse (source ))
263
+ return mbrowser .tree
363
264
364
265
365
266
def _main ():
0 commit comments