Generate the ignored.tsv file.

python · ericsnowcurrently · Sep 11, 2019 · Jun 21, 2019 · Jun 22, 2019 · Jun 22, 2019
commit fc6c97ec9955794586978bbb0f4e0e23ecf0ca1d
diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer_common/_generate.py
@@ -1,10 +1,5 @@
 # The code here consists of hacks for pre-populating the known.tsv file.
 
-import contextlib
-import glob
-import os.path
-import re
-
 from c_parser.preprocessor import _iter_clean_lines
 from c_parser.naive import (
         iter_variables, parse_variable_declaration, find_variables,
@@ -14,7 +9,7 @@
 from . import SOURCE_DIRS, REPO_ROOT
 from .known import DATA_FILE as KNOWN_FILE, HEADER as KNOWN_HEADER
 from .info import UNKNOWN, ID
-from .util import run_cmd, write_tsv
+from .util import write_tsv
 from .files import iter_cpython_files
 
 
@@ -311,14 +306,15 @@ def known_rows(symbols, *,
             yield _as_known(variable.id, variable.vartype)
 
 
-def known_file(symbols, filename=None, *,
-               _generate_rows=known_rows,
-               ):
+def generate(symbols, filename=None, *,
+             _generate_rows=known_rows,
+             _write_tsv=write_tsv,
+             ):
     if not filename:
         filename = KNOWN_FILE + '.new'
 
     rows = _generate_rows(symbols)
-    write_tsv(filename, KNOWN_HEADER, rows)
+    _write_tsv(filename, KNOWN_HEADER, rows)
 
 
 if __name__ == '__main__':
@@ -327,4 +323,4 @@ def known_file(symbols, filename=None, *,
             binary.PYTHON,
             find_local_symbol=None,
             )
-    known_file(symbols)
+    generate(symbols)
diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py
@@ -77,7 +77,7 @@ def _find_globals(dirnames, known, ignored):
     unknown = set()
     knownvars = (known or {}).get('variables')
     for variable in find.globals_from_binary(knownvars=knownvars,
-                                           dirnames=dirnames):
+                                             dirnames=dirnames):
     #for variable in find.globals(dirnames, known, kind='platform'):
         if variable.vartype == UNKNOWN:
             unknown.add(variable)

diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/c_globals/supported.py
@@ -1,133 +1,147 @@
-import csv
 import os.path
 import re
 
 from c_analyzer_common import DATA_DIR
 from c_analyzer_common.info import ID
-from c_analyzer_common.util import read_tsv
+from c_analyzer_common.util import read_tsv, write_tsv
 
 
-def is_supported(variable, ignored=None, known=None):
-    """Return True if the given global variable is okay in CPython."""
-    if _is_ignored(variable, ignored and ignored.get('variables')):
-        return True
-    elif _is_vartype_okay(variable.vartype, ignored.get('types')):
-        return True
-    else:
-        return False
+IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv')
 
+IGNORED_COLUMNS = ('filename', 'funcname', 'name', 'kind', 'reason')
+IGNORED_HEADER = '\t'.join(IGNORED_COLUMNS)
 
 # XXX Move these to ignored.tsv.
 IGNORED = {
         # global
-        'PyImport_FrozenModules',
-        'M___hello__',
-        'inittab_copy',
-        'PyHash_Func',
-        '_Py_HashSecret_Initialized',
-        '_TARGET_LOCALES',
-        'runtime_initialized',
+        'PyImport_FrozenModules': 'process-global',
+        'M___hello__': 'process-global',
+        'inittab_copy': 'process-global',
+        'PyHash_Func': 'process-global',
+        '_Py_HashSecret_Initialized': 'process-global',
+        '_TARGET_LOCALES': 'process-global',
 
         # startup
-        'static_arg_parsers',
-        'orig_argv',
-        'opt_ptr',
-        '_preinit_warnoptions',
-        '_Py_StandardStreamEncoding',
-        '_Py_StandardStreamErrors',
+        'runtime_initialized': 'runtime startup',
+        'static_arg_parsers': 'runtime startup',
+        'orig_argv': 'runtime startup',
+        'opt_ptr': 'runtime startup',
+        '_preinit_warnoptions': 'runtime startup',
+        '_Py_StandardStreamEncoding': 'runtime startup',
+        '_Py_StandardStreamErrors': 'runtime startup',
 
         # should be const
-        'tracemalloc_empty_traceback',
-        '_empty_bitmap_node',
-        'posix_constants_pathconf',
-        'posix_constants_confstr',
-        'posix_constants_sysconf',
+        'tracemalloc_empty_traceback': 'const',
+        '_empty_bitmap_node': 'const',
+        'posix_constants_pathconf': 'const',
+        'posix_constants_confstr': 'const',
+        'posix_constants_sysconf': 'const',
 
         # signals are main-thread only
-        'faulthandler_handlers',
-        'user_signals',
+        'faulthandler_handlers': 'signals are main-thread only',
+        'user_signals': 'signals are main-thread only',
         }
 
 
-def _is_ignored(variable, ignoredvars=None):
-    if variable.name in IGNORED:
+def is_supported(variable, ignored=None, known=None, *,
+                 _ignored=(lambda *a, **k: _is_ignored(*a, **k)),
+                 _vartype_okay=(lambda *a, **k: _is_vartype_okay(*a, **k)),
+                 ):
+    """Return True if the given global variable is okay in CPython."""
+    if _ignored(variable,
+                ignored and ignored.get('variables')):
         return True
-
-    if ignoredvars and variable.id in ignoredvars:
+    elif _vartype_okay(variable.vartype,
+                       ignored.get('types')):
         return True
+    else:
+        return False
+
+
+def _is_ignored(variable, ignoredvars=None, *,
+                _IGNORED=IGNORED,
+                ):
+    """Return the reason if the variable is a supported global.
+
+    Return None if the variable is not a supported global.
+    """
+    if ignoredvars and (reason := ignoredvars.get(variable.id)):
+        return reason
+
+    if variable.funcname is None:
+        if reason := _IGNORED.get(variable.name):
+            return reason
 
     # compiler
     if variable.filename == 'Python/graminit.c':
         if variable.vartype.startswith('static state '):
-            return True
+            return 'compiler'
     if variable.filename == 'Python/symtable.c':
         if variable.vartype.startswith('static identifier '):
-            return True
+            return 'compiler'
     if variable.filename == 'Python/Python-ast.c':
         # These should be const.
         if variable.name.endswith('_field'):
-            return True
+            return 'compiler'
         if variable.name.endswith('_attribute'):
-            return True
+            return 'compiler'
 
     # other
     if variable.filename == 'Python/dtoa.c':
         # guarded by lock?
         if variable.name in ('p5s', 'freelist'):
-            return True
+            return 'dtoa is thread-safe?'
         if variable.name in ('private_mem', 'pmem_next'):
-            return True
+            return 'dtoa is thread-safe?'
 
-    return False
+    return None
 
 
 def _is_vartype_okay(vartype, ignoredtypes=None):
     if _is_object(vartype):
-        return False
+        return None
 
     if vartype.startswith('static const '):
-        return True
+        return 'const'
 
     # components for TypeObject definitions
     for name in ('PyMethodDef', 'PyGetSetDef', 'PyMemberDef'):
         if name in vartype:
-            return True
+            return 'const'
     for name in ('PyNumberMethods', 'PySequenceMethods', 'PyMappingMethods',
                  'PyBufferProcs', 'PyAsyncMethods'):
         if name in vartype:
-            return True
+            return 'const'
     for name in ('slotdef', 'newfunc'):
         if name in vartype:
-            return True
+            return 'const'
 
     # structseq
     for name in ('PyStructSequence_Desc', 'PyStructSequence_Field'):
         if name in vartype:
-            return True
+            return 'const'
 
     # other definiitions
     if 'PyModuleDef' in vartype:
-        return True
+        return 'const'
 
     # thread-safe
     if '_Py_atomic_int' in vartype:
-        return True
+        return 'thread-safe'
     if 'pthread_condattr_t' in vartype:
-        return True
+        return 'thread-safe'
 
     # startup
     if '_Py_PreInitEntry' in va
10000
rtype:
-        return True
+        return 'startup'
 
     # global
-    if 'PyMemAllocatorEx' in vartype:
-        return True
+#    if 'PyMemAllocatorEx' in vartype:
+#        return True
 
     # others
-    if 'PyThread_type_lock' in vartype:
-        return True
-    #if '_Py_hashtable_t' in vartype:
-    #    return True  # ???
+#    if 'PyThread_type_lock' in vartype:
+#        return True
 
     # XXX ???
     # _Py_tss_t
@@ -137,12 +151,12 @@ def _is_vartype_okay(vartype, ignoredtypes=None):
 
     # functions
     if '(' in vartype and '[' not in vartype:
-        return True
+        return 'function pointer'
 
     # XXX finish!
     # * allow const values?
     #raise NotImplementedError
-    return False
+    return None
 
 
 def _is_object(vartype):
@@ -172,26 +186,17 @@ def _is_object(vartype):
     return False
 
 
-#############################
-# ignored
-
-IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv')
-
-COLUMNS = ('filename', 'funcname', 'name', 'kind', 'reason')
-HEADER = '\t'.join(COLUMNS)
-
-
 def ignored_from_file(infile, *,
                       _read_tsv=read_tsv,
                       ):
-    """Yield StaticVar for each ignored var in the file."""
+    """Yield a Variable for each ignored var in the file."""
     ignored = {
         'variables': {},
         #'types': {},
         #'constants': {},
         #'macros': {},
         }
-    for row in _read_tsv(infile, HEADER):
+    for row in _read_tsv(infile, IGNORED_HEADER):
         filename, funcname, name, kind, reason = row
         if not funcname or funcname == '-':
             funcname = None
@@ -202,3 +207,63 @@ def ignored_from_file(infile, *,
             raise ValueError(f'unsupported kind in row {row}')
         values[id] = reason
     return ignored
+
+
+##################################
+# generate
+
+def _get_row(varid, reason):
+    return (
+            varid.filename,
+            varid.funcname or '-',
+            varid.name,
+            'variable',
+            str(reason),
+            )
+
+
+def _get_rows(variables, ignored=None, *,
+              _as_row=_get_row,
+              _is_ignored=_is_ignored,
+              _vartype_okay=_is_vartype_okay,
+              ):
+    count = 0
+    for variable in variables:
+        reason = _is_ignored(variable,
+                             ignored and ignored.get('variables'),
+                             )
+        if not reason:
+            reason = _vartype_okay(variable.vartype,
+                                   ignored and ignored.get('types'))
+        if not reason:
+            continue
+
+        print(' ', variable, repr(reason))
+        yield _as_row(variable.id, reason)
+        count += 1
+    print(f'total: {count}')
+
+
+def _generate_ignored_file(variables, filename=None, *,
+                           _generate_rows=_get_rows,
+                           _write_tsv=write_tsv,
+                           ):
+    if not filename:
+        filename = IGNORED_FILE + '.new'
+    rows = _generate_rows(variables)
+    _write_tsv(filename, IGNORED_HEADER, rows)
+
+
+if __name__ == '__main__':
+    from c_analyzer_common import SOURCE_DIRS
+    from c_analyzer_common.known import (
+        from_file as known_from_file,
+        DATA_FILE as KNOWN_FILE,
+        )
+    from . import find
+    known = known_from_file(KNOWN_FILE)
+    knownvars = (known or {}).get('variables')
+    variables = find.globals_from_binary(knownvars=knownvars,
+                                         dirnames=SOURCE_DIRS)
+
+    _generate_ignored_file(variables)