8000 gh-90110: Fix the c-analyzer Tool (gh-96731) · python/cpython@1756ffd · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 1756ffd

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-90110: Fix the c-analyzer Tool (gh-96731)
This includes: * update the whitelists * fixes so we can stop ignoring some of the files * ensure Include/cpython/*.h get analyzed
1 parent 662782e commit 1756ffd

File tree

10 files changed

+497
-116
lines changed

10 files changed

+497
-116
lines changed

‎Include/internal/pycore_interp.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ extern "C" {
1818
#include "pycore_exceptions.h" // struct _Py_exc_state
1919
#include "pycore_floatobject.h" // struct _Py_float_state
2020
#include "pycore_genobject.h" // struct _Py_async_gen_state
21-
#include "pycore_gil.h" // struct _gil_runtime_state
2221
#include "pycore_gc.h" // struct _gc_runtime_state
2322
#include "pycore_list.h" // struct _Py_list_state
2423
#include "pycore_tuple.h" // struct _Py_tuple_state

‎Tools/c-analyzer/c_common/fsutil.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,25 @@ def format_filename(filename, relroot=USE_CWD, *,
104104
return filename
105105

106106

107+
def match_path_tail(path1, path2):
108+
"""Return True if one path ends the other."""
109+
if path1 == path2:
110+
return True
111+
if os.path.isabs(path1):
112+
if os.path.isabs(path2):
113+
return False
114+
return _match_tail(path1, path2)
115+
elif os.path.isabs(path2):
116+
return _match_tail(path2, path1)
117+
else:
118+
return _match_tail(path1, path2) or _match_tail(path2, path1)
119+
120+
121+
def _match_tail(path, tail):
122+
assert not os.path.isabs(tail), repr(tail)
123+
return path.endswith(os.path.sep + tail)
124+
125+
107126
##################################
108127
# find files
109128

‎Tools/c-analyzer/c_parser/__init__.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ def parse_files(filenames, *,
2222
if get_file_preprocessor is None:
2323
get_file_preprocessor = _get_preprocessor()
2424
for filename in filenames:
25-
yield from _parse_file(
26-
filename, match_kind, get_file_preprocessor, file_maxsizes)
25+
try:
26+
yield from _parse_file(
27+
filename, match_kind, get_file_preprocessor, file_maxsizes)
28+
except Exception:
29+
print(f'# requested file: <{filename}>')
30+
raise # re-raise
2731

2832

2933
def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):

‎Tools/c-analyzer/c_parser/preprocessor/__init__.py

Lines changed: 76 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,27 +35,39 @@
3535

3636
def preprocess(source, *,
3737
incldirs=None,
38+
includes=None,
3839
macros=None,
3940
samefiles=None,
4041
filename=None,
42+
cwd=None,
4143
tool=True,
4244
):
4345
"""...
4446
4547
CWD should be the project root and "source" should be relative.
4648
"""
4749
if tool:
48-
logger.debug(f'CWD: {os.getcwd()!r}')
49-
logger.debug(f'incldirs: {incldirs!r}')
50-
logger.debug(f'macros: {macros!r}')
50+
if not cwd:
51+
cwd = os.getcwd()
52+
logger.debug(f'CWD: {cwd!r}')
53+
logger.debug(f'incldirs: {incldirs!r}')
54+
logger.debug(f'includes: {includes!r}')
55+
logger.debug(f'macros: {macros!r}')
5156
logger.debug(f'samefiles: {samefiles!r}')
5257
_preprocess = _get_preprocessor(tool)
5358
with _good_file(source, filename) as source:
54-
return _preprocess(source, incldirs, macros, samefiles) or ()
59+
return _preprocess(
60+
source,
61+
incldirs,
62+
includes,
63+
macros,
64+
samefiles,
65+
cwd,
66+
) or ()
5567
else:
5668
source, filename = _resolve_source(source, filename)
5769
# We ignore "includes", "macros", etc.
58-
return _pure.preprocess(source, filename)
70+
return _pure.preprocess(source, filename, cwd)
5971

6072
# if _run() returns just the lines:
6173
# text = _run(source)
@@ -72,6 +84,7 @@ def preprocess(source, *,
7284

7385
def get_preprocessor(*,
7486
file_macros=None,
87+
file_includes=None,
7588
file_incldirs=None,
7689
file_same=None,
7790
ignore_exc=False,
@@ -80,27 +93,39 @@ def get_preprocessor(*,
8093
_preprocess = preprocess
8194
if file_macros:
8295
file_macros = tuple(_parse_macros(file_macros))
96+
if file_includes:
97+
file_includes = tuple(_parse_includes(file_includes)) 10000
8398
if file_incldirs:
8499
file_incldirs = tuple(_parse_incldirs(file_incldirs))
85100
if file_same:
86-
file_same = tuple(file_same)
101+
file_same = dict(file_same or ())
87102
if not callable(ignore_exc):
88103
ignore_exc = (lambda exc, _ig=ignore_exc: _ig)
89104

90105
def get_file_preprocessor(filename):
91106
filename = filename.strip()
92107
if file_macros:
93108
macros = list(_resolve_file_values(filename, file_macros))
109+
if file_includes:
110+
# There's a small chance we could need to filter out any
111+
# includes that import "filename". It isn't clear that it's
112+
# a problem any longer. If we do end up filtering then
113+
# it may make sense to use c_common.fsutil.match_path_tail().
114+
includes = [i for i, in _resolve_file_values(filename, file_includes)]
94115
if file_incldirs:
95116
incldirs = [v for v, in _resolve_file_values(filename, file_incldirs)]
117+
if file_same:
118+
samefiles = _resolve_samefiles(filename, file_same)
96119

97120
def preprocess(**kwargs):
98121
if file_macros and 'macros' not in kwargs:
99122
kwargs['macros'] = macros
123+
if file_includes and 'includes' not in kwargs:
124+
kwargs['includes'] = includes
100125
if file_incldirs and 'incldirs' not in kwargs:
101-
kwargs['incldirs'] = [v for v, in _resolve_file_values(filename, file_incldirs)]
102-
if file_same and 'file_same' not in kwargs:
103-
kwargs['samefiles'] = file_same
126+
kwargs['incldirs'] = incldirs
127+
if file_same and 'samefiles' not in kwargs:
128+
kwargs['samefiles'] = samefiles
104129
kwargs.setdefault('filename', filename)
105130
with handling_errors(ignore_exc, log_err=log_err):
106131
return _preprocess(filename, **kwargs)
@@ -120,6 +145,11 @@ def _parse_macros(macros):
120145
yield row
121146

122147

148+
def _parse_includes(includes):
149+
for row, srcfile in _parse_table(includes, '\t', 'glob\tinclude', default=None):
150+
yield row
151+
152+
123153
def _parse_incldirs(incldirs):
124154
for row, srcfile in _parse_table(incldirs, '\t', 'glob\tdirname', default=None):
125155
glob, dirname = row
@@ -130,6 +160,43 @@ def _parse_incldirs(incldirs):
130160
yield row
131161

132162

163+
def _resolve_samefiles(filename, file_same):
164+
assert '*' not in filename, (filename,)
165+
assert os.path.normpath(filename) == filename, (filename,)
166+
_, suffix = os.path.splitext(filename)
167+
samefiles = []
168+
for patterns, in _resolve_file_values(filename, file_same.items()):
169+
for pattern in patterns:
170+
same = _resolve_samefile(filename, pattern, suffix)
171+
if not same:
172+
continue
173+
samefiles.append(same)
174+
return samefiles
175+
176+
177+
def _resolve_samefile(filename, pattern, suffix):
178+
if pattern == filename:
179+
return None
180+
if pattern.endswith(os.path.sep):
181+
pattern += f'*{suffix}'
182+
assert os.path.normpath(pattern) == pattern, (pattern,)
183+
if '*' in os.path.dirname(pattern):
184+
raise NotImplementedError((filename, pattern))
185+
if '*' not in os.path.basename(pattern):
186+
return pattern
187+
188+
common = os.path.commonpath([filename, pattern])
189+
relpattern = pattern[len(common) + len(os.path.sep):]
190+
relpatterndir = os.path.dirname(relpattern)
191+
relfile = filename[len(common) + len(os.path.sep):]
192+
if os.path.basename(pattern) == '*':
193+
return os.path.join(common, relpatterndir, relfile)
194+
elif os.path.basename(relpattern) == '*' + suffix:
195+
return os.path.join(common, relpatterndir, relfile)
196+
else:
197+
raise NotImplementedError((filename, pattern))
198+
199+
133200
@contextlib.contextmanager
134201
def handling_errors(ignore_exc=None, *, log_err=None):
135202
try:

‎Tools/c-analyzer/c_parser/preprocessor/common.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def run_cmd(argv, *,
4444
return proc.stdout
4545

4646

47-
def preprocess(tool, filename, **kwargs):
47+
def preprocess(tool, filename, cwd=None, **kwargs):
4848
argv = _build_argv(tool, filename, **kwargs)
4949
logger.debug(' '.join(shlex.quote(v) for v in argv))
5050

@@ -59,19 +59,24 @@ def preprocess(tool, filename, **kwargs):
5959
# distutil compiler object's preprocess() method, since that
6060
# one writes to stdout/stderr and it's simpler to do it directly
6161
# through subprocess.
62-
return run_cmd(argv)
62+
return run_cmd(argv, cwd=cwd)
6363

6464

6565
def _build_argv(
6666
tool,
6767
filename,
6868
incldirs=None,
69+
includes=None,
6970
macros=None,
7071
preargs=None,
7172
postargs=None,
7273
executable=None,
7374
compiler=None,
7475
):
76+
if includes:
77+
includes = tuple(f'-include{i}' for i in includes)
78+
postargs = (includes + postargs) if postargs else includes
79+
7580
compiler = distutils.ccompiler.new_compiler(
7681
compiler=compiler or tool,
7782
)

0 commit comments

Comments
 (0)
0