Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1756ffd

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-90110: Fix the c-analyzer Tool (gh-96731)
This includes: * update the whitelists * fixes so we can stop ignoring some of the files * ensure Include/cpython/*.h get analyzed
1 parent 662782e commit 1756ffd

10 files changed

Lines changed: 497 additions & 116 deletions

File tree

‎Include/internal/pycore_interp.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ extern "C" {
1818
#include "pycore_exceptions.h" // struct _Py_exc_state
1919
#include "pycore_floatobject.h" // struct _Py_float_state
2020
#include "pycore_genobject.h" // struct _Py_async_gen_state
21-
#include "pycore_gil.h" // struct _gil_runtime_state
2221
#include "pycore_gc.h" // struct _gc_runtime_state
2322
#include "pycore_list.h" // struct _Py_list_state
2423
#include "pycore_tuple.h" // struct _Py_tuple_state

‎Tools/c-analyzer/c_common/fsutil.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,25 @@ def format_filename(filename, relroot=USE_CWD, *,
104104
return filename
105105

106106

107+
def match_path_tail(path1, path2):
108+
"""Return True if one path ends the other."""
109+
if path1 == path2:
110+
return True
111+
if os.path.isabs(path1):
112+
if os.path.isabs(path2):
113+
return False
114+
return _match_tail(path1, path2)
115+
elif os.path.isabs(path2):
116+
return _match_tail(path2, path1)
117+
else:
118+
return _match_tail(path1, path2) or _match_tail(path2, path1)
119+
120+
121+
def _match_tail(path, tail):
122+
assert not os.path.isabs(tail), repr(tail)
123+
return path.endswith(os.path.sep + tail)
124+
125+
107126
##################################
108127
# find files
109128

‎Tools/c-analyzer/c_parser/__init__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,12 @@ def parse_files(filenames, *,
2222
if get_file_preprocessor is None:
2323
get_file_preprocessor = _get_preprocessor()
2424
for filename in filenames:
25-
yield from _parse_file(
26-
filename, match_kind, get_file_preprocessor, file_maxsizes)
25+
try:
26+
yield from _parse_file(
27+
filename, match_kind, get_file_preprocessor, file_maxsizes)
28+
except Exception:
29+
print(f'# requested file: <{filename}>')
30+
raise # re-raise
2731

2832

2933
def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):

‎Tools/c-analyzer/c_parser/preprocessor/__init__.py‎

Lines changed: 76 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,27 +35,39 @@
3535

3636
def preprocess(source, *,
3737
incldirs=None,
38+
includes=None,
3839
macros=None,
3940
samefiles=None,
4041
filename=None,
42+
cwd=None,
4143
tool=True,
4244
):
4345
"""...
4446
4547
CWD should be the project root and "source" should be relative.
4648
"""
4749
if tool:
48-
logger.debug(f'CWD: {os.getcwd()!r}')
49-
logger.debug(f'incldirs: {incldirs!r}')
50-
logger.debug(f'macros: {macros!r}')
50+
if not cwd:
51+
cwd = os.getcwd()
52+
logger.debug(f'CWD: {cwd!r}')
53+
logger.debug(f'incldirs: {incldirs!r}')
54+
logger.debug(f'includes: {includes!r}')
55+
logger.debug(f'macros: {macros!r}')
5156
logger.debug(f'samefiles: {samefiles!r}')
5257
_preprocess = _get_preprocessor(tool)
5358
with _good_file(source, filename) as source:
54-
return _preprocess(source, incldirs, macros, samefiles) or ()
59+
return _preprocess(
60+
source,
61+
incldirs,
62+
includes,
63+
macros,
64+
samefiles,
65+
cwd,
66+
) or ()
5567
else:
5668
source, filename = _resolve_source(source, filename)
5769
# We ignore "includes", "macros", etc.
58-
return _pure.preprocess(source, filename)
70+
return _pure.preprocess(source, filename, cwd)
5971

6072
# if _run() returns just the lines:
6173
# text = _run(source)
@@ -72,6 +84,7 @@ def preprocess(source, *,
7284

7385
def get_preprocessor(*,
7486
file_macros=None,
87+
file_includes=None,
7588
file_incldirs=None,
7689
file_same=None,
7790
ignore_exc=False,
@@ -80,27 +93,39 @@ def get_preprocessor(*,
8093
_preprocess = preprocess
8194
if file_macros:
8295
file_macros = tuple(_parse_macros(file_macros))
96+
if file_includes:
97+
file_includes = tuple(_parse_includes(file_includes))
8398
if file_incldirs:
8499
file_incldirs = tuple(_parse_incldirs(file_incldirs))
85100
if file_same:
86-
file_same = tuple(file_same)
101+
file_same = dict(file_same or ())
87102
if not callable(ignore_exc):
88103
ignore_exc = (lambda exc, _ig=ignore_exc: _ig)
89104

90105
def get_file_preprocessor(filename):
91106
filename = filename.strip()
92107
if file_macros:
93108
macros = list(_resolve_file_values(filename, file_macros))
109+
if file_includes:
110+
# There's a small chance we could need to filter out any
111+
# includes that import "filename". It isn't clear that it's
112+
# a problem any longer. If we do end up filtering then
113+
# it may make sense to use c_common.fsutil.match_path_tail().
114+
includes = [i for i, in _resolve_file_values(filename, file_includes)]
94115
if file_incldirs:
95116
incldirs = [v for v, in _resolve_file_values(filename, file_incldirs)]
117+
if file_same:
118+
samefiles = _resolve_samefiles(filename, file_same)
96119

97120
def preprocess(**kwargs):
98121
if file_macros and 'macros' not in kwargs:
99122
kwargs['macros'] = macros
123+
if file_includes and 'includes' not in kwargs:
124+
kwargs['includes'] = includes
100125
if file_incldirs and 'incldirs' not in kwargs:
101-
kwargs['incldirs'] = [v for v, in _resolve_file_values(filename, file_incldirs)]
102-
if file_same and 'file_same' not in kwargs:
103-
kwargs['samefiles'] = file_same
126+
kwargs['incldirs'] = incldirs
127+
if file_same and 'samefiles' not in kwargs:
128+
kwargs['samefiles'] = samefiles
104129
kwargs.setdefault('filename', filename)
105130
with handling_errors(ignore_exc, log_err=log_err):
106131
return _preprocess(filename, **kwargs)
@@ -120,6 +145,11 @@ def _parse_macros(macros):
120145
yield row
121146

122147

148+
def _parse_includes(includes):
149+
for row, srcfile in _parse_table(includes, '\t', 'glob\tinclude', default=None):
150+
yield row
151+
152+
123153
def _parse_incldirs(incldirs):
124154
for row, srcfile in _parse_table(incldirs, '\t', 'glob\tdirname', default=None):
125155
glob, dirname = row
@@ -130,6 +160,43 @@ def _parse_incldirs(incldirs):
130160
yield row
131161

132162

163+
def _resolve_samefiles(filename, file_same):
164+
assert '*' not in filename, (filename,)
165+
assert os.path.normpath(filename) == filename, (filename,)
166+
_, suffix = os.path.splitext(filename)
167+
samefiles = []
168+
for patterns, in _resolve_file_values(filename, file_same.items()):
169+
for pattern in patterns:
170+
same = _resolve_samefile(filename, pattern, suffix)
171+
if not same:
172+
continue
173+
samefiles.append(same)
174+
return samefiles
175+
176+
177+
def _resolve_samefile(filename, pattern, suffix):
178+
if pattern == filename:
179+
return None
180+
if pattern.endswith(os.path.sep):
181+
pattern += f'*{suffix}'
182+
assert os.path.normpath(pattern) == pattern, (pattern,)
183+
if '*' in os.path.dirname(pattern):
184+
raise NotImplementedError((filename, pattern))
185+
if '*' not in os.path.basename(pattern):
186+
return pattern
187+
188+
common = os.path.commonpath([filename, pattern])
189+
relpattern = pattern[len(common) + len(os.path.sep):]
190+
relpatterndir = os.path.dirname(relpattern)
191+
relfile = filename[len(common) + len(os.path.sep):]
192+
if os.path.basename(pattern) == '*':
193+
return os.path.join(common, relpatterndir, relfile)
194+
elif os.path.basename(relpattern) == '*' + suffix:
195+
return os.path.join(common, relpatterndir, relfile)
196+
else:
197+
raise NotImplementedError((filename, pattern))
198+
199+
133200
@contextlib.contextmanager
134201
def handling_errors(ignore_exc=None, *, log_err=None):
135202
try:

‎Tools/c-analyzer/c_parser/preprocessor/common.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def run_cmd(argv, *,
4444
return proc.stdout
4545

4646

47-
def preprocess(tool, filename, **kwargs):
47+
def preprocess(tool, filename, cwd=None, **kwargs):
4848
argv = _build_argv(tool, filename, **kwargs)
4949
logger.debug(' '.join(shlex.quote(v) for v in argv))
5050

@@ -59,19 +59,24 @@ def preprocess(tool, filename, **kwargs):
5959
# distutil compiler object's preprocess() method, since that
6060
# one writes to stdout/stderr and it's simpler to do it directly
6161
# through subprocess.
62-
return run_cmd(argv)
62+
return run_cmd(argv, cwd=cwd)
6363

6464

6565
def _build_argv(
6666
tool,
6767
filename,
6868
incldirs=None,
69+
includes=None,
6970
macros=None,
7071
preargs=None,
7172
postargs=None,
7273
executable=None,
7374
compiler=None,
7475
):
76+
if includes:
77+
includes = tuple(f'-include{i}' for i in includes)
78+
postargs = (includes + postargs) if postargs else includes
79+
7580
compiler = distutils.ccompiler.new_compiler(
7681
compiler=compiler or tool,
7782
)

0 commit comments

Comments
 (0)