From 169c822cf33dba742111df3279ac34dfb8b192fc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 13:11:35 -0700 Subject: [PATCH 01/17] Factor out the cpython._files module. --- Tools/c-analyzer/cpython/_files.py | 38 +++++++++++++++++++++++++++++ Tools/c-analyzer/cpython/_parser.py | 36 +-------------------------- 2 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 Tools/c-analyzer/cpython/_files.py diff --git a/Tools/c-analyzer/cpython/_files.py b/Tools/c-analyzer/cpython/_files.py new file mode 100644 index 00000000000000..8eb0f18308588a --- /dev/null +++ b/Tools/c-analyzer/cpython/_files.py @@ -0,0 +1,38 @@ +import os.path + +from c_common.fsutil import expand_filenames, iter_files_by_suffix +from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS + + +GLOBS = [ + 'Include/*.h', + 'Include/internal/*.h', + 'Modules/**/*.h', + 'Modules/**/*.c', + 'Objects/**/*.h', + 'Objects/**/*.c', + 'Python/**/*.h', + 'Parser/**/*.c', + 'Python/**/*.h', + 'Parser/**/*.c', +] + + +def resolve_filename(filename): + orig = filename + filename = os.path.normcase(os.path.normpath(filename)) + if os.path.isabs(filename): + if os.path.relpath(filename, REPO_ROOT).startswith('.'): + raise Exception(f'{orig!r} is outside the repo ({REPO_ROOT})') + return filename + else: + return os.path.join(REPO_ROOT, filename) + + +def iter_filenames(*, search=False): + if search: + yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',)) + yield from iter_files_by_suffix(SOURCE_DIRS, ('.c',)) + else: + globs = (os.path.join(REPO_ROOT, file) for file in GLOBS) + yield from expand_filenames(globs) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index eef758495386c4..ef06a9fcb69033 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -1,7 +1,6 @@ import os.path import re -from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_parser.preprocessor import ( get_preprocessor as _get_preprocessor, ) @@ -9,7 +8,7 @@ parse_file as _parse_file, parse_files as _parse_files, ) -from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS +from . import REPO_ROOT GLOB_ALL = '**/*' @@ -43,19 +42,6 @@ def clean_lines(text): @end=sh@ ''' -GLOBS = [ - 'Include/*.h', - 'Include/internal/*.h', - 'Modules/**/*.h', - 'Modules/**/*.c', - 'Objects/**/*.h', - 'Objects/**/*.c', - 'Python/**/*.h', - 'Parser/**/*.c', - 'Python/**/*.h', - 'Parser/**/*.c', -] - EXCLUDED = clean_lines(''' # @begin=conf@ @@ -280,26 +266,6 @@ def clean_lines(text): ] -def resolve_filename(filename): - orig = filename - filename = os.path.normcase(os.path.normpath(filename)) - if os.path.isabs(filename): - if os.path.relpath(filename, REPO_ROOT).startswith('.'): - raise Exception(f'{orig!r} is outside the repo ({REPO_ROOT})') - return filename - else: - return os.path.join(REPO_ROOT, filename) - - -def iter_filenames(*, search=False): - if search: - yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',)) - yield from iter_files_by_suffix(SOURCE_DIRS, ('.c',)) - else: - globs = (os.path.join(REPO_ROOT, file) for file in GLOBS) - yield from expand_filenames(globs) - - def get_preprocessor(*, file_macros=None, file_incldirs=None, From 042e756df38b8ac1f1c1620c3db84ed6395a4b87 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 18:40:43 -0700 Subject: [PATCH 02/17] Add table rendering for CLI use. --- Tools/c-analyzer/c_common/tables.py | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/Tools/c-analyzer/c_common/tables.py b/Tools/c-analyzer/c_common/tables.py index 411152e3f9498f..b9e18e8e6d8d56 100644 --- a/Tools/c-analyzer/c_common/tables.py +++ b/Tools/c-analyzer/c_common/tables.py @@ -212,3 +212,117 @@ def _normalize_table_file_props(header, sep): else: sep = None return header, sep + + +################################## +# stdout tables + +WIDTH = 20 + + +def build_table(columns, *, sep=' '): + if isinstance(columns, str): + columns = columns.replace(',', ' ').strip().split() + columns = _parse_columns(columns) + return _build_table(columns, sep=sep) + + +def _parse_colspec(raw): + width = align = fmt = None + if raw.isdigit(): + width = int(raw) + align = 'left' + elif raw == '<': + align = 'left' + elif raw == '^': + align = 'middle' + elif raw == '>': + align = 'right' + else: + # XXX Handle combined specs. + fmt = raw + return width, align, fmt + + +def _render_colspec(spec): + if not spec: + return '' + width, align, colfmt = spec + + parts = [] + if align: + if align == 'left': + align = '<' + elif align == 'middle': + align = '^' + elif align == 'right': + align = '>' + else: + raise NotImplementedError + parts.append(align) + if width: + parts.append(str(width)) + if colfmt: + raise NotImplementedError + return ''.join(parts) + + +def _parse_column(raw): + if isinstance(raw, str): + colname, _, rawspec = raw.partition(':') + spec = _parse_colspec(rawspec) + return (colname, spec) + else: + raise NotImplementedError + + +def _render_column(colname, spec): + spec = _render_colspec(spec) + if spec: + return f'{colname}:{spec}' + else: + return colname + + +def _parse_columns(columns): + parsed = [] + for raw in columns: + column = _parse_column(raw) + parsed.append(column) + return parsed + + +def _resolve_width(width, colname, defaultwidth): + if width: + if not isinstance(width, int): + raise NotImplementedError + return width + + if not defaultwidth: + return WIDTH + elif not hasattr(defaultwidth, 'get'): + return defaultwidth or WIDTH + + defaultwidths = defaultwidth + defaultwidth = defaultwidths.get(None) or WIDTH + return defaultwidths.get(colname) or defaultwidth + + +def _build_table(columns, *, sep=' ', defaultwidth=None): + header = [] + div = [] + rowfmt = [] + for colname, spec in columns: + width, align, colfmt = spec + width = _resolve_width(width, colname, defaultwidth) + if width != spec[0]: + spec = width, align, colfmt + + header.append(f' {{:^{width}}} '.format(colname)) + div.append('-' * (width + 2)) + rowfmt.append(' {' + _render_column(colname, spec) + '} ') + return ( + sep.join(header), + sep.join(div), + sep.join(rowfmt), + ) From acc696de733b0adf2f79d9808d07d6598d23d218 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 18:41:51 -0700 Subject: [PATCH 03/17] Add iter_header_files(). --- Tools/c-analyzer/cpython/_files.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Tools/c-analyzer/cpython/_files.py b/Tools/c-analyzer/cpython/_files.py index 8eb0f18308588a..3e397880977ab2 100644 --- a/Tools/c-analyzer/cpython/_files.py +++ b/Tools/c-analyzer/cpython/_files.py @@ -16,6 +16,11 @@ 'Python/**/*.h', 'Parser/**/*.c', ] +LEVEL_GLOBS = { + 'stable': 'Include/*.h', + 'cpython': 'Include/cpython/*.h', + 'internal': 'Include/internal/*.h', +} def resolve_filename(filename): @@ -36,3 +41,29 @@ def iter_filenames(*, search=False): else: globs = (os.path.join(REPO_ROOT, file) for file in GLOBS) yield from expand_filenames(globs) + + +def iter_header_files(filenames=None, *, levels=None): + if not filenames: + if levels: + levels = set(levels) + if 'private' in levels: + levels.add('stable') + levels.add('cpython') + for level, glob in LEVEL_GLOBS.items(): + if level in levels: + yield from expand_filenames([glob]) + else: + yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',)) + return + + for filename in filenames: + orig = filename + filename = resolve_filename(filename) + if filename.endswith(os.path.sep): + yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',)) + elif filename.endswith('.h'): + yield filename + else: + # XXX Log it and continue instead? + raise ValueError(f'expected .h file, got {orig!r}') From 35171bee49bd4b50e2068bf5859495f1190a6f11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 18:42:55 -0700 Subject: [PATCH 04/17] Add helpers for C-API analysis. --- Tools/c-analyzer/cpython/_capi.py | 248 ++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 Tools/c-analyzer/cpython/_capi.py diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py new file mode 100644 index 00000000000000..89cf8bdf7a4946 --- /dev/null +++ b/Tools/c-analyzer/cpython/_capi.py @@ -0,0 +1,248 @@ +from collections import namedtuple +import os.path +import re +import textwrap + +from c_common.tables import build_table +from c_parser.parser._regexes import _ind +from ._files import iter_header_files, resolve_filename +from . import REPO_ROOT + + +INCLUDE_ROOT = os.path.join(REPO_ROOT, 'Include') +INCLUDE_CPYTHON = os.path.join(INCLUDE_ROOT, 'cpython') +INCLUDE_INTERNAL = os.path.join(INCLUDE_ROOT, 'internal') + +_MAYBE_NESTED_PARENS = textwrap.dedent(r''' + (?: + (?: [^(]* [(] [^()]* [)] )* [^(]* + ) +''') + +CAPI_FUNC = textwrap.dedent(rf''' + (?: + ^ + \s* + PyAPI_FUNC \s* + [(] + {_ind(_MAYBE_NESTED_PARENS, 2)} + [)] \s* + (\w+) # + \s* [(] + ) +''') +CAPI_DATA = textwrap.dedent(rf''' + (?: + ^ + \s* + PyAPI_DATA \s* + [(] + {_ind(_MAYBE_NESTED_PARENS, 2)} + [)] \s* + (\w+) # + \b [^(] + ) +''') +CAPI_MACRO = textwrap.dedent(r''' + (?: + (\w+) # + \s* [(] + ) +''') +CAPI_CONSTANT = textwrap.dedent(r''' + (?: + (\w+) # + \s+ [^(] + ) +''') +CAPI_DEFINE = textwrap.dedent(rf''' + (?: + ^ + \s* [#] \s* define \s+ + (?: + {_ind(CAPI_MACRO, 3)} + | + {_ind(CAPI_CONSTANT, 3)} + | + (?: + # ignored + \w+ # + \s* + $ + ) + ) + ) +''') +CAPI_RE = re.compile(textwrap.dedent(rf''' + (?: + {_ind(CAPI_FUNC, 2)} + | + {_ind(CAPI_DATA, 2)} + | + {_ind(CAPI_DEFINE, 2)} + ) +'''), re.VERBOSE) + +KINDS = [ + 'func', + 'data', + 'macro', + 'constant', +] + + +def _parse_line(line): + m = CAPI_RE.match(line) + if not m: + #if 'PyAPI_' in line or '#define ' in line or ' define ' in line: + # print(line) + return None + results = zip(KINDS, m.groups()) + for kind, name in results: + if name: + return name, kind + # It was a plain #define. + return None + + +LEVELS = { + 'stable', + 'cpython', + 'private', + 'internal', +} + +def _get_level(filename, name, *, + _cpython=INCLUDE_CPYTHON + os.path.sep, + _internal=INCLUDE_INTERNAL + os.path.sep, + ): + if filename.startswith(_internal): + return 'internal' + elif name.startswith('_'): + return 'private' + elif os.path.dirname(filename) == INCLUDE_ROOT: + return 'stable' + elif filename.startswith(_cpython): + return 'cpython' + else: + raise NotImplementedError + #return '???' + + +class CAPIItem(namedtuple('CAPIItem', 'file lno name kind level')): + + @classmethod + def from_line(cls, line, filename, lno): + parsed = _parse_line(line) + if not parsed: + return None + name, kind = parsed + level = _get_level(filename, name) + return cls(filename, lno, name, kind, level) + + @property + def relfile(self): + return self.file[len(REPO_ROOT) + 1:] + + +def _parse_capi(lines, filename): + if isinstance(lines, str): + lines = lines.splitlines() + for lno, line in enumerate(lines, 1): + yield CAPIItem.from_line(line, filename, lno) + + +def iter_capi(filenames=None): + for filename in iter_header_files(filenames): + with open(filename) as infile: + for item in _parse_capi(infile, filename): + if item is not None: + yield item + + +def _collate_by_kind(items): + maxfilename = maxname = maxlevel = 0 + collated = {} + for item in items: + if item.kind in collated: + collated[item.kind].append(item) + else: + collated[item.kind] = [item] + maxfilename = max(len(item.relfile), maxfilename) + maxname = max(len(item.name), maxname) + maxlevel = max(len(item.name), maxlevel) + return collated, maxfilename, maxname, maxlevel + + +def render_table(items, *, verbose=False): + collated, maxfilename, maxname, maxlevel = _collate_by_kind(items) + maxlevel = max(len(level) for level in LEVELS) + header, div, fmt = build_table([ + f'filename:{maxfilename}', + f'name:{maxname}', + *([f'level:{maxlevel}'] + if verbose + else [ + f'S:1', + f'C:1', + f'P:1', + f'I:1', + ] + ), + ]) + total = 0 + for kind in KINDS: + if kind not in collated: + continue + yield '' + yield f' === {kind} ===' + yield '' + yield header + yield div + for item in collated[kind]: + yield fmt.format( + filename=item.relfile, + name=item.name, + **(dict(level=item.level) + if verbose + else dict( + S='S' if item.level == 'stable' else '', + C='C' if item.level == 'cpython' else '', + P='P' if item.level == 'private' else '', + I='I' if item.level == 'internal' else '', + ) + ) + + ) + yield div + subtotal = len(collated[kind]) + yield f' sub-total: {subtotal}' + total += subtotal + yield '' + yield f'total: {total}' + + +def render_summary(items, *, bykind=True): + total = 0 + summary = summarize(items, bykind=bykind) + for outer, counts in summary.items(): + subtotal = sum(c for _, c in counts.items()) + yield f'{outer + ":":20} ({subtotal})' + for inner, count in counts.items(): + yield f' {inner + ":":9} {count}' + total += subtotal + yield f'{"total:":20} ({total})' + + +def summarize(items, *, bykind=True): + if bykind: + summary = {kind: {l: 0 for l in LEVELS} + for kind in KINDS} + for item in items: + summary[item.kind][item.level] += 1 + else: + summary = {level: {k: 0 for k in KINDS} + for level in LEVELS} + for item in items: + summary[item.level][item.kind] += 1 + return summary From 8d454895240b11f509a7c97dd212d004bc5dc958 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 18:43:29 -0700 Subject: [PATCH 05/17] Add a "capi" subcommand to the c-analyzer script. --- Tools/c-analyzer/cpython/__main__.py | 101 ++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 6d78af299bb6f8..8fe6ff9cce1f68 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -3,11 +3,14 @@ from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_common.scriptutil import ( + VERBOSITY, add_verbosity_cli, add_traceback_cli, add_commands_cli, add_kind_filtering_cli, add_files_cli, + add_progress_cli, + main_for_filenames, process_args_by_key, configure_logger, get_prog, @@ -17,7 +20,7 @@ import c_analyzer.__main__ as c_analyzer import c_analyzer as _c_analyzer from c_analyzer.info import UNKNOWN -from . import _analyzer, _parser, REPO_ROOT +from . import _analyzer, _capi, _files, _parser, REPO_ROOT logger = logging.getLogger(__name__) @@ -25,9 +28,9 @@ def _resolve_filenames(filenames): if filenames: - resolved = (_parser.resolve_filename(f) for f in filenames) + resolved = (_files.resolve_filename(f) for f in filenames) else: - resolved = _parser.iter_filenames() + resolved = _files.iter_filenames() return resolved @@ -204,6 +207,90 @@ def analyze(files, **kwargs): ) +def _cli_capi(parser): + parser.add_argument('--levels', action='append', metavar='LEVEL[,...]') + parser.add_argument(f'--public', dest='levels', + action='append_const', const='public') + parser.add_argument(f'--no-public', dest='levels', + action='append_const', const='no-public') + for level in _capi.LEVELS: + parser.add_argument(f'--{level}', dest='levels', + action='append_const', const=level) + def process_levels(args): + levels = [] + for raw in args.levels or (): + for level in raw.replace(',', ' ').strip().split(): + if level == 'public': + levels.append('stable') + levels.append('cpython') + elif level == 'no-public': + levels.append('private') + levels.append('internal') + elif level in _capi.LEVELS: + levels.append(level) + else: + parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}') + args.levels = set(levels) + + parser.add_argument('--kinds', action='append', metavar='KIND[,...]') + for kind in _capi.KINDS: + parser.add_argument(f'--{kind}', dest='kinds', + action='append_const', const=kind) + def process_kinds(args): + kinds = [] + for raw in args.kinds or (): + for kind in raw.replace(',', ' ').strip().split(): + if kind in _capi.KINDS: + kind.append(kind) + else: + parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}') + args.kinds = set(kinds) + + parser.add_argument('--summary', nargs='?') + def process_summary(args): + if args.summary: + if args.summary not in ('kind', 'level'): + if not args.summary.endswith('.h'): + if args.summary != 'Include': + if not args.summary.startswith('Include' + os.path.sep): + parser.error(f'expected SUMMARY to be one of {["kind", "level"]}, got {args.summary!r}') + args.filenames.insert(0, args.summary) + args.summary = None + + parser.add_argument('filenames', nargs='*', metavar='FILENAME') + process_progress = add_progress_cli(parser) + + return [ + process_levels, + process_summary, + process_progress, + ] + + +def cmd_capi(filenames=None, *, + levels=None, + kinds=None, + track_progress=None, + verbosity=VERBOSITY, + summary=None, + **kwargs + ): + filenames = _files.iter_header_files(filenames, levels=levels) + #filenames = (file for file, _ in main_for_filenames(filenames)) + filenames = track_progress(filenames) + items = _capi.iter_capi(filenames) + if levels: + items = (item for item in items if item.level in levels) + if kinds: + items = (item for item in items if item.kind in kinds) + if summary: + for line in _capi.render_summary(items, bykind=summary == 'kind'): + print(line) + else: + for line in _capi.render_table(items, verbose=verbosity > VERBOSITY): + print(line) + + # We do not define any other cmd_*() handlers here, # favoring those defined elsewhere. @@ -228,6 +315,11 @@ def analyze(files, **kwargs): [_cli_data], cmd_data, ), + 'capi': ( + 'inspect the C-API', + [_cli_capi], + cmd_capi, + ), } @@ -269,6 +361,9 @@ def parse_args(argv=sys.argv[1:], prog=None, *, subset=None): if cmd != 'parse': # "verbosity" is sent to the commands, so we put it back. args.verbosity = verbosity + if cmd == 'capi': + if not args.summary and '--summary' in argv: + args.summary = 'level' return cmd, ns, verbosity, traceback_cm From 601a43b1735bdcadaff094eca05e928904723c94 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 19 Dec 2020 22:04:33 -0700 Subject: [PATCH 06/17] Handle static inline too. --- Tools/c-analyzer/cpython/_capi.py | 63 +++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index 89cf8bdf7a4946..bf7a25f9dddf26 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -27,7 +27,7 @@ [(] {_ind(_MAYBE_NESTED_PARENS, 2)} [)] \s* - (\w+) # + (\w+) # \s* [(] ) ''') @@ -39,19 +39,30 @@ [(] {_ind(_MAYBE_NESTED_PARENS, 2)} [)] \s* - (\w+) # + (\w+) # \b [^(] ) ''') +CAPI_INLINE = textwrap.dedent(r''' + (?: + ^ + \s* + static \s+ inline \s+ + .*? + \s+ + ( \w+ ) # + \s* [(] + ) +''') CAPI_MACRO = textwrap.dedent(r''' (?: - (\w+) # + (\w+) # \s* [(] ) ''') CAPI_CONSTANT = textwrap.dedent(r''' (?: - (\w+) # + (\w+) # \s+ [^(] ) ''') @@ -79,6 +90,8 @@ | {_ind(CAPI_DATA, 2)} | + {_ind(CAPI_INLINE, 2)} + | {_ind(CAPI_DEFINE, 2)} ) '''), re.VERBOSE) @@ -86,14 +99,19 @@ KINDS = [ 'func', 'data', + 'inline', 'macro', 'constant', ] -def _parse_line(line): +def _parse_line(line, prev=None): + if prev: + line = prev + ' ' + line m = CAPI_RE.match(line) if not m: + if not prev and line.startswith('static inline '): + return line # the new "prev" #if 'PyAPI_' in line or '#define ' in line or ' define ' in line: # print(line) return None @@ -132,13 +150,15 @@ def _get_level(filename, name, *, class CAPIItem(namedtuple('CAPIItem', 'file lno name kind level')): @classmethod - def from_line(cls, line, filename, lno): - parsed = _parse_line(line) + def from_line(cls, line, filename, lno, prev=None): + parsed = _parse_line(line, prev) if not parsed: - return None + return None, None + if isinstance(parsed, str): + return None, parsed name, kind = parsed level = _get_level(filename, name) - return cls(filename, lno, name, kind, level) + return cls(filename, lno, name, kind, level), None @property def relfile(self): @@ -148,16 +168,18 @@ def relfile(self): def _parse_capi(lines, filename): if isinstance(lines, str): lines = lines.splitlines() + prev = None for lno, line in enumerate(lines, 1): - yield CAPIItem.from_line(line, filename, lno) + parsed, prev = CAPIItem.from_line(line, filename, lno, prev) + if parsed: + yield parsed def iter_capi(filenames=None): for filename in iter_header_files(filenames): with open(filename) as infile: for item in _parse_capi(infile, filename): - if item is not None: - yield item + yield item def _collate_by_kind(items): @@ -225,6 +247,7 @@ def render_table(items, *, verbose=False): def render_summary(items, *, bykind=True): total = 0 summary = summarize(items, bykind=bykind) + # XXX Stablize the sorting to match KINDS/LEVELS. for outer, counts in summary.items(): subtotal = sum(c for _, c in counts.items()) yield f'{outer + ":":20} ({subtotal})' @@ -236,13 +259,21 @@ def render_summary(items, *, bykind=True): def summarize(items, *, bykind=True): if bykind: - summary = {kind: {l: 0 for l in LEVELS} - for kind in KINDS} + outers = KINDS + inners = LEVELS + else: + outers = LEVELS + inners = KINDS + summary = {} + for outer in outers: + summary[outer] = _outer = {} + for inner in inners: + _outer[inner] = 0 + + if bykind: for item in items: summary[item.kind][item.level] += 1 else: - summary = {level: {k: 0 for k in KINDS} - for level in LEVELS} for item in items: summary[item.level][item.kind] += 1 return summary From 4838a07353abb61cf737dca89f9b0a422a8dcf7e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 12:19:27 -0700 Subject: [PATCH 07/17] Fix the macro regex. --- Tools/c-analyzer/cpython/_capi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index bf7a25f9dddf26..d3a262c4fe842f 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -57,7 +57,7 @@ CAPI_MACRO = textwrap.dedent(r''' (?: (\w+) # - \s* [(] + [(] ) ''') CAPI_CONSTANT = textwrap.dedent(r''' From 57f857cb6fcaf387187fcfc5b1a1b957a2166013 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 12:20:32 -0700 Subject: [PATCH 08/17] Handle missing "track_progress". --- Tools/c-analyzer/cpython/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 8fe6ff9cce1f68..502b74c409bf30 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -277,7 +277,8 @@ def cmd_capi(filenames=None, *, ): filenames = _files.iter_header_files(filenames, levels=levels) #filenames = (file for file, _ in main_for_filenames(filenames)) - filenames = track_progress(filenames) + if track_progress is not None: + filenames = track_progress(filenames) items = _capi.iter_capi(filenames) if levels: items = (item for item in items if item.level in levels) From b4995d5a79f169e3c95ee8801f0b7ea844b954a4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 12:39:28 -0700 Subject: [PATCH 09/17] Add --format option to capi CLI. --- Tools/c-analyzer/cpython/__main__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 502b74c409bf30..6f2f2b828d08b5 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -246,8 +246,12 @@ def process_kinds(args): parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}') args.kinds = set(kinds) + parser.add_argument('--format', choices=['brief', 'summary']) parser.add_argument('--summary', nargs='?') - def process_summary(args): + def process_format(args): + if not args.format: + args.format = 'brief' + if args.summary: if args.summary not in ('kind', 'level'): if not args.summary.endswith('.h'): @@ -256,13 +260,15 @@ def process_summary(args): parser.error(f'expected SUMMARY to be one of {["kind", "level"]}, got {args.summary!r}') args.filenames.insert(0, args.summary) args.summary = None + elif args.format == 'summary': + args.summary = 'level' parser.add_argument('filenames', nargs='*', metavar='FILENAME') process_progress = add_progress_cli(parser) return [ process_levels, - process_summary, + process_format, process_progress, ] From d4a3235cb8d30945fdbe9263e3aefe8fe5de6f23 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 12:53:28 -0700 Subject: [PATCH 10/17] Pass argv to scriptutil.process_args() and all CLI processors. --- Tools/c-analyzer/c_analyzer/__main__.py | 9 +++---- Tools/c-analyzer/c_common/scriptutil.py | 24 +++++++++---------- Tools/c-analyzer/c_parser/__main__.py | 3 ++- .../c_parser/preprocessor/__main__.py | 5 ++-- Tools/c-analyzer/check-c-globals.py | 1 + Tools/c-analyzer/cpython/__main__.py | 12 +++++----- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 44325f2952e28c..24fc6cd182656b 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -263,7 +263,7 @@ def fmt_full(analysis): def add_output_cli(parser, *, default='summary'): parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS)) - def process_args(args): + def process_args(args, *, argv=None): pass return process_args @@ -280,7 +280,7 @@ def _cli_check(parser, checks=None, **kwargs): process_checks = add_checks_cli(parser) elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]): check = checks[0][1:-1] - def process_checks(args): + def process_checks(args, *, argv=None): args.checks = [check] else: process_checks = add_checks_cli(parser, checks=checks) @@ -428,9 +428,9 @@ def _cli_data(parser, filenames=None, known=None): if known is None: sub.add_argument('--known', required=True) - def process_args(args): + def process_args(args, *, argv): if args.datacmd == 'dump': - process_progress(args) + process_progress(args, argv) return process_args @@ -515,6 +515,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None): verbosity, traceback_cm = process_args_by_key( args, + argv, processors[cmd], ['verbosity', 'traceback_cm'], ) diff --git a/Tools/c-analyzer/c_common/scriptutil.py b/Tools/c-analyzer/c_common/scriptutil.py index 50dd7548869193..ce69af2b6bdee5 100644 --- a/Tools/c-analyzer/c_common/scriptutil.py +++ b/Tools/c-analyzer/c_common/scriptutil.py @@ -192,7 +192,7 @@ def add_verbosity_cli(parser): parser.add_argument('-q', '--quiet', action='count', default=0) parser.add_argument('-v', '--verbose', action='count', default=0) - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) key = 'verbosity' if key in ns: @@ -208,7 +208,7 @@ def add_traceback_cli(parser): parser.add_argument('--no-traceback', '--no-tb', dest='traceback', action='store_const', const=False) - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) key = 'traceback_cm' if key in ns: @@ -262,7 +262,7 @@ def add_sepval_cli(parser, opt, dest, choices, *, sep=',', **kwargs): #kwargs.setdefault('metavar', opt.upper()) parser.add_argument(opt, dest=dest, action='append', **kwargs) - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) # XXX Use normalize_selection()? @@ -293,7 +293,7 @@ def add_file_filtering_cli(parser, *, excluded=None): excluded = tuple(excluded or ()) - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) key = 'iter_filenames' if key in ns: @@ -323,7 +323,7 @@ def add_progress_cli(parser, *, threshold=VERBOSITY, **kwargs): parser.add_argument('--no-progress', dest='track_progress', action='store_false') parser.set_defaults(track_progress=True) - def process_args(args): + def process_args(args, *, argv=None): if args.track_progress: ns = vars(args) verbosity = ns.get('verbosity', VERBOSITY) @@ -339,7 +339,7 @@ def add_failure_filtering_cli(parser, pool, *, default=False): metavar=f'"{{all|{"|".join(sorted(pool))}}},..."') parser.add_argument('--no-fail', dest='fail', action='store_const', const=()) - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) fail = ns.pop('fail') @@ -371,7 +371,7 @@ def ignore_exc(exc): def add_kind_filtering_cli(parser, *, default=None): parser.add_argument('--kinds', action='append') - def process_args(args): + def process_args(args, *, argv=None): ns = vars(args) kinds = [] @@ -486,18 +486,18 @@ def _flatten_processors(processors): yield from _flatten_processors(proc) -def process_args(args, processors, *, keys=None): +def process_args(args, argv, processors, *, keys=None): processors = _flatten_processors(processors) ns = vars(args) extracted = {} if keys is None: for process_args in processors: - for key in process_args(args): + for key in process_args(args, argv=argv): extracted[key] = ns.pop(key) else: remainder = set(keys) for process_args in processors: - hanging = process_args(args) + hanging = process_args(args, argv=argv) if isinstance(hanging, str): hanging = [hanging] for key in hanging or (): @@ -510,8 +510,8 @@ def process_args(args, processors, *, keys=None): return extracted -def process_args_by_key(args, processors, keys): - extracted = process_args(args, processors, keys=keys) +def process_args_by_key(args, argv, processors, keys): + extracted = process_args(args, argv, processors, keys=keys) return [extracted[key] for key in keys] diff --git a/Tools/c-analyzer/c_parser/__main__.py b/Tools/c-analyzer/c_parser/__main__.py index 539cec509cecb4..78f47a1808f50b 100644 --- a/Tools/c-analyzer/c_parser/__main__.py +++ b/Tools/c-analyzer/c_parser/__main__.py @@ -149,7 +149,7 @@ def add_output_cli(parser): parser.add_argument('--showfwd', action='store_true', default=None) parser.add_argument('--no-showfwd', dest='showfwd', action='store_false', default=None) - def process_args(args): + def process_args(args, *, argv=None): pass return process_args @@ -243,6 +243,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset='parse'): verbosity, traceback_cm = process_args_by_key( args, + argv, processors[cmd], ['verbosity', 'traceback_cm'], ) diff --git a/Tools/c-analyzer/c_parser/preprocessor/__main__.py b/Tools/c-analyzer/c_parser/preprocessor/__main__.py index a6054307c25759..bfc61949a76e4e 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/__main__.py +++ b/Tools/c-analyzer/c_parser/preprocessor/__main__.py @@ -40,10 +40,10 @@ def add_common_cli(parser, *, get_preprocessor=_get_preprocessor): parser.add_argument('--same', action='append') process_fail_arg = add_failure_filtering_cli(parser, FAIL) - def process_args(args): + def process_args(args, *, argv): ns = vars(args) - process_fail_arg(args) + process_fail_arg(args, argv) ignore_exc = ns.pop('ignore_exc') # We later pass ignore_exc to _get_preprocessor(). @@ -174,6 +174,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, verbosity, traceback_cm = process_args_by_key( args, + argv, processors[cmd], ['verbosity', 'traceback_cm'], ) diff --git a/Tools/c-analyzer/check-c-globals.py b/Tools/c-analyzer/check-c-globals.py index 3fe2bdcae14603..b1364a612bb7d3 100644 --- a/Tools/c-analyzer/check-c-globals.py +++ b/Tools/c-analyzer/check-c-globals.py @@ -22,6 +22,7 @@ def parse_args(): cmd = 'check' verbosity, traceback_cm = process_args_by_key( args, + argv, processors, ['verbosity', 'traceback_cm'], ) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 6f2f2b828d08b5..545e47cb5c006b 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -216,7 +216,7 @@ def _cli_capi(parser): for level in _capi.LEVELS: parser.add_argument(f'--{level}', dest='levels', action='append_const', const=level) - def process_levels(args): + def process_levels(args, *, argv=None): levels = [] for raw in args.levels or (): for level in raw.replace(',', ' ').strip().split(): @@ -236,7 +236,7 @@ def process_levels(args): for kind in _capi.KINDS: parser.add_argument(f'--{kind}', dest='kinds', action='append_const', const=kind) - def process_kinds(args): + def process_kinds(args, *, argv=None): kinds = [] for raw in args.kinds or (): for kind in raw.replace(',', ' ').strip().split(): @@ -248,7 +248,7 @@ def process_kinds(args): parser.add_argument('--format', choices=['brief', 'summary']) parser.add_argument('--summary', nargs='?') - def process_format(args): + def process_format(args, *, argv): if not args.format: args.format = 'brief' @@ -262,6 +262,8 @@ def process_format(args): args.summary = None elif args.format == 'summary': args.summary = 'level' + elif '--summary' in argv: + args.summary = 'level' parser.add_argument('filenames', nargs='*', metavar='FILENAME') process_progress = add_progress_cli(parser) @@ -362,15 +364,13 @@ def parse_args(argv=sys.argv[1:], prog=None, *, subset=None): verbosity, traceback_cm = process_args_by_key( args, + argv, processors[cmd], ['verbosity', 'traceback_cm'], ) if cmd != 'parse': # "verbosity" is sent to the commands, so we put it back. args.verbosity = verbosity - if cmd == 'capi': - if not args.summary and '--summary' in argv: - args.summary = 'level' return cmd, ns, verbosity, traceback_cm From c4a526f5ea3a70a8c49d050c843d8453e927df46 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 13:29:23 -0700 Subject: [PATCH 11/17] Add --group-by option to capi CLI. --- Tools/c-analyzer/cpython/__main__.py | 43 ++++++-------- Tools/c-analyzer/cpython/_capi.py | 86 +++++++++++++++++++--------- 2 files changed, 75 insertions(+), 54 deletions(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 545e47cb5c006b..259577b0f867f0 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -246,31 +246,18 @@ def process_kinds(args, *, argv=None): parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}') args.kinds = set(kinds) - parser.add_argument('--format', choices=['brief', 'summary']) - parser.add_argument('--summary', nargs='?') - def process_format(args, *, argv): - if not args.format: - args.format = 'brief' - - if args.summary: - if args.summary not in ('kind', 'level'): - if not args.summary.endswith('.h'): - if args.summary != 'Include': - if not args.summary.startswith('Include' + os.path.sep): - parser.error(f'expected SUMMARY to be one of {["kind", "level"]}, got {args.summary!r}') - args.filenames.insert(0, args.summary) - args.summary = None - elif args.format == 'summary': - args.summary = 'level' - elif '--summary' in argv: - args.summary = 'level' + parser.add_argument('--group-by', dest='groupby', + choices=['level', 'kind'], default='kind') + + parser.add_argument('--format', choices=['brief', 'summary'], default='brief') + parser.add_argument('--summary', dest='format', + action='store_const', const='summary') parser.add_argument('filenames', nargs='*', metavar='FILENAME') process_progress = add_progress_cli(parser) return [ process_levels, - process_format, process_progress, ] @@ -278,11 +265,17 @@ def process_format(args, *, argv): def cmd_capi(filenames=None, *, levels=None, kinds=None, + groupby='kind', + format='brief', track_progress=None, verbosity=VERBOSITY, - summary=None, **kwargs ): + try: + render = _capi.FORMATS[format or 'brief'] + except KeyError: + raise ValueError(f'unsupported format {format!r}') + filenames = _files.iter_header_files(filenames, levels=levels) #filenames = (file for file, _ in main_for_filenames(filenames)) if track_progress is not None: @@ -292,12 +285,10 @@ def cmd_capi(filenames=None, *, items = (item for item in items if item.level in levels) if kinds: items = (item for item in items if item.kind in kinds) - if summary: - for line in _capi.render_summary(items, bykind=summary == 'kind'): - print(line) - else: - for line in _capi.render_table(items, verbose=verbosity > VERBOSITY): - print(line) + + lines = render(items, groupby=groupby, verbose=verbosity > VERBOSITY) + for line in lines: + print(line) # We do not define any other cmd_*() handlers here, diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index d3a262c4fe842f..1fccf815beebd3 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -165,6 +165,47 @@ def relfile(self): return self.file[len(REPO_ROOT) + 1:] +def _parse_groupby(raw): + if not raw: + raw = 'kind' + + if isinstance(raw, str): + groupby = raw.replace(',', ' ').strip().split() + else: + raise NotImplementedError + + if not all(v in ('kind', 'level') for v in groupby): + raise ValueError(f'invalid groupby value {raw!r}') + return groupby + + +def summarize(items, *, groupby='kind'): + summary = {} + + groupby = _parse_groupby(groupby)[0] + if groupby == 'kind': + outers = KINDS + inners = LEVELS + def increment(item): + summary[item.kind][item.level] += 1 + elif groupby == 'level': + outers = LEVELS + inners = KINDS + def increment(item): + summary[item.level][item.kind] += 1 + else: + raise NotImplementedError + + for outer in outers: + summary[outer] = _outer = {} + for inner in inners: + _outer[inner] = 0 + for item in items: + increment(item) + + return summary + + def _parse_capi(lines, filename): if isinstance(lines, str): lines = lines.splitlines() @@ -182,22 +223,27 @@ def iter_capi(filenames=None): yield item -def _collate_by_kind(items): +def _collate(items, groupby): + groupby = _parse_groupby(groupby)[0] maxfilename = maxname = maxlevel = 0 collated = {} for item in items: - if item.kind in collated: - collated[item.kind].append(item) + key = getattr(item, groupby) + if key in collated: + collated[key].append(item) else: - collated[item.kind] = [item] + collated[key] = [item] maxfilename = max(len(item.relfile), maxfilename) maxname = max(len(item.name), maxname) maxlevel = max(len(item.name), maxlevel) return collated, maxfilename, maxname, maxlevel -def render_table(items, *, verbose=False): - collated, maxfilename, maxname, maxlevel = _collate_by_kind(items) +def render_table(items, *, groupby='kind', verbose=False): + groupby = groupby or 'kind' + if groupby != 'kind': + raise NotImplementedError + collated, maxfilename, maxname, maxlevel = _collate(items, groupby) maxlevel = max(len(level) for level in LEVELS) header, div, fmt = build_table([ f'filename:{maxfilename}', @@ -244,9 +290,9 @@ def render_table(items, *, verbose=False): yield f'total: {total}' -def render_summary(items, *, bykind=True): +def render_summary(items, *, groupby='kind', verbose=False): total = 0 - summary = summarize(items, bykind=bykind) + summary = summarize(items, groupby=groupby) # XXX Stablize the sorting to match KINDS/LEVELS. for outer, counts in summary.items(): subtotal = sum(c for _, c in counts.items()) @@ -257,23 +303,7 @@ def render_summary(items, *, bykind=True): yield f'{"total:":20} ({total})' -def summarize(items, *, bykind=True): - if bykind: - outers = KINDS - inners = LEVELS - else: - outers = LEVELS - inners = KINDS - summary = {} - for outer in outers: - summary[outer] = _outer = {} - for inner in inners: - _outer[inner] = 0 - - if bykind: - for item in items: - summary[item.kind][item.level] += 1 - else: - for item in items: - summary[item.level][item.kind] += 1 - return summary +FORMATS = { + 'brief': render_table, + 'summary': render_summary, +} From bf687b3e6c69b4b0cda56e91f34aa53cad8e9e3c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 14:32:54 -0700 Subject: [PATCH 12/17] Make render_table() more flexible. --- Tools/c-analyzer/cpython/_capi.py | 102 +++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index 1fccf815beebd3..f17d1f10ad9a85 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -225,7 +225,7 @@ def iter_capi(filenames=None): def _collate(items, groupby): groupby = _parse_groupby(groupby)[0] - maxfilename = maxname = maxlevel = 0 + maxfilename = maxname = maxkind = maxlevel = 0 collated = {} for item in items: key = getattr(item, groupby) @@ -235,55 +235,95 @@ def _collate(items, groupby): collated[key] = [item] maxfilename = max(len(item.relfile), maxfilename) maxname = max(len(item.name), maxname) - maxlevel = max(len(item.name), maxlevel) - return collated, maxfilename, maxname, maxlevel + maxkind = max(len(item.kind), maxkind) + maxlevel = max(len(item.level), maxlevel) + maxextra = { + 'kind': maxkind, + 'level': maxlevel, + } + return collated, groupby, maxfilename, maxname, maxextra + + +_LEVEL_MARKERS = { + 'S': 'stable', + 'C': 'cpython', + 'P': 'private', + 'I': 'internal', +} +_KIND_MARKERS = { + 'F': 'func', + 'D': 'data', + 'I': 'inline', + 'M': 'macro', + 'C': 'constant', +} def render_table(items, *, groupby='kind', verbose=False): - groupby = groupby or 'kind' - if groupby != 'kind': + if groupby: + collated, groupby, maxfilename, maxname, maxextra = _collate(items, groupby) + if groupby == 'kind': + groups = KINDS + extras = ['level'] + markers = {'level': _LEVEL_MARKERS} + elif groupby == 'level': + groups = LEVELS + extras = ['kind'] + markers = {'kind': _KIND_MARKERS} + else: + raise NotImplementedError + else: + # XXX Support no grouping? raise NotImplementedError - collated, maxfilename, maxname, maxlevel = _collate(items, groupby) - maxlevel = max(len(level) for level in LEVELS) + + if verbose: + maxextra['kind'] = max(len(kind) for kind in KINDS) + maxextra['level'] = max(len(level) for level in LEVELS) + extracols = [f'{extra}:{maxextra[extra]}' + for extra in extras] + def get_extra(item): + return {extra: getattr(item, extra) + for extra in extras} + elif len(extras) == 1: + extra, = extras + extracols = [f'{m}:1' for m in markers[extra]] + def get_extra(item): + return {m: m if getattr(item, extra) == markers[extra][m] else '' + for m in markers[extra]} + else: + raise NotImplementedError + #extracols = [[f'{m}:1' for m in markers[extra]] + # for extra in extras] + #def get_extra(item): + # values = {} + # for extra in extras: + # cur = markers[extra] + # for m in cur: + # values[m] = m if getattr(item, m) == cur[m] else '' + # return values + header, div, fmt = build_table([ f'filename:{maxfilename}', f'name:{maxname}', - *([f'level:{maxlevel}'] - if verbose - else [ - f'S:1', - f'C:1', - f'P:1', - f'I:1', - ] - ), + *extracols, ]) total = 0 - for kind in KINDS: - if kind not in collated: + for group in groups: + if group not in collated: continue yield '' - yield f' === {kind} ===' + yield f' === {group} ===' yield '' yield header yield div - for item in collated[kind]: + for item in collated[group]: yield fmt.format( filename=item.relfile, name=item.name, - **(dict(level=item.level) - if verbose - else dict( - S='S' if item.level == 'stable' else '', - C='C' if item.level == 'cpython' else '', - P='P' if item.level == 'private' else '', - I='I' if item.level == 'internal' else '', - ) - ) - + **get_extra(item), ) yield div - subtotal = len(collated[kind]) + subtotal = len(collated[group]) yield f' sub-total: {subtotal}' total += subtotal yield '' From c5e248755b9b493247557480b13cf38053b0e71b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 14:49:17 -0700 Subject: [PATCH 13/17] Add the "full" format. --- Tools/c-analyzer/cpython/__main__.py | 5 +++-- Tools/c-analyzer/cpython/_capi.py | 31 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 259577b0f867f0..3ccd19abbd5604 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -247,9 +247,9 @@ def process_kinds(args, *, argv=None): args.kinds = set(kinds) parser.add_argument('--group-by', dest='groupby', - choices=['level', 'kind'], default='kind') + choices=['level', 'kind']) - parser.add_argument('--format', choices=['brief', 'summary'], default='brief') + parser.add_argument('--format', choices=['brief', 'full', 'summary'], default='brief') parser.add_argument('--summary', dest='format', action='store_const', const='summary') @@ -287,6 +287,7 @@ def cmd_capi(filenames=None, *, items = (item for item in items if item.kind in kinds) lines = render(items, groupby=groupby, verbose=verbosity > VERBOSITY) + print() for line in lines: print(line) diff --git a/Tools/c-analyzer/cpython/_capi.py b/Tools/c-analyzer/cpython/_capi.py index f17d1f10ad9a85..75619d4ce6b0f9 100644 --- a/Tools/c-analyzer/cpython/_capi.py +++ b/Tools/c-analyzer/cpython/_capi.py @@ -330,6 +330,36 @@ def get_extra(item): yield f'total: {total}' +def render_full(items, *, groupby=None, verbose=False): + if groupby: + collated, groupby, _, _, _ = _collate(items, groupby) + for group, grouped in collated.items(): + yield '#' * 25 + yield f'# {group} ({len(grouped)})' + yield '#' * 25 + yield '' + if not grouped: + continue + for item in grouped: + yield from _render_item_full(item, groupby, verbose) + yield '' + else: + for item in items: + yield from _render_item_full(item, None, verbose) + yield '' + + +def _render_item_full(item, groupby, verbose): + yield item.name + yield f' {"filename:":10} {item.relfile}' + for extra in ('kind', 'level'): + if groupby != extra: + yield f' {extra+":":10} {getattr(item, extra)}' + if verbose: + # Show the actual text. + raise NotImplementedError + + def render_summary(items, *, groupby='kind', verbose=False): total = 0 summary = summarize(items, groupby=groupby) @@ -345,5 +375,6 @@ def render_summary(items, *, groupby='kind', verbose=False): FORMATS = { 'brief': render_table, + 'full': render_full, 'summary': render_summary, } From 8fdda67dbc2f3c92ebe97f47f49a1264d8350b25 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 23 Dec 2020 18:13:56 -0700 Subject: [PATCH 14/17] Support flexible formats. --- Tools/c-analyzer/c_common/tables.py | 188 ++++++++++++++++++--------- Tools/c-analyzer/cpython/__main__.py | 14 +- Tools/c-analyzer/cpython/_capi.py | 93 ++++++++----- 3 files changed, 196 insertions(+), 99 deletions(-) diff --git a/Tools/c-analyzer/c_common/tables.py b/Tools/c-analyzer/c_common/tables.py index b9e18e8e6d8d56..85b501925715d3 100644 --- a/Tools/c-analyzer/c_common/tables.py +++ b/Tools/c-analyzer/c_common/tables.py @@ -1,4 +1,6 @@ import csv +import re +import textwrap from . import NOT_SET, strutil, fsutil @@ -220,83 +222,141 @@ def _normalize_table_file_props(header, sep): WIDTH = 20 -def build_table(columns, *, sep=' '): - if isinstance(columns, str): - columns = columns.replace(',', ' ').strip().split() - columns = _parse_columns(columns) - return _build_table(columns, sep=sep) +def resolve_columns(specs): + if isinstance(specs, str): + specs = specs.replace(',', ' ').strip().split() + return _resolve_colspecs(specs) + + +def build_table(specs, *, sep=' ', defaultwidth=None): + columns = resolve_columns(specs) + return _build_table(columns, sep=sep, defaultwidth=defaultwidth) + + +_COLSPEC_RE = re.compile(textwrap.dedent(r''' + ^ + (?: + [[] + ( + (?: [^\s\]] [^\]]* )? + [^\s\]] + ) #