From 8ce7d62d661d906df91de0e98a91e79901ae1048 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 27 Sep 2019 11:30:29 -0600 Subject: [PATCH 01/27] Move show.py to c_analyzer_common. --- .../test_show.py | 2 +- Tools/c-analyzer/c_analyzer_common/show.py | 11 +++++++++++ Tools/c-analyzer/c_globals/__main__.py | 4 ++-- Tools/c-analyzer/c_globals/show.py | 16 ---------------- 4 files changed, 14 insertions(+), 19 deletions(-) rename Lib/test/test_tools/test_c_analyzer/{test_c_globals => test_c_analyzer_common}/test_show.py (98%) create mode 100644 Tools/c-analyzer/c_analyzer_common/show.py delete mode 100644 Tools/c-analyzer/c_globals/show.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py similarity index 98% rename from Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py rename to Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py index ce1dad85db1b87..b689aafb19314c 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py @@ -3,7 +3,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): from c_parser import info - from c_globals.show import basic + from c_analyzer_common.show import basic TYPICAL = [ diff --git a/Tools/c-analyzer/c_analyzer_common/show.py b/Tools/c-analyzer/c_analyzer_common/show.py new file mode 100644 index 00000000000000..5f3cb1c2fb0b59 --- /dev/null +++ b/Tools/c-analyzer/c_analyzer_common/show.py @@ -0,0 +1,11 @@ + +def basic(variables, *, + _print=print): + """Print each row simply.""" + for var in variables: + if var.funcname: + line = f'{var.filename}:{var.funcname}():{var.name}' + else: + line = f'{var.filename}:{var.name}' + line = f'{line:<64} {var.vartype}' + _print(line) diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index 9570fb6a14c4e6..42f71b0ccba466 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -3,13 +3,13 @@ import re import sys -from c_analyzer_common import SOURCE_DIRS, REPO_ROOT +from c_analyzer_common import SOURCE_DIRS, REPO_ROOT, show from c_analyzer_common.info import UNKNOWN from c_analyzer_common.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) -from . import find, show +from . import find from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object diff --git a/Tools/c-analyzer/c_globals/show.py b/Tools/c-analyzer/c_globals/show.py deleted file mode 100644 index f4298b17b67884..00000000000000 --- a/Tools/c-analyzer/c_globals/show.py +++ /dev/null @@ -1,16 +0,0 @@ - -def basic(globals, *, - _print=print): - """Print each row simply.""" - for variable in globals: - if variable.funcname: - line = f'{variable.filename}:{variable.funcname}():{variable.name}' - else: - line = f'{variable.filename}:{variable.name}' - vartype = variable.vartype - #if vartype.startswith('static '): - # vartype = vartype.partition(' ')[2] - #else: - # vartype = '=' + vartype - line = f'{line:<64} {vartype}' - _print(line) From c40c247563414ae4f82f486d76f9f7930a592710 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 27 Sep 2019 11:31:49 -0600 Subject: [PATCH 02/27] Small cleanups around _check_results(). --- .../test_c_globals/test___main__.py | 82 ++++++++++++------ Tools/c-analyzer/c_globals/__main__.py | 83 +++++++++++-------- 2 files changed, 107 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py index 5f52c588d7c8bd..21c850a7a541ba 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py @@ -46,6 +46,8 @@ class CMDBase(unittest.TestCase): maxDiff = None + _return_known_from_file = None + _return_ignored_from_file = None _return_find = () @property @@ -56,6 +58,14 @@ def calls(self): self._calls = [] return self._calls + def _known_from_file(self, *args): + self.calls.append(('_known_from_file', args)) + return self._return_known_from_file or {} + + def _ignored_from_file(self, *args): + self.calls.append(('_ignored_from_file', args)) + return self._return_ignored_from_file or {} + def _find(self, *args): self.calls.append(('_find', args)) return self._return_find @@ -70,49 +80,61 @@ def _print(self, *args): class CheckTests(CMDBase): def test_defaults(self): + known = self._return_known_from_file = object() + ignored = self._return_ignored_from_file = object() self._return_find = [] cmd_check('check', + _known_from_file=self._known_from_file, + _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, ) - self.assertEqual(self.calls[0], ( - '_find', ( - SOURCE_DIRS, - KNOWN_FILE, - IGNORED_FILE, - ), - )) + self.assertEqual(self.calls[:3], [ + ('_known_from_file', (KNOWN_FILE,)), + ('_ignored_from_file', (IGNORED_FILE,)), + ('_find', (SOURCE_DIRS, known, ignored)), + ]) def test_all_supported(self): + known = self._return_known_from_file = object() + ignored = self._return_ignored_from_file = object() self._return_find = [(v, s) for v, s in TYPICAL if s] dirs = ['src1', 'src2', 'Include'] cmd_check('check', - dirs, - ignored='ignored.tsv', - known='known.tsv', - _find=self._find, - _show=self._show, - _print=self._print, - ) + dirs, + known='known.tsv', + ignored='ignored.tsv', + _known_from_file=self._known_from_file, + _ignored_from_file=self._ignored_from_file, + _find=self._find, + _show=self._show, + _print=self._print, + ) self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv')), + ('_known_from_file', ('known.tsv',)), + ('_ignored_from_file', ('ignored.tsv',)), + ('_find', (dirs, known, ignored)), #('_print', ('okay',)), ]) def test_some_unsupported(self): + known = self._return_known_from_file = object() + ignored = self._return_ignored_from_file = object() self._return_find = TYPICAL dirs = ['src1', 'src2', 'Include'] with self.assertRaises(SystemExit) as cm: cmd_check('check', dirs, - ignored='ignored.tsv', known='known.tsv', + ignored='ignored.tsv', + _known_from_file=self._known_from_file, + _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, @@ -120,7 +142,9 @@ def test_some_unsupported(self): unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv')), + ('_known_from_file', ('known.tsv',)), + ('_ignored_from_file', ('ignored.tsv',)), + ('_find', (dirs, known, ignored)), ('_print', ('ERROR: found unsupported global variables',)), ('_print', ()), ('_show', (sorted(unsupported),)), @@ -132,23 +156,27 @@ def test_some_unsupported(self): class ShowTests(CMDBase): def test_defaults(self): + known = self._return_known_from_file = object() + ignored = self._return_ignored_from_file = object() self._return_find = [] cmd_show('show', + _known_from_file=self._known_from_file, + _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, ) - self.assertEqual(self.calls[0], ( - '_find', ( - SOURCE_DIRS, - KNOWN_FILE, - IGNORED_FILE, - ), - )) + self.assertEqual(self.calls[:3], [ + ('_known_from_file', (KNOWN_FILE,)), + ('_ignored_from_file', (IGNORED_FILE,)), + ('_find', (SOURCE_DIRS, known, ignored)), + ]) def test_typical(self): + known = self._return_known_from_file = object() + ignored = self._return_ignored_from_file = object() self._return_find = TYPICAL dirs = ['src1', 'src2', 'Include'] @@ -156,6 +184,8 @@ def test_typical(self): dirs, known='known.tsv', ignored='ignored.tsv', + _known_from_file=self._known_from_file, + _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, @@ -164,7 +194,9 @@ def test_typical(self): supported = [v for v, s in TYPICAL if s] unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv')), + ('_known_from_file', ('known.tsv',)), + ('_ignored_from_file', ('ignored.tsv',)), + ('_find', (dirs, known, ignored)), ('_print', ('supported:',)), ('_print', ('----------',)), ('_show', (sorted(supported),)), diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index 42f71b0ccba466..b7b3153ef545ca 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -13,30 +13,29 @@ from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object -def _match_unused_global(variable, knownvars, used): - found = [] - for varid in knownvars: - if varid in used: - continue - if varid.funcname is not None: - continue - if varid.name != variable.name: - continue - if variable.filename and variable.filename != UNKNOWN: - if variable.filename == varid.filename: +def _check_results(unknown, knownvars, used): + def _match_unused_global(variable): + found = [] + for varid in knownvars: + if varid in used: + continue + if varid.funcname is not None: + continue + if varid.name != variable.name: + continue + if variable.filename and variable.filename != UNKNOWN: + if variable.filename == varid.filename: + found.append(varid) + else: found.append(varid) - else: - found.append(varid) - return found + return found - -def _check_results(unknown, knownvars, used): badknown = set() for variable in sorted(unknown): msg = None if variable.funcname != UNKNOWN: msg = f'could not find global symbol {variable.id}' - elif m := _match_unused_global(variable, knownvars, used): + elif m := _match_unused_global(variable): assert isinstance(m, list) badknown.update(m) elif variable.name in ('completed', 'id'): # XXX Figure out where these variables are. @@ -69,27 +68,21 @@ def _find_globals(dirnames, known, ignored): if dirnames == SOURCE_DIRS: dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames] - ignored = ignored_from_file(ignored) - known = known_from_file(known) - - used = set() - unknown = set() knownvars = (known or {}).get('variables') for variable in find.globals_from_binary(knownvars=knownvars, dirnames=dirnames): - #for variable in find.globals(dirnames, known, kind='platform'): if variable.vartype == UNKNOWN: - unknown.add(variable) - continue - yield variable, is_supported(variable, ignored, known) - used.add(variable.id) - - #_check_results(unknown, knownvars, used) + yield variable, None + else: + yield variable, is_supported(variable, ignored, known) def cmd_check(cmd, dirs=SOURCE_DIRS, *, - ignored=IGNORED_FILE, known=KNOWN_FILE, + ignored=IGNORED_FILE, + checkfound=False, + _known_from_file=known_from_file, + _ignored_from_file=ignored_from_file, _find=_find_globals, _show=show.basic, _print=print, @@ -100,7 +93,24 @@ def cmd_check(cmd, dirs=SOURCE_DIRS, *, In the failure case, the list of unsupported variables will be printed out. """ - unsupported = [v for v, s in _find(dirs, known, ignored) if not s] + known = _known_from_file(known) + ignored = _ignored_from_file(ignored) + + used = set() + unknown = set() + unsupported = [] + for var, supported in _find(dirs, known, ignored): + if supported is None: + unknown.add(var) + continue + used.add(var.id) + if not supported: + unsupported.append(var) + + if checkfound: + # XXX Move this check to its own command. + _check_results(unknown, known['variables'], used) + if not unsupported: #_print('okay') return @@ -113,10 +123,12 @@ def cmd_check(cmd, dirs=SOURCE_DIRS, *, def cmd_show(cmd, dirs=SOURCE_DIRS, *, - ignored=IGNORED_FILE, known=KNOWN_FILE, + ignored=IGNORED_FILE, skip_objects=False, - _find=_find_globals, + _known_from_file=known_from_file, + _ignored_from_file=ignored_from_file, + _find=_find_globals, _show=show.basic, _print=print, ): @@ -125,9 +137,14 @@ def cmd_show(cmd, dirs=SOURCE_DIRS, *, The variables will be distinguished as "supported" or "unsupported". """ + known = _known_from_file(known) + ignored = _ignored_from_file(ignored) + allsupported = [] allunsupported = [] for found, supported in _find(dirs, known, ignored): + if supported is None: + continue if skip_objects: # XXX Support proper filters instead. if _is_object(found.vartype): continue From bf3bd21dbc8fbc139ceb369f26195a56f156c0d2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 27 Sep 2019 13:25:01 -0600 Subject: [PATCH 03/27] Merge c_parser.info into c_analyzer_common.info. --- .../test_c_analyzer_common/test_info.py | 240 ++++++++++++++++- .../test_c_analyzer_common/test_known.py | 3 +- .../test_c_analyzer_common/test_show.py | 2 +- .../test_c_globals/test___main__.py | 3 +- .../test_c_globals/test_find.py | 2 +- .../test_c_globals/test_supported.py | 11 +- .../test_c_parser/test_info.py | 244 ------------------ .../c-analyzer/c_analyzer_common/_generate.py | 3 +- Tools/c-analyzer/c_analyzer_common/info.py | 104 ++++++++ Tools/c-analyzer/c_analyzer_common/known.py | 4 +- Tools/c-analyzer/c_globals/find.py | 8 +- Tools/c-analyzer/c_parser/info.py | 106 -------- Tools/c-analyzer/c_parser/naive.py | 3 +- Tools/c-analyzer/c_parser/preprocessor.py | 3 +- Tools/c-analyzer/c_symbols/resolve.py | 10 +- 15 files changed, 365 insertions(+), 381 deletions(-) delete mode 100644 Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py delete mode 100644 Tools/c-analyzer/c_parser/info.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py index 2d386713b9989c..671cc511410fe6 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py @@ -4,7 +4,11 @@ from ..util import PseudoStr, StrProxy, Object from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID + from c_analyzer_common.info import ( + UNKNOWN, + ID, + normalize_vartype, Variable + ) class IDTests(unittest.TestCase): @@ -192,3 +196,237 @@ def test_validate_bad_field(self): id = id._replace(**{field: value}) id.validate() # This does not fail. + + +class NormalizeVartypeTests(unittest.TestCase): + + def test_basic(self): + tests = [ + (None, None), + ('', ''), + ('int', 'int'), + (PseudoStr('int'), 'int'), + (StrProxy('int'), 'int'), + ] + for vartype, expected in tests: + with self.subTest(vartype): + normalized = normalize_vartype(vartype) + + self.assertEqual(normalized, expected) + + +class VariableTests(unittest.TestCase): + + VALID_ARGS = ( + ('x/y/z/spam.c', 'func', 'eggs'), + 'static', + 'int', + ) + VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS)) + VALID_EXPECTED = VALID_ARGS + + def test_init_typical_global(self): + for storage in ('static', 'extern', 'implicit'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname=None, + name='eggs', + ), + storage=storage, + vartype='int', + ) + + self.assertEqual(static, ( + ('x/y/z/spam.c', None, 'eggs'), + storage, + 'int', + )) + + def test_init_typical_local(self): + for storage in ('static', 'local'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) + + self.assertEqual(static, ( + ('x/y/z/spam.c', 'func', 'eggs'), + storage, + 'int', + )) + + def test_init_all_missing(self): + for value in ('', None): + with self.subTest(repr(value)): + static = Variable( + id=value, + storage=value, + vartype=value, + ) + + self.assertEqual(static, ( + None, + None, + None, + )) + + def test_init_all_coerced(self): + id = ID('x/y/z/spam.c', 'func', 'spam') + tests = [ + ('str subclass', + dict( + id=( + PseudoStr('x/y/z/spam.c'), + PseudoStr('func'), + PseudoStr('spam'), + ), + storage=PseudoStr('static'), + vartype=PseudoStr('int'), + ), + (id, + 'static', + 'int', + )), + ('non-str 1', + dict( + id=id, + storage=Object(), + vartype=Object(), + ), + (id, + '', + '', + )), + ('non-str 2', + dict( + id=id, + storage=StrProxy('static'), + vartype=StrProxy('variable'), + ), + (id, + 'static', + 'variable', + )), + ('non-str', + dict( + id=id, + storage=('a', 'b', 'c'), + vartype=('x', 'y', 'z'), + ), + (id, + "('a', 'b', 'c')", + "('x', 'y', 'z')", + )), + ] + for summary, kwargs, expected in tests: + with self.subTest(summary): + static = Variable(**kwargs) + + for field in Variable._fields: + value = getattr(static, field) + if field == 'id': + self.assertIs(type(value), ID) + else: + self.assertIs(type(value), str) + self.assertEqual(tuple(static), expected) + + def test_iterable(self): + static = Variable(**self.VALID_KWARGS) + + id, storage, vartype = static + + values = (id, storage, vartype) + for value, expected in zip(values, self.VALID_EXPECTED): + self.assertEqual(value, expected) + + def test_fields(self): + static = Variable(('a', 'b', 'z'), 'x', 'y') + + self.assertEqual(static.id, ('a', 'b', 'z')) + self.assertEqual(static.storage, 'x') + self.assertEqual(static.vartype, 'y') + + def test___getattr__(self): + static = Variable(('a', 'b', 'z'), 'x', 'y') + + self.assertEqual(static.filename, 'a') + self.assertEqual(static.funcname, 'b') + self.assertEqual(static.name, 'z') + + def test_validate_typical(self): + validstorage = ('static', 'extern', 'implicit', 'local') + self.assertEqual(set(validstorage), set(Variable.STORAGE)) + + for storage in validstorage: + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) + + static.validate() # This does not fail. + + def test_validate_missing_field(self): + for field in Variable._fields: + with self.subTest(field): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: None}) + + with self.assertRaises(TypeError): + static.validate() + for field in ('storage', 'vartype'): + with self.subTest(field): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: UNKNOWN}) + + with self.assertRaises(TypeError): + static.validate() + + def test_validate_bad_field(self): + badch = tuple(c for c in string.punctuation + string.digits) + notnames = ( + '1a', + 'a.b', + 'a-b', + '&a', + 'a++', + ) + badch + tests = [ + ('id', ()), # Any non-empty str is okay. + ('storage', ('external', 'global') + notnames), + ('vartype', ()), # Any non-empty str is okay. + ] + seen = set() + for field, invalid in tests: + for value in invalid: + seen.add(value) + with self.subTest(f'{field}={value!r}'): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: value}) + + with self.assertRaises(ValueError): + static.validate() + + for field, invalid in tests: + if field == 'id': + continue + valid = seen - set(invalid) + for value in valid: + with self.subTest(f'{field}={value!r}'): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: value}) + + static.validate() # This does not fail. diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py index 215023da577b9c..99753ff6d6023e 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py @@ -4,8 +4,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_parser.info import Variable - from c_analyzer_common.info import ID + from c_analyzer_common.info import ID, Variable from c_analyzer_common.known import from_file diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py index b689aafb19314c..b19656961fc3a5 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py @@ -2,7 +2,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_parser import info + from c_analyzer_common import info from c_analyzer_common.show import basic diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py index 21c850a7a541ba..b604e1c00fd082 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py @@ -3,9 +3,8 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common import SOURCE_DIRS + from c_analyzer_common import SOURCE_DIRS, info from c_analyzer_common.known import DATA_FILE as KNOWN_FILE - from c_parser import info import c_globals as cg from c_globals.supported import IGNORED_FILE from c_globals.__main__ import cmd_check, cmd_show, parse_args, main diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py index 828899201b7cad..b7a1caa648817c 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py @@ -2,7 +2,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_parser import info + from c_analyzer_common import info from c_globals.find import globals_from_binary, globals diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py index 1e7d40e2afcbda..696b5612cccb43 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py @@ -4,8 +4,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID - from c_parser import info + from c_analyzer_common.info import ID, Variable from c_globals.supported import is_supported, ignored_from_file @@ -14,8 +13,8 @@ class IsSupportedTests(unittest.TestCase): @unittest.expectedFailure def test_supported(self): statics = [ - info.StaticVar('src1/spam.c', None, 'var1', 'const char *'), - info.StaticVar('src1/spam.c', None, 'var1', 'int'), + Variable('src1/spam.c', None, 'var1', 'const char *'), + Variable('src1/spam.c', None, 'var1', 'int'), ] for static in statics: with self.subTest(static): @@ -26,8 +25,8 @@ def test_supported(self): @unittest.expectedFailure def test_not_supported(self): statics = [ - info.StaticVar('src1/spam.c', None, 'var1', 'PyObject *'), - info.StaticVar('src1/spam.c', None, 'var1', 'PyObject[10]'), + Variable('src1/spam.c', None, 'var1', 'PyObject *'), + Variable('src1/spam.c', None, 'var1', 'PyObject[10]'), ] for static in statics: with self.subTest(static): diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py deleted file mode 100644 index d1a966c58904db..00000000000000 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py +++ /dev/null @@ -1,244 +0,0 @@ -import string -import unittest - -from ..util import PseudoStr, StrProxy, Object -from .. import tool_imports_for_tests -with tool_imports_for_tests(): - from c_analyzer_common.info import ID, UNKNOWN - from c_parser.info import ( - normalize_vartype, Variable, - ) - - -class NormalizeVartypeTests(unittest.TestCase): - - def test_basic(self): - tests = [ - (None, None), - ('', ''), - ('int', 'int'), - (PseudoStr('int'), 'int'), - (StrProxy('int'), 'int'), - ] - for vartype, expected in tests: - with self.subTest(vartype): - normalized = normalize_vartype(vartype) - - self.assertEqual(normalized, expected) - - -class VariableTests(unittest.TestCase): - - VALID_ARGS = ( - ('x/y/z/spam.c', 'func', 'eggs'), - 'static', - 'int', - ) - VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS)) - VALID_EXPECTED = VALID_ARGS - - def test_init_typical_global(self): - for storage in ('static', 'extern', 'implicit'): - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname=None, - name='eggs', - ), - storage=storage, - vartype='int', - ) - - self.assertEqual(static, ( - ('x/y/z/spam.c', None, 'eggs'), - storage, - 'int', - )) - - def test_init_typical_local(self): - for storage in ('static', 'local'): - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - storage=storage, - vartype='int', - ) - - self.assertEqual(static, ( - ('x/y/z/spam.c', 'func', 'eggs'), - storage, - 'int', - )) - - def test_init_all_missing(self): - for value in ('', None): - with self.subTest(repr(value)): - static = Variable( - id=value, - storage=value, - vartype=value, - ) - - self.assertEqual(static, ( - None, - None, - None, - )) - - def test_init_all_coerced(self): - id = ID('x/y/z/spam.c', 'func', 'spam') - tests = [ - ('str subclass', - dict( - id=( - PseudoStr('x/y/z/spam.c'), - PseudoStr('func'), - PseudoStr('spam'), - ), - storage=PseudoStr('static'), - vartype=PseudoStr('int'), - ), - (id, - 'static', - 'int', - )), - ('non-str 1', - dict( - id=id, - storage=Object(), - vartype=Object(), - ), - (id, - '', - '', - )), - ('non-str 2', - dict( - id=id, - storage=StrProxy('static'), - vartype=StrProxy('variable'), - ), - (id, - 'static', - 'variable', - )), - ('non-str', - dict( - id=id, - storage=('a', 'b', 'c'), - vartype=('x', 'y', 'z'), - ), - (id, - "('a', 'b', 'c')", - "('x', 'y', 'z')", - )), - ] - for summary, kwargs, expected in tests: - with self.subTest(summary): - static = Variable(**kwargs) - - for field in Variable._fields: - value = getattr(static, field) - if field == 'id': - self.assertIs(type(value), ID) - else: - self.assertIs(type(value), str) - self.assertEqual(tuple(static), expected) - - def test_iterable(self): - static = Variable(**self.VALID_KWARGS) - - id, storage, vartype = static - - values = (id, storage, vartype) - for value, expected in zip(values, self.VALID_EXPECTED): - self.assertEqual(value, expected) - - def test_fields(self): - static = Variable(('a', 'b', 'z'), 'x', 'y') - - self.assertEqual(static.id, ('a', 'b', 'z')) - self.assertEqual(static.storage, 'x') - self.assertEqual(static.vartype, 'y') - - def test___getattr__(self): - static = Variable(('a', 'b', 'z'), 'x', 'y') - - self.assertEqual(static.filename, 'a') - self.assertEqual(static.funcname, 'b') - self.assertEqual(static.name, 'z') - - def test_validate_typical(self): - validstorage = ('static', 'extern', 'implicit', 'local') - self.assertEqual(set(validstorage), set(Variable.STORAGE)) - - for storage in validstorage: - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - storage=storage, - vartype='int', - ) - - static.validate() # This does not fail. - - def test_validate_missing_field(self): - for field in Variable._fields: - with self.subTest(field): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: None}) - - with self.assertRaises(TypeError): - static.validate() - for field in ('storage', 'vartype'): - with self.subTest(field): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: UNKNOWN}) - - with self.assertRaises(TypeError): - static.validate() - - def test_validate_bad_field(self): - badch = tuple(c for c in string.punctuation + string.digits) - notnames = ( - '1a', - 'a.b', - 'a-b', - '&a', - 'a++', - ) + badch - tests = [ - ('id', ()), # Any non-empty str is okay. - ('storage', ('external', 'global') + notnames), - ('vartype', ()), # Any non-empty str is okay. - ] - seen = set() - for field, invalid in tests: - for value in invalid: - seen.add(value) - with self.subTest(f'{field}={value!r}'): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: value}) - - with self.assertRaises(ValueError): - static.validate() - - for field, invalid in tests: - if field == 'id': - continue - valid = seen - set(invalid) - for value in valid: - with self.subTest(f'{field}={value!r}'): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: value}) - - static.validate() # This does not fail. diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer_common/_generate.py index 9b2fc9edb5c824..8843af0b37c82e 100644 --- a/Tools/c-analyzer/c_analyzer_common/_generate.py +++ b/Tools/c-analyzer/c_analyzer_common/_generate.py @@ -4,11 +4,10 @@ from c_parser.naive import ( iter_variables, parse_variable_declaration, find_variables, ) -from c_parser.info import Variable from . import SOURCE_DIRS, REPO_ROOT from .known import DATA_FILE as KNOWN_FILE, HEADER as KNOWN_HEADER -from .info import UNKNOWN, ID +from .info import UNKNOWN, ID, Variable from .util import write_tsv from .files import iter_cpython_files diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer_common/info.py index e2173804064576..cdf5c216543ba1 100644 --- a/Tools/c-analyzer/c_analyzer_common/info.py +++ b/Tools/c-analyzer/c_analyzer_common/info.py @@ -67,3 +67,107 @@ def validate(self): @property def islocal(self): return self.funcname is not None + + +############################# +# variables + +def normalize_vartype(vartype): + """Return the canonical form for a variable type (or func signature).""" + # We allow empty strring through for semantic reasons. + if vartype is None: + return None + + # XXX finish! + # XXX Return (modifiers, type, pointer)? + return str(vartype) + + +def extract_storage(decl, *, isfunc=False): + """Return (storage, vartype) based on the given declaration. + + The default storage is "implicit" or "local". + """ + if decl == UNKNOWN: + return decl, decl + if decl.startswith('static '): + return 'static', decl + #return 'static', decl.partition(' ')[2].strip() + elif decl.startswith('extern '): + return 'extern', decl + #return 'extern', decl.partition(' ')[2].strip() + elif re.match('.*\b(static|extern)\b', decl): + raise NotImplementedError + elif isfunc: + return 'local', decl + else: + return 'implicit', decl + + +class Variable(_NTBase, + namedtuple('Variable', 'id storage vartype')): + """Information about a single variable declaration.""" + + __slots__ = () + + STORAGE = ( + 'static', + 'extern', + 'implicit', + 'local', + ) + + @classonly + def from_parts(cls, filename, funcname, name, decl, storage=None): + if storage is None: + storage, decl = extract_storage(decl, isfunc=funcname) + id = ID(filename, funcname, name) + self = cls(id, storage, decl) + return self + + def __new__(cls, id, storage, vartype): + self = super().__new__( + cls, + id=ID.from_raw(id), + storage=str(storage) if storage else None, + vartype=normalize_vartype(vartype) if vartype else None, + ) + return self + + def __hash__(self): + return hash(self.id) + + def __getattr__(self, name): + return getattr(self.id, name) + + def _validate_id(self): + if not self.id: + raise TypeError('missing id') + + if not self.filename or self.filename == UNKNOWN: + raise TypeError(f'id missing filename ({self.id})') + + if self.funcname and self.funcname == UNKNOWN: + raise TypeError(f'id missing funcname ({self.id})') + + self.id.validate() + + def validate(self): + """Fail if the object is invalid (i.e. init with bad data).""" + self._validate_id() + + if self.storage is None or self.storage == UNKNOWN: + raise TypeError('missing storage') + elif self.storage not in self.STORAGE: + raise ValueError(f'unsupported storage {self.storage:r}') + + if self.vartype is None or self.vartype == UNKNOWN: + raise TypeError('missing vartype') + + @property + def isglobal(self): + return self.storage != 'local' + + @property + def isconst(self): + return 'const' in self.vartype.split() diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py index dec1e1d2e09273..b39755efa5ce0b 100644 --- a/Tools/c-analyzer/c_analyzer_common/known.py +++ b/Tools/c-analyzer/c_analyzer_common/known.py @@ -1,10 +1,8 @@ import csv import os.path -from c_parser.info import Variable - from . import DATA_DIR -from .info import ID, UNKNOWN +from .info import ID, UNKNOWN, Variable from .util import read_tsv diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py index a51b947cbdf10d..8d82ed398a1002 100644 --- a/Tools/c-analyzer/c_globals/find.py +++ b/Tools/c-analyzer/c_globals/find.py @@ -1,12 +1,12 @@ from c_analyzer_common import SOURCE_DIRS -from c_analyzer_common.info import UNKNOWN +from c_analyzer_common.info import UNKNOWN, Variable from c_symbols import ( info as s_info, binary as b_symbols, source as s_symbols, resolve, ) -from c_parser import info, declarations +from c_parser import declarations # XXX needs tests: @@ -72,12 +72,12 @@ def iter_variables(kind='platform', *, ) elif kind == 'declarations': for decl in _iter_raw(dirnames): - if not isinstance(decl, info.Variable): + if not isinstance(decl, Variable): continue yield decl elif kind == 'preprocessed': for decl in _iter_preprocessed(dirnames): - if not isinstance(decl, info.Variable): + if not isinstance(decl, Variable): continue yield decl else: diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py deleted file mode 100644 index a4e32d75eed73f..00000000000000 --- a/Tools/c-analyzer/c_parser/info.py +++ /dev/null @@ -1,106 +0,0 @@ -from collections import namedtuple -import re - -from c_analyzer_common import info, util -from c_analyzer_common.util import classonly, _NTBase - - -def normalize_vartype(vartype): - """Return the canonical form for a variable type (or func signature).""" - # We allow empty strring through for semantic reasons. - if vartype is None: - return None - - # XXX finish! - # XXX Return (modifiers, type, pointer)? - return str(vartype) - - -def extract_storage(decl, *, isfunc=False): - """Return (storage, vartype) based on the given declaration. - - The default storage is "implicit" or "local". - """ - if decl == info.UNKNOWN: - return decl, decl - if decl.startswith('static '): - return 'static', decl - #return 'static', decl.partition(' ')[2].strip() - elif decl.startswith('extern '): - return 'extern', decl - #return 'extern', decl.partition(' ')[2].strip() - elif re.match('.*\b(static|extern)\b', decl): - raise NotImplementedError - elif isfunc: - return 'local', decl - else: - return 'implicit', decl - - -class Variable(_NTBase, - namedtuple('Variable', 'id storage vartype')): - """Information about a single variable declaration.""" - - __slots__ = () - - STORAGE = ( - 'static', - 'extern', - 'implicit', - 'local', - ) - - @classonly - def from_parts(cls, filename, funcname, name, decl, storage=None): - if storage is None: - storage, decl = extract_storage(decl, isfunc=funcname) - id = info.ID(filename, funcname, name) - self = cls(id, storage, decl) - return self - - def __new__(cls, id, storage, vartype): - self = super().__new__( - cls, - id=info.ID.from_raw(id), - storage=str(storage) if storage else None, - vartype=normalize_vartype(vartype) if vartype else None, - ) - return self - - def __hash__(self): - return hash(self.id) - - def __getattr__(self, name): - return getattr(self.id, name) - - def _validate_id(self): - if not self.id: - raise TypeError('missing id') - - if not self.filename or self.filename == info.UNKNOWN: - raise TypeError(f'id missing filename ({self.id})') - - if self.funcname and self.funcname == info.UNKNOWN: - raise TypeError(f'id missing funcname ({self.id})') - - self.id.validate() - - def validate(self): - """Fail if the object is invalid (i.e. init with bad data).""" - self._validate_id() - - if self.storage is None or self.storage == info.UNKNOWN: - raise TypeError('missing storage') - elif self.storage not in self.STORAGE: - raise ValueError(f'unsupported storage {self.storage:r}') - - if self.vartype is None or self.vartype == info.UNKNOWN: - raise TypeError('missing vartype') - - @property - def isglobal(self): - return self.storage != 'local' - - @property - def isconst(self): - return 'const' in self.vartype.split() diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_parser/naive.py index 160f96c279e261..dc4e0278d12ed3 100644 --- a/Tools/c-analyzer/c_parser/naive.py +++ b/Tools/c-analyzer/c_parser/naive.py @@ -1,8 +1,7 @@ import re -from c_analyzer_common.info import UNKNOWN +from c_analyzer_common.info import UNKNOWN, Variable -from .info import Variable from .preprocessor import _iter_clean_lines diff --git a/Tools/c-analyzer/c_parser/preprocessor.py b/Tools/c-analyzer/c_parser/preprocessor.py index 0e2866e4873e67..35b807a6c62978 100644 --- a/Tools/c-analyzer/c_parser/preprocessor.py +++ b/Tools/c-analyzer/c_parser/preprocessor.py @@ -3,8 +3,7 @@ import os import re -from c_analyzer_common import util -from . import info +from c_analyzer_common import util, info CONTINUATION = '\\' + os.linesep diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py index 56210cefeb8269..c031eea03d0143 100644 --- a/Tools/c-analyzer/c_symbols/resolve.py +++ b/Tools/c-analyzer/c_symbols/resolve.py @@ -1,8 +1,8 @@ import os.path from c_analyzer_common import files -from c_analyzer_common.info import UNKNOWN -from c_parser import declarations, info +from c_analyzer_common.info import UNKNOWN, Variable +from c_parser import declarations from .info import Symbol from .source import _find_symbol @@ -72,7 +72,7 @@ def find_in_source(symbol, dirnames, *, ) = _find_symbol(symbol.name, filenames, _perfilecache) if filename == UNKNOWN: return None - return info.Variable.from_parts(filename, funcname, symbol.name, decl) + return Variable.from_parts(filename, funcname, symbol.name, decl) def get_resolver(knownvars=None, dirnames=None, *, @@ -130,7 +130,7 @@ def symbols_to_variables(symbols, *, Use get_resolver() for a "resolve" func to use. """ for symbol in symbols: - if isinstance(symbol, info.Variable): + if isinstance(symbol, Variable): # XXX validate? yield symbol continue @@ -139,7 +139,7 @@ def symbols_to_variables(symbols, *, resolved = resolve(symbol) if resolved is None: #raise NotImplementedError(symbol) - resolved = info.Variable( + resolved = Variable( id=symbol.id, storage=UNKNOWN, vartype=UNKNOWN, From 52113aeb4790b7bd3ec7cea59deacdebd25a5e87 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 27 Sep 2019 13:54:24 -0600 Subject: [PATCH 04/27] c_symbols.binary -> c_symbols.find. --- .../c-analyzer/c_analyzer_common/__init__.py | 5 ++ Tools/c-analyzer/c_globals/find.py | 6 +- .../c_symbols/{binary.py => _nm.py} | 76 ++++++------------- Tools/c-analyzer/c_symbols/find.py | 42 ++++++++++ 4 files changed, 73 insertions(+), 56 deletions(-) rename Tools/c-analyzer/c_symbols/{binary.py => _nm.py} (68%) create mode 100644 Tools/c-analyzer/c_symbols/find.py diff --git a/Tools/c-analyzer/c_analyzer_common/__init__.py b/Tools/c-analyzer/c_analyzer_common/__init__.py index 888b16ff41d10d..25fa5e1afc246f 100644 --- a/Tools/c-analyzer/c_analyzer_common/__init__.py +++ b/Tools/c-analyzer/c_analyzer_common/__init__.py @@ -1,4 +1,5 @@ import os.path +import sys PKG_ROOT = os.path.dirname(__file__) @@ -14,6 +15,10 @@ 'Modules', ]] +#PYTHON = os.path.join(REPO_ROOT, 'python') +PYTHON = sys.executable + # Clean up the namespace. +del sys del os diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py index 8d82ed398a1002..fb55a6e8335c14 100644 --- a/Tools/c-analyzer/c_globals/find.py +++ b/Tools/c-analyzer/c_globals/find.py @@ -1,8 +1,8 @@ -from c_analyzer_common import SOURCE_DIRS +from c_analyzer_common import SOURCE_DIRS, PYTHON from c_analyzer_common.info import UNKNOWN, Variable from c_symbols import ( info as s_info, - binary as b_symbols, + find as b_symbols, source as s_symbols, resolve, ) @@ -12,7 +12,7 @@ # XXX needs tests: # * iter_variables -def globals_from_binary(binfile=b_symbols.PYTHON, *, +def globals_from_binary(binfile=PYTHON, *, knownvars=None, dirnames=None, _iter_symbols=b_symbols.iter_symbols, diff --git a/Tools/c-analyzer/c_symbols/binary.py b/Tools/c-analyzer/c_symbols/_nm.py similarity index 68% rename from Tools/c-analyzer/c_symbols/binary.py rename to Tools/c-analyzer/c_symbols/_nm.py index e125dbd5b5edc5..c933f536193851 100644 --- a/Tools/c-analyzer/c_symbols/binary.py +++ b/Tools/c-analyzer/c_symbols/_nm.py @@ -1,46 +1,24 @@ -import os import os.path import shutil -import sys from c_analyzer_common import util, info -from . import source -from .info import Symbol - -#PYTHON = os.path.join(REPO_ROOT, 'python') -PYTHON = sys.executable +from .info import Symbol -def iter_symbols(binary=PYTHON, dirnames=None, *, - # Alternately, use look_up_known_symbol() - # from c_globals.supported. - find_local_symbol=source.find_symbol, - _file_exists=os.path.exists, - _iter_symbols_nm=(lambda b, *a: _iter_symbols_nm(b, *a)), - ): - """Yield a Symbol for each symbol found in the binary.""" - if not _file_exists(binary): - raise Exception('executable missing (need to build it first?)') - - if find_local_symbol: - cache = {} - def find_local_symbol(name, *, _find=find_local_symbol): - return _find(name, dirnames, _perfilecache=cache) - else: - find_local_symbol = None - - if os.name == 'nt': - # XXX Support this. - raise NotImplementedError - else: - yield from _iter_symbols_nm(binary, find_local_symbol) +# XXX need tests: +# * iter_symbols - -############################# -# binary format (e.g. ELF) +NM_KINDS = { + 'b': Symbol.KIND.VARIABLE, # uninitialized + 'd': Symbol.KIND.VARIABLE, # initialized + #'g': Symbol.KIND.VARIABLE, # uninitialized + #'s': Symbol.KIND.VARIABLE, # initialized + 't': Symbol.KIND.FUNCTION, + } SPECIAL_SYMBOLS = { + # binary format (e.g. ELF) '__bss_start', '__data_start', '__dso_handle', @@ -63,29 +41,21 @@ def _is_special_symbol(name): return False -############################# -# "nm" - -NM_KINDS = { - 'b': Symbol.KIND.VARIABLE, # uninitialized - 'd': Symbol.KIND.VARIABLE, # initialized - #'g': Symbol.KIND.VARIABLE, # uninitialized - #'s': Symbol.KIND.VARIABLE, # initialized - 't': Symbol.KIND.FUNCTION, - } - +def iter_symbols(binfile, find_local_symbol=None, + *, + nm=None, + _which=shutil.which, + _run=util.run_cmd, + ): + """Yield a Symbol for each relevant entry reported by the "nm" command.""" + if nm is None: + nm = _which('nm') + if not nm: + raise NotImplementedError -def _iter_symbols_nm(binary, find_local_symbol=None, - *, - _which=shutil.which, - _run=util.run_cmd, - ): - nm = _which('nm') - if not nm: - raise NotImplementedError argv = [nm, '--line-numbers', - binary, + binfile, ] try: output = _run(argv) diff --git a/Tools/c-analyzer/c_symbols/find.py b/Tools/c-analyzer/c_symbols/find.py new file mode 100644 index 00000000000000..8d7d3772f31d88 --- /dev/null +++ b/Tools/c-analyzer/c_symbols/find.py @@ -0,0 +1,42 @@ +import os +import os.path +import shutil + +from c_analyzer_common import PYTHON + +from . import _nm, source + +# XXX need tests: +# * iter_symbols + + +def _get_platform_tool(): + if os.name == 'nt': + # XXX Support this. + raise NotImplementedError + elif nm := shutil.which('nm'): + return lambda b, fls: _nm.iter_symbols(b, fls, nm=nm) + else: + raise NotImplementedError + + +def iter_symbols(binfile=PYTHON, dirnames=None, *, + # Alternately, use look_up_known_symbol() + # from c_globals.supported. + find_local_symbol=source.find_symbol, + _file_exists=os.path.exists, + _get_platform_tool=_get_platform_tool, + ): + """Yield a Symbol for each symbol found in the binary.""" + if not _file_exists(binfile): + raise Exception('executable missing (need to build it first?)') + + if find_local_symbol: + cache = {} + def find_local_symbol(name, *, _find=find_local_symbol): + return _find(name, dirnames, _perfilecache=cache) + else: + find_local_symbol = None + + _iter_symbols = _get_platform_tool() + yield from _iter_symbols(binfile, find_local_symbol) From dc8333fb94f7fc50e708b60e4ee1c858880bbee1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 11:51:11 -0600 Subject: [PATCH 05/27] Add ID.match(). --- Tools/c-analyzer/c_analyzer_common/info.py | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer_common/info.py index cdf5c216543ba1..9248b39c90ec50 100644 --- a/Tools/c-analyzer/c_analyzer_common/info.py +++ b/Tools/c-analyzer/c_analyzer_common/info.py @@ -3,6 +3,9 @@ from .util import classonly, _NTBase +# XXX need tests: +# * ID.match() + UNKNOWN = '???' @@ -68,6 +71,72 @@ def validate(self): def islocal(self): return self.funcname is not None + def match(self, other, *, + match_files=(lambda f1, f2: f1 == f2), + ): + """Return True if the two match. + + At least one of the two must be completely valid (no UNKNOWN + anywhere). Otherwise False is returned. The remaining one + *may* have UNKNOWN for both funcname and filename. It must + have a valid name though. + + The caller is responsible for knowing which of the two is valid + (and which to use if both are valid). + """ + # First check the name. + if self.name is None: + return False + if other.name != self.name: + return False + + # Then check the filename. + if self.filename is None: + return False + if other.filename is None: + return False + if self.filename == UNKNOWN: + # "other" must be the valid one. + if other.funcname == UNKNOWN: + return False + elif self.funcname != UNKNOWN: + # XXX Try matching funcname even though we don't + # know the filename? + raise NotImplementedError + else: + return True + elif other.filename == UNKNOWN: + # "self" must be the valid one. + if self.funcname == UNKNOWN: + return False + elif other.funcname != UNKNOWN: + # XXX Try matching funcname even though we don't + # know the filename? + raise NotImplementedError + else: + return True + elif not match_files(self.filename, other.filename): + return False + + # Finally, check the funcname. + if self.funcname == UNKNOWN: + # "other" must be the valid one. + if other.funcname == UNKNOWN: + return False + else: + return other.funcname is not None + elif other.funcname == UNKNOWN: + # "self" must be the valid one. + if self.funcname == UNKNOWN: + return False + else: + return self.funcname is not None + elif self.funcname == other.funcname: + # Both are valid. + return True + + return False + ############################# # variables From 9b475964c9d3b47bbb22c726a6be342cbb0f7fc6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 11:52:12 -0600 Subject: [PATCH 06/27] Add known.look_up_variable(). --- Tools/c-analyzer/c_analyzer_common/known.py | 47 +++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py index b39755efa5ce0b..0c1f3fbc5c4d63 100644 --- a/Tools/c-analyzer/c_analyzer_common/known.py +++ b/Tools/c-analyzer/c_analyzer_common/known.py @@ -5,6 +5,10 @@ from .info import ID, UNKNOWN, Variable from .util import read_tsv +# XXX need tests: +# * from_file() +# * look_up_variable() + DATA_FILE = os.path.join(DATA_DIR, 'known.tsv') @@ -12,9 +16,6 @@ HEADER = '\t'.join(COLUMNS) -# XXX need tests: -# * from_file() - def from_file(infile, *, _read_tsv=read_tsv, ): @@ -70,3 +71,43 @@ def _get_storage(decl): return 'extern' # implicit or local return None + + +def look_up_variable(varid, knownvars, *, + match_files=(lambda f1, f2: f1 == f2), + ): + """Return the known variable matching the given ID. + + "knownvars" is a mapping of ID to Variable. + + "match_files" is used to verify if two filenames point to + the same file. + + If no match is found then None is returned. + """ + if not knownvars: + return None + + if varid.funcname == UNKNOWN: + if not varid.filename or varid.filename == UNKNOWN: + for varid in knownvars: + if not varid.funcname: + continue + if varid.name == varid.name: + return knownvars[varid] + else: + return None + else: + for varid in knownvars: + if not varid.funcname: + continue + if not match_files(varid.filename, varid.filename): + continue + if varid.name == varid.name: + return knownvars[varid] + else: + return None + elif not varid.filename or varid.filename == UNKNOWN: + raise NotImplementedError + else: + return knownvars.get(varid.id) From 67eea78d68478ad632804942e62c389403179cb8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 11:59:05 -0600 Subject: [PATCH 07/27] Add c_parser.find module. --- Tools/c-analyzer/c_parser/find.py | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 Tools/c-analyzer/c_parser/find.py diff --git a/Tools/c-analyzer/c_parser/find.py b/Tools/c-analyzer/c_parser/find.py new file mode 100644 index 00000000000000..524843f76f4502 --- /dev/null +++ b/Tools/c-analyzer/c_parser/find.py @@ -0,0 +1,76 @@ +from c_analyzer_common.info import Variable, UNKNOWN + +from . import declarations + +# XXX need tests: +# * variables +# * variable +# * variable_from_id + + +def _iter_vars(filenames, + _iter_variables=declarations.iter_variables, + ): + for filename in filenames or (): + for funcname, name, decl in _iter_variables(filename): + yield Variable.from_parts(filename, funcname, name, decl) + + +def variables(*filenames, + perfilecache=None, + _iter_vars=_iter_vars, + ): + """Yield a Variable for each one found in the given files.""" + if perfilecache is None: + yield from _iter_vars(filenames) + else: + # XXX Cache per-file variables (e.g. `{filename: [Variable]}`). + raise NotImplementedError + + +def variable(name, filenames, *, + local=False, + perfilecache=None, + _iter_variables=variables, + ): + """Return the first found Variable that matches. + + If "local" is True then the first matching local variable in the + file will always be returned. To avoid that, pass perfilecache and + pop each variable from the cache after using it. + """ + for var in _iter_variables(filenames, perfilecache=perfilecache): + if var.name != name: + continue + if local: + if var.funcname: + if var.funcname == UNKNOWN: + raise NotImplementedError + return var + elif not var.funcname: + return var + else: + return None # No matching variable was found. + + +def variable_from_id(id, filenames, *, + perfilecache=None, + _get_var=variable, + ): + """Return the first found Variable that matches.""" + local = False + if isinstance(id, str): + name = id + else: + if id.funcname == UNKNOWN: + local = True + elif id.funcname: + raise NotImplementedError + + name = id.name + if id.filename and id.filename != UNKNOWN: + filenames = [id.filename] + return _get_var(id, filenames, + local=local, + perfilecache=perfilecache, + ) From b61b704bd2877731e032faa7dd9fb78b9103a022 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 12:01:33 -0600 Subject: [PATCH 08/27] Add c_symbols.find module. --- Tools/c-analyzer/c_globals/find.py | 4 +- Tools/c-analyzer/c_symbols/find.py | 140 ++++++++++++++++++++++++----- 2 files changed, 122 insertions(+), 22 deletions(-) diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py index fb55a6e8335c14..168cc24be445b8 100644 --- a/Tools/c-analyzer/c_globals/find.py +++ b/Tools/c-analyzer/c_globals/find.py @@ -15,7 +15,7 @@ def globals_from_binary(binfile=PYTHON, *, knownvars=None, dirnames=None, - _iter_symbols=b_symbols.iter_symbols, + _iter_symbols=b_symbols.symbols, _resolve=resolve.symbols_to_variables, _get_symbol_resolver=resolve.get_resolver, ): @@ -50,7 +50,7 @@ def iter_variables(kind='platform', *, dirnames=None, _resolve_symbols=resolve.symbols_to_variables, _get_symbol_resolver=resolve.get_resolver, - _symbols_from_binary=b_symbols.iter_symbols, + _symbols_from_binary=b_symbols.symbols, _symbols_from_source=s_symbols.iter_symbols, _iter_raw=declarations.iter_all, _iter_preprocessed=declarations.iter_preprocessed, diff --git a/Tools/c-analyzer/c_symbols/find.py b/Tools/c-analyzer/c_symbols/find.py index 8d7d3772f31d88..d854862b085618 100644 --- a/Tools/c-analyzer/c_symbols/find.py +++ b/Tools/c-analyzer/c_symbols/find.py @@ -2,12 +2,108 @@ import os.path import shutil -from c_analyzer_common import PYTHON +from c_analyzer_common import PYTHON, known, files +from c_analyzer_common.info import UNKNOWN, Variable +from c_parser import find as p_find -from . import _nm, source +from . import _nm +from .info import Symbol # XXX need tests: -# * iter_symbols +# * get_resolver +# * symbol +# * symbols +# * variables + + +def _resolve_known(symbol, knownvars): + for varid in knownvars: + if symbol.match(varid): + break + else: + return None + return knownvars.pop(varid) + + +def get_resolver(known=None, dirnames=(), *, + filenames=None, + perfilecache=None, + _iter_files=files.iter_files_by_suffix, +# _look_up_known=known.look_up_variable, + _from_source=p_find.variable_from_id, + ): + """Return a "resolver" func for the given known vars/types and filenames. + + The returned func takes a single Symbol and returns a corresponding + Variable. If the symbol was located then the variable will be + valid, populated with the corresponding information. Otherwise None + is returned. + """ + knownvars = (known or {}).get('variables') + if filenames: + filenames = list(filenames) + def check_filename(filename): + return filename in filenames + elif dirnames: + dirnames = [d if d.endswith(os.path.sep) else d + os.path.sep + for d in dirnames] + filenames = list(_iter_files(dirnames, ('.c', '.h'))) + def check_filename(filename): + for dirname in dirnames: + if filename.startswith(dirname): + return True + else: + return False + + if knownvars: + knownvars = dict(knownvars) # a copy + if filenames: + def resolve(symbol): + # XXX Check "found" instead? + if not check_filename(symbol.filename): + return None + found = _resolve_known(symbol, knownvars) + if found is None: + #return None + found = _from_source(symbol, filenames, + perfilecache=perfilecache, + ) + return found + else: + def resolve(symbol): + return _resolve_known(symbol, knownvars) + elif filenames: + def resolve(symbol): + return _from_source(symbol, filenames, + perfilecache=perfilecache, + ) + else: + def resolve(symbol): + return None + return resolve + + +def symbol(symbol, filenames, known=None, *, + perfilecache=None, + _get_resolver=get_resolver, + ): + """Return the Variable matching the given symbol. + + "symbol" can be one of several objects: + + * Symbol - use the contained info + * name (str) - look for a global variable with that name + * (filename, name) - look for named global in file + * (filename, funcname, name) - look for named local in file + + A name is always required. If the filename is None, "", or + "UNKNOWN" then all files will be searched. If the funcname is + "" or "UNKNOWN" then only local variables will be searched for. + """ + resolve = _get_resolver(known, filenames, + perfilecache=perfilecache, + ) + return resolve(symbol) def _get_platform_tool(): @@ -15,28 +111,32 @@ def _get_platform_tool(): # XXX Support this. raise NotImplementedError elif nm := shutil.which('nm'): - return lambda b, fls: _nm.iter_symbols(b, fls, nm=nm) + return lambda b: _nm.iter_symbols(b, nm=nm) else: raise NotImplementedError -def iter_symbols(binfile=PYTHON, dirnames=None, *, - # Alternately, use look_up_known_symbol() - # from c_globals.supported. - find_local_symbol=source.find_symbol, - _file_exists=os.path.exists, - _get_platform_tool=_get_platform_tool, - ): - """Yield a Symbol for each symbol found in the binary.""" +def symbols(binfile=PYTHON, *, + _file_exists=os.path.exists, + _get_platform_tool=_get_platform_tool, + ): + """Yield a Symbol for each one found in the binary.""" if not _file_exists(binfile): raise Exception('executable missing (need to build it first?)') - if find_local_symbol: - cache = {} - def find_local_symbol(name, *, _find=find_local_symbol): - return _find(name, dirnames, _perfilecache=cache) - else: - find_local_symbol = None - _iter_symbols = _get_platform_tool() - yield from _iter_symbols(binfile, find_local_symbol) + yield from _iter_symbols(binfile) + + +def variables(binfile=PYTHON, *, + resolve=(lambda s: symbol(s, ())), + _iter_symbols=symbols, + ): + """Yield (Variable, Symbol) for each found symbol.""" + for symbol in _iter_symbols(binfile): + if symbol.kind != Symbol.KIND.VARIABLE: + continue + var = resolve(symbol) + #if var is None: + # var = Variable(symbol.id, UNKNOWN) + yield var, symbol From a041e4a3500a335c37a51c823f3a881b09da9438 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 12:03:21 -0600 Subject: [PATCH 09/27] Avoid variable resolution in _nm.iter_symbols(). --- Tools/c-analyzer/c_symbols/_nm.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/Tools/c-analyzer/c_symbols/_nm.py b/Tools/c-analyzer/c_symbols/_nm.py index c933f536193851..20dff5a4ebe070 100644 --- a/Tools/c-analyzer/c_symbols/_nm.py +++ b/Tools/c-analyzer/c_symbols/_nm.py @@ -41,8 +41,7 @@ def _is_special_symbol(name): return False -def iter_symbols(binfile, find_local_symbol=None, - *, +def iter_symbols(binfile, *, nm=None, _which=shutil.which, _run=util.run_cmd, @@ -65,15 +64,12 @@ def iter_symbols(binfile, find_local_symbol=None, raise NotImplementedError raise for line in output.splitlines(): - (name, kind, external, filename, funcname, vartype, - ) = _parse_nm_line(line, - _find_local_symbol=find_local_symbol, - ) + (name, kind, external, filename, funcname, + ) = _parse_nm_line(line) if kind != Symbol.KIND.VARIABLE: continue elif _is_special_symbol(name): continue - assert vartype is None yield Symbol( id=(filename, funcname, name), kind=kind, @@ -81,7 +77,7 @@ def iter_symbols(binfile, find_local_symbol=None, ) -def _parse_nm_line(line, *, _find_local_symbol=None): +def _parse_nm_line(line): _origline = line _, _, line = line.partition(' ') # strip off the address line = line.strip() @@ -98,18 +94,9 @@ def _parse_nm_line(line, *, _find_local_symbol=None): else: filename = info.UNKNOWN - vartype = None name, islocal = _parse_nm_name(name, kind) - if islocal: - funcname = info.UNKNOWN - if _find_local_symbol is not None: - filename, funcname, vartype = _find_local_symbol(name) - filename = filename or info.UNKNOWN - funcname = funcname or info.UNKNOWN - else: - funcname = None - # XXX fine filename and vartype? - return name, kind, external, filename, funcname, vartype + funcname = info.UNKNOWN if islocal else None + return name, kind, external, filename, funcname def _parse_nm_name(name, kind): From b9b049985bb0dd670334f2a75e31ec6102b9c864 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 12:07:58 -0600 Subject: [PATCH 10/27] Add c_analyzer_common.find module. --- .../test_c_analyzer_common/test_find.py | 115 ++++++++++++++++++ Tools/c-analyzer/c_analyzer_common/find.py | 62 ++++++++++ 2 files changed, 177 insertions(+) create mode 100644 Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py create mode 100644 Tools/c-analyzer/c_analyzer_common/find.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py new file mode 100644 index 00000000000000..da57e58603e585 --- /dev/null +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py @@ -0,0 +1,115 @@ +import unittest + +from .. import tool_imports_for_tests +with tool_imports_for_tests(): + from c_analyzer_common import info + from c_analyzer_common.find import vars_from_binary + + +class _Base(unittest.TestCase): + + maxDiff = None + + @property + def calls(self): + try: + return self._calls + except AttributeError: + self._calls = [] + return self._calls + + +class VarsFromBinaryTests(_Base): + + _return_iter_vars = () + _return_get_symbol_resolver = None + + def setUp(self): + super().setUp() + + self.kwargs = dict( + _iter_vars=self._iter_vars, + _get_symbol_resolver=self._get_symbol_resolver, + ) + + def _iter_vars(self, binfile, resolve): + self.calls.append(('_iter_vars', (binfile, resolve))) + return [(v, v) for v in self._return_iter_vars] + + def _get_symbol_resolver(self, known, dirnames=None, perfilecache=None): + self.calls.append(('_get_symbol_resolver', (known, dirnames, perfilecache))) + return self._return_get_symbol_resolver + + def test_typical(self): + resolver = self._return_get_symbol_resolver = object() + variables = self._return_iter_vars = [ + info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'), + info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'), + info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'), + info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', 'const char *'), + info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'), + info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), + ] + known = object() + dirnames = object() + + found = list(vars_from_binary('python', + known=known, + dirnames=dirnames, + **self.kwargs)) + + self.assertEqual(found, [ + info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'), + info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'), + info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'), + info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', 'const char *'), + info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'), + info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), + ]) + self.assertEqual(self.calls, [ + ('_get_symbol_resolver', (known, dirnames, {})), + ('_iter_vars', ('python', resolver)), + ]) + +# self._return_iter_symbols = [ +# s_info.Symbol(('dir1/spam.c', None, 'var1'), 'variable', False), +# s_info.Symbol(('dir1/spam.c', None, 'var2'), 'variable', False), +# s_info.Symbol(('dir1/spam.c', None, 'func1'), 'function', False), +# s_info.Symbol(('dir1/spam.c', None, 'func2'), 'function', True), +# s_info.Symbol(('dir1/spam.c', None, 'var3'), 'variable', False), +# s_info.Symbol(('dir1/spam.c', 'func2', 'var4'), 'variable', False), +# s_info.Symbol(('dir1/ham.c', None, 'var1'), 'variable', True), +# s_info.Symbol(('dir1/eggs.c', None, 'var1'), 'variable', False), +# s_info.Symbol(('dir1/eggs.c', None, 'xyz'), 'other', False), +# s_info.Symbol(('dir1/eggs.c', '???', 'var2'), 'variable', False), +# s_info.Symbol(('???', None, 'var_x'), 'variable', False), +# s_info.Symbol(('???', '???', 'var_y'), 'variable', False), +# s_info.Symbol((None, None, '???'), 'other', False), +# ] +# known = object() +# +# vars_from_binary('python', knownvars=known, **this.kwargs) +# found = list(globals_from_symbols(['dir1'], self.iter_symbols)) +# +# self.assertEqual(found, [ +# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), +# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), +# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), +# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), +# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), +# ]) +# self.assertEqual(self.calls, [ +# ('iter_symbols', (['dir1'],)), +# ]) +# +# def test_no_symbols(self): +# self._return_iter_symbols = [] +# +# found = list(globals_from_symbols(['dir1'], self.iter_symbols)) +# +# self.assertEqual(found, []) +# self.assertEqual(self.calls, [ +# ('iter_symbols', (['dir1'],)), +# ]) + + # XXX need functional test diff --git a/Tools/c-analyzer/c_analyzer_common/find.py b/Tools/c-analyzer/c_analyzer_common/find.py new file mode 100644 index 00000000000000..46611b1214538b --- /dev/null +++ b/Tools/c-analyzer/c_analyzer_common/find.py @@ -0,0 +1,62 @@ +from c_analyzer_common import SOURCE_DIRS, PYTHON +from c_analyzer_common.info import UNKNOWN, Variable +from c_symbols import ( + info as s_info, + find as s_find, + ) + + +# XXX needs tests: +# * vars_from_source +# * vars_from_preprocessed + + +def _remove_cached(cache, var): + if not cache: + return + try: + cached = cache[var.filename] + cached.remove(var) + except (KeyError, IndexError): + pass + + +def vars_from_binary(binfile=PYTHON, *, + known=None, + dirnames=SOURCE_DIRS, + _iter_vars=s_find.variables, + _get_symbol_resolver=s_find.get_resolver, + ): + """Yield a Variable for each found Symbol. + + Details are filled in from the given "known" variables and types. + """ + cache = {} + resolve = _get_symbol_resolver(known, dirnames, + perfilecache=cache, + ) + for var, symbol in _iter_vars(binfile, resolve=resolve): + if var is None: + var = Variable(symbol.id, UNKNOWN, UNKNOWN) + yield var + _remove_cached(cache, var) + + +def vars_from_source(filenames, *, + known=None, + ): + """Yield a Variable for each declaration in the raw source code. + + Details are filled in from the given "known" variables and types. + """ + raise NotImplementedError + + +def vars_from_preprocessed(filenames, *, + known=None, + ): + """Yield a Variable for each found declaration. + + Details are filled in from the given "known" variables and types. + """ + raise NotImplementedError From 15b60dbabaa6825e013617b3efea0d84e4df0895 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 12:08:39 -0600 Subject: [PATCH 11/27] Use the new modules in __main__. --- Tools/c-analyzer/c_globals/__main__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index b7b3153ef545ca..e9b6fbdb0ea03a 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -4,12 +4,12 @@ import sys from c_analyzer_common import SOURCE_DIRS, REPO_ROOT, show +from c_analyzer_common.find import vars_from_binary from c_analyzer_common.info import UNKNOWN from c_analyzer_common.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) -from . import find from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object @@ -67,14 +67,17 @@ def _match_unused_global(variable): def _find_globals(dirnames, known, ignored): if dirnames == SOURCE_DIRS: dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames] + # For now we only use known variables (no source lookup). + dirnames = None knownvars = (known or {}).get('variables') - for variable in find.globals_from_binary(knownvars=knownvars, - dirnames=dirnames): - if variable.vartype == UNKNOWN: - yield variable, None + for var in vars_from_binary(known=known, dirnames=dirnames): + if not var.isglobal: + continue + elif var.vartype == UNKNOWN: + yield var, None else: - yield variable, is_supported(variable, ignored, known) + yield var, is_supported(var, ignored, known) def cmd_check(cmd, dirs=SOURCE_DIRS, *, From 2f9958fea8a91d116af0d3fd4dae9b8409b9078e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 12:12:24 -0600 Subject: [PATCH 12/27] Drop the old code. --- .../test_c_globals/test_find.py | 335 ------------------ Tools/c-analyzer/c_globals/find.py | 95 ----- Tools/c-analyzer/c_symbols/resolve.py | 147 -------- Tools/c-analyzer/c_symbols/source.py | 58 --- 4 files changed, 635 deletions(-) delete mode 100644 Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py delete mode 100644 Tools/c-analyzer/c_globals/find.py delete mode 100644 Tools/c-analyzer/c_symbols/resolve.py delete mode 100644 Tools/c-analyzer/c_symbols/source.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py deleted file mode 100644 index b7a1caa648817c..00000000000000 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py +++ /dev/null @@ -1,335 +0,0 @@ -import unittest - -from .. import tool_imports_for_tests -with tool_imports_for_tests(): - from c_analyzer_common import info - from c_globals.find import globals_from_binary, globals - - -class _Base(unittest.TestCase): - - maxDiff = None - - @property - def calls(self): - try: - return self._calls - except AttributeError: - self._calls = [] - return self._calls - - -class StaticsFromBinaryTests(_Base): - - _return_iter_symbols = () - _return_resolve_symbols = () - _return_get_symbol_resolver = None - - def setUp(self): - super().setUp() - - self.kwargs = dict( - _iter_symbols=self._iter_symbols, - _resolve=self._resolve_symbols, - _get_symbol_resolver=self._get_symbol_resolver, - ) - - def _iter_symbols(self, binfile, find_local_symbol): - self.calls.append(('_iter_symbols', (binfile, find_local_symbol))) - return self._return_iter_symbols - - def _resolve_symbols(self, symbols, resolve): - self.calls.append(('_resolve_symbols', (symbols, resolve,))) - return self._return_resolve_symbols - - def _get_symbol_resolver(self, knownvars, dirnames=None): - self.calls.append(('_get_symbol_resolver', (knownvars, dirnames))) - return self._return_get_symbol_resolver - - def test_typical(self): - symbols = self._return_iter_symbols = () - resolver = self._return_get_symbol_resolver = object() - variables = self._return_resolve_symbols = [ - info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'), - info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'), - info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'), - info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', 'const char *'), - info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'), - info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), - ] - knownvars = object() - - found = list(globals_from_binary('python', - knownvars=knownvars, - **self.kwargs)) - - self.assertEqual(found, [ - info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'), - info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'), - info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'), - info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'), - info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), - ]) - self.assertEqual(self.calls, [ - ('_iter_symbols', ('python', None)), - ('_get_symbol_resolver', (knownvars, None)), - ('_resolve_symbols', (symbols, resolver)), - ]) - -# self._return_iter_symbols = [ -# s_info.Symbol(('dir1/spam.c', None, 'var1'), 'variable', False), -# s_info.Symbol(('dir1/spam.c', None, 'var2'), 'variable', False), -# s_info.Symbol(('dir1/spam.c', None, 'func1'), 'function', False), -# s_info.Symbol(('dir1/spam.c', None, 'func2'), 'function', True), -# s_info.Symbol(('dir1/spam.c', None, 'var3'), 'variable', False), -# s_info.Symbol(('dir1/spam.c', 'func2', 'var4'), 'variable', False), -# s_info.Symbol(('dir1/ham.c', None, 'var1'), 'variable', True), -# s_info.Symbol(('dir1/eggs.c', None, 'var1'), 'variable', False), -# s_info.Symbol(('dir1/eggs.c', None, 'xyz'), 'other', False), -# s_info.Symbol(('dir1/eggs.c', '???', 'var2'), 'variable', False), -# s_info.Symbol(('???', None, 'var_x'), 'variable', False), -# s_info.Symbol(('???', '???', 'var_y'), 'variable', False), -# s_info.Symbol((None, None, '???'), 'other', False), -# ] -# known = object() -# -# globals_from_binary('python', knownvars=known, **this.kwargs) -# found = list(globals_from_symbols(['dir1'], self.iter_symbols)) -# -# self.assertEqual(found, [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ]) -# self.assertEqual(self.calls, [ -# ('iter_symbols', (['dir1'],)), -# ]) -# -# def test_no_symbols(self): -# self._return_iter_symbols = [] -# -# found = list(globals_from_symbols(['dir1'], self.iter_symbols)) -# -# self.assertEqual(found, []) -# self.assertEqual(self.calls, [ -# ('iter_symbols', (['dir1'],)), -# ]) - - # XXX need functional test - - -#class StaticFromDeclarationsTests(_Base): -# -# _return_iter_declarations = () -# -# def iter_declarations(self, dirnames): -# self.calls.append(('iter_declarations', (dirnames,))) -# return iter(self._return_iter_declarations) -# -# def test_typical(self): -# self._return_iter_declarations = [ -# None, -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# object(), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# object(), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# object(), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# object(), -# ] -# -# found = list(globals_from_declarations(['dir1'], self.iter_declarations)) -# -# self.assertEqual(found, [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ]) -# self.assertEqual(self.calls, [ -# ('iter_declarations', (['dir1'],)), -# ]) -# -# def test_no_declarations(self): -# self._return_iter_declarations = [] -# -# found = list(globals_from_declarations(['dir1'], self.iter_declarations)) -# -# self.assertEqual(found, []) -# self.assertEqual(self.calls, [ -# ('iter_declarations', (['dir1'],)), -# ]) - - -#class IterVariablesTests(_Base): -# -# _return_from_symbols = () -# _return_from_declarations = () -# -# def _from_symbols(self, dirnames, iter_symbols): -# self.calls.append(('_from_symbols', (dirnames, iter_symbols))) -# return iter(self._return_from_symbols) -# -# def _from_declarations(self, dirnames, iter_declarations): -# self.calls.append(('_from_declarations', (dirnames, iter_declarations))) -# return iter(self._return_from_declarations) -# -# def test_typical(self): -# expected = [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ] -# self._return_from_symbols = expected -# -# found = list(iter_variables(['dir1'], -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, expected) -# self.assertEqual(self.calls, [ -# ('_from_symbols', (['dir1'], b_symbols.iter_symbols)), -# ]) -# -# def test_no_symbols(self): -# self._return_from_symbols = [] -# -# found = list(iter_variables(['dir1'], -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, []) -# self.assertEqual(self.calls, [ -# ('_from_symbols', (['dir1'], b_symbols.iter_symbols)), -# ]) -# -# def test_from_binary(self): -# expected = [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ] -# self._return_from_symbols = expected -# -# found = list(iter_variables(['dir1'], 'platform', -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, expected) -# self.assertEqual(self.calls, [ -# ('_from_symbols', (['dir1'], b_symbols.iter_symbols)), -# ]) -# -# def test_from_symbols(self): -# expected = [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ] -# self._return_from_symbols = expected -# -# found = list(iter_variables(['dir1'], 'symbols', -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, expected) -# self.assertEqual(self.calls, [ -# ('_from_symbols', (['dir1'], s_symbols.iter_symbols)), -# ]) -# -# def test_from_declarations(self): -# expected = [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ] -# self._return_from_declarations = expected -# -# found = list(iter_variables(['dir1'], 'declarations', -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, expected) -# self.assertEqual(self.calls, [ -# ('_from_declarations', (['dir1'], declarations.iter_all)), -# ]) -# -# def test_from_preprocessed(self): -# expected = [ -# info.Variable.from_parts('dir1/spam.c', None, 'var1', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var2', '???'), -# info.Variable.from_parts('dir1/spam.c', None, 'var3', '???'), -# info.Variable.from_parts('dir1/spam.c', 'func2', 'var4', '???'), -# info.Variable.from_parts('dir1/eggs.c', None, 'var1', '???'), -# ] -# self._return_from_declarations = expected -# -# found = list(iter_variables(['dir1'], 'preprocessed', -# _from_symbols=self._from_symbols, -# _from_declarations=self._from_declarations)) -# -# self.assertEqual(found, expected) -# self.assertEqual(self.calls, [ -# ('_from_declarations', (['dir1'], declarations.iter_preprocessed)), -# ]) - - -class StaticsTest(_Base): - - _return_iter_variables = None - - def _iter_variables(self, kind, *, known, dirnames): - self.calls.append( - ('_iter_variables', (kind, known, dirnames))) - return iter(self._return_iter_variables or ()) - - def test_typical(self): - self._return_iter_variables = [ - info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'), - info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'), - info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'), - info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'), # skipped - info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'), - info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'), - info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'), - info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'), - info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'), - info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'), - info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'), - ] - dirnames = object() - known = object() - - found = list(globals(dirnames, known, - kind='platform', - _iter_variables=self._iter_variables, - )) - - self.assertEqual(found, [ - info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'), - info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'), - info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'), - info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'), - info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'), - info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'), - info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'), - info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'), - info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'), - info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'), - ]) - self.assertEqual(self.calls, [ - ('_iter_variables', ('platform', known, dirnames)), - ]) diff --git a/Tools/c-analyzer/c_globals/find.py b/Tools/c-analyzer/c_globals/find.py deleted file mode 100644 index 168cc24be445b8..00000000000000 --- a/Tools/c-analyzer/c_globals/find.py +++ /dev/null @@ -1,95 +0,0 @@ -from c_analyzer_common import SOURCE_DIRS, PYTHON -from c_analyzer_common.info import UNKNOWN, Variable -from c_symbols import ( - info as s_info, - find as b_symbols, - source as s_symbols, - resolve, - ) -from c_parser import declarations - - -# XXX needs tests: -# * iter_variables - -def globals_from_binary(binfile=PYTHON, *, - knownvars=None, - dirnames=None, - _iter_symbols=b_symbols.symbols, - _resolve=resolve.symbols_to_variables, - _get_symbol_resolver=resolve.get_resolver, - ): - """Yield a Variable for each found Symbol. - - Details are filled in from the given "known" variables and types. - """ - symbols = _iter_symbols(binfile, find_local_symbol=None) - #symbols = list(symbols) - for variable in _resolve(symbols, - resolve=_get_symbol_resolver(knownvars, dirnames), - ): - # Skip each non-global variable (unless we couldn't find it). - # XXX Drop the "UNKNOWN" condition? - if not variable.isglobal and variable.vartype != UNKNOWN: - continue - yield variable - - -def globals_from_declarations(dirnames=SOURCE_DIRS, *, - known=None, - ): - """Yield a Variable for each found declaration. - - Details are filled in from the given "known" variables and types. - """ - raise NotImplementedError - - -def iter_variables(kind='platform', *, - known=None, - dirnames=None, - _resolve_symbols=resolve.symbols_to_variables, - _get_symbol_resolver=resolve.get_resolver, - _symbols_from_binary=b_symbols.symbols, - _symbols_from_source=s_symbols.iter_symbols, - _iter_raw=declarations.iter_all, - _iter_preprocessed=declarations.iter_preprocessed, - ): - """Yield a Variable for each one found (e.g. in files).""" - kind = kind or 'platform' - - if kind == 'symbols': - knownvars = (known or {}).get('variables') - yield from _resolve_symbols( - _symbols_from_source(dirnames, known), - resolve=_get_symbol_resolver(knownvars, dirnames), - ) - elif kind == 'platform': - knownvars = (known or {}).get('variables') - yield from _resolve_symbols( - _symbols_from_binary(find_local_symbol=None), - resolve=_get_symbol_resolver(knownvars, dirnames), - ) - elif kind == 'declarations': - for decl in _iter_raw(dirnames): - if not isinstance(decl, Variable): - continue - yield decl - elif kind == 'preprocessed': - for decl in _iter_preprocessed(dirnames): - if not isinstance(decl, Variable): - continue - yield decl - else: - raise ValueError(f'unsupported kind {kind!r}') - - -def globals(dirnames, known, *, - kind=None, # Use the default. - _iter_variables=iter_variables, - ): - """Return a list of (StaticVar, ) for each found global var.""" - for found in _iter_variables(kind, known=known, dirnames=dirnames): - if not found.isglobal: - continue - yield found diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py deleted file mode 100644 index c031eea03d0143..00000000000000 --- a/Tools/c-analyzer/c_symbols/resolve.py +++ /dev/null @@ -1,147 +0,0 @@ -import os.path - -from c_analyzer_common import files -from c_analyzer_common.info import UNKNOWN, Variable -from c_parser import declarations -from .info import Symbol -from .source import _find_symbol - - -# XXX need tests: -# * look_up_known_symbol() -# * symbol_from_source() -# * get_resolver() -# * symbols_to_variables() - -def look_up_known_symbol(symbol, knownvars, *, - match_files=(lambda f1, f2: f1 == f2), - ): - """Return the known variable matching the given symbol. - - "knownvars" is a mapping of common.ID to parser.Variable. - - "match_files" is used to verify if two filenames point to - the same file. - """ - if not knownvars: - return None - - if symbol.funcname == UNKNOWN: - if not symbol.filename or symbol.filename == UNKNOWN: - for varid in knownvars: - if not varid.funcname: - continue - if varid.name == symbol.name: - return knownvars[varid] - else: - return None - else: - for varid in knownvars: - if not varid.funcname: - continue - if not match_files(varid.filename, symbol.filename): - continue - if varid.name == symbol.name: - return knownvars[varid] - else: - return None - elif not symbol.filename or symbol.filename == UNKNOWN: - raise NotImplementedError - else: - return knownvars.get(symbol.id) - - -def find_in_source(symbol, dirnames, *, - _perfilecache={}, - _find_symbol=_find_symbol, - _iter_files=files.iter_files_by_suffix, - ): - """Return the Variable matching the given Symbol. - - If there is no match then return None. - """ - if symbol.filename and symbol.filename != UNKNOWN: - filenames = [symbol.filename] - else: - filenames = _iter_files(dirnames, ('.c', '.h')) - - if symbol.funcname and symbol.funcname != UNKNOWN: - raise NotImplementedError - - (filename, funcname, decl - ) = _find_symbol(symbol.name, filenames, _perfilecache) - if filename == UNKNOWN: - return None - return Variable.from_parts(filename, funcname, symbol.name, decl) - - -def get_resolver(knownvars=None, dirnames=None, *, - _look_up_known=look_up_known_symbol, - _from_source=find_in_source, - ): - """Return a "resolver" func for the given known vars and dirnames. - - The func takes a single Symbol and returns a corresponding Variable. - If the symbol was located then the variable will be valid, populated - with the corresponding information. Otherwise None is returned. - """ - if knownvars: - knownvars = dict(knownvars) # a copy - def resolve_known(symbol): - found = _look_up_known(symbol, knownvars) - if found is None: - return None - elif symbol.funcname == UNKNOWN: - knownvars.pop(found.id) - elif not symbol.filename or symbol.filename == UNKNOWN: - knownvars.pop(found.id) - return found - if dirnames: - def resolve(symbol): - found = resolve_known(symbol) - if found is None: - return None - #return _from_source(symbol, dirnames) - else: - for dirname in dirnames: - if not dirname.endswith(os.path.sep): - dirname += os.path.sep - if found.filename.startswith(dirname): - break - else: - return None - return found - else: - resolve = resolve_known - elif dirnames: - def resolve(symbol): - return _from_source(symbol, dirnames) - else: - def resolve(symbol): - return None - return resolve - - -def symbols_to_variables(symbols, *, - resolve=(lambda s: look_up_known_symbol(s, None)), - ): - """Yield the variable the matches each given symbol. - - Use get_resolver() for a "resolve" func to use. - """ - for symbol in symbols: - if isinstance(symbol, Variable): - # XXX validate? - yield symbol - continue - if symbol.kind != Symbol.KIND.VARIABLE: - continue - resolved = resolve(symbol) - if resolved is None: - #raise NotImplementedError(symbol) - resolved = Variable( - id=symbol.id, - storage=UNKNOWN, - vartype=UNKNOWN, - ) - yield resolved diff --git a/Tools/c-analyzer/c_symbols/source.py b/Tools/c-analyzer/c_symbols/source.py deleted file mode 100644 index a7248104c94ca9..00000000000000 --- a/Tools/c-analyzer/c_symbols/source.py +++ /dev/null @@ -1,58 +0,0 @@ -from c_analyzer_common import files -from c_analyzer_common.info import UNKNOWN -from c_parser import declarations - - -# XXX need tests: -# * find_symbol() - -def find_symbol(name, dirnames, *, - _perfilecache, - _iter_files=files.iter_files_by_suffix, - **kwargs - ): - """Return (filename, funcname, vartype) for the matching Symbol.""" - filenames = _iter_files(dirnames, ('.c', '.h')) - return _find_symbol(name, filenames, _perfilecache, **kwargs) - - -def _get_symbols(filename, *, - _iter_variables=declarations.iter_variables, - ): - """Return the list of Symbols found in the given file.""" - symbols = {} - for funcname, name, vartype in _iter_variables(filename): - if not funcname: - continue - try: - instances = symbols[name] - except KeyError: - instances = symbols[name] = [] - instances.append((funcname, vartype)) - return symbols - - -def _find_symbol(name, filenames, _perfilecache, *, - _get_local_symbols=_get_symbols, - ): - for filename in filenames: - try: - symbols = _perfilecache[filename] - except KeyError: - symbols = _perfilecache[filename] = _get_local_symbols(filename) - - try: - instances = symbols[name] - except KeyError: - continue - - funcname, vartype = instances.pop(0) - if not instances: - symbols.pop(name) - return filename, funcname, vartype - else: - return UNKNOWN, UNKNOWN, UNKNOWN - - -def iter_symbols(): - raise NotImplementedError From b05272c0406c47d4a4ba36c1822ddeacf8ef603b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 13:19:40 -0600 Subject: [PATCH 13/27] Specify "preprocessed" option as an arg. --- Tools/c-analyzer/c_analyzer_common/find.py | 39 ++++++++++++---------- Tools/c-analyzer/c_parser/declarations.py | 3 ++ Tools/c-analyzer/c_parser/find.py | 24 ++++++++++--- Tools/c-analyzer/c_symbols/find.py | 6 +++- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer_common/find.py b/Tools/c-analyzer/c_analyzer_common/find.py index 46611b1214538b..212f0047fb556e 100644 --- a/Tools/c-analyzer/c_analyzer_common/find.py +++ b/Tools/c-analyzer/c_analyzer_common/find.py @@ -1,14 +1,15 @@ -from c_analyzer_common import SOURCE_DIRS, PYTHON +from c_analyzer_common import SOURCE_DIRS, PYTHON, files from c_analyzer_common.info import UNKNOWN, Variable from c_symbols import ( info as s_info, find as s_find, ) +from c_parser import ( + find as p_find, + ) - -# XXX needs tests: +# XXX need tests: # * vars_from_source -# * vars_from_preprocessed def _remove_cached(cache, var): @@ -42,21 +43,25 @@ def vars_from_binary(binfile=PYTHON, *, _remove_cached(cache, var) -def vars_from_source(filenames, *, - known=None, +def vars_from_source(filenames=None, *, + iter_vars=p_find.variables, + preprocessed=None, + known=None, # for types + dirnames=SOURCE_DIRS, + _iter_files=files.iter_files_by_suffix, ): """Yield a Variable for each declaration in the raw source code. Details are filled in from the given "known" variables and types. """ - raise NotImplementedError - - -def vars_from_preprocessed(filenames, *, - known=None, - ): - """Yield a Variable for each found declaration. - - Details are filled in from the given "known" variables and types. - """ - raise NotImplementedError + if filenames is None: + if not dirnames: + return + filenames = _iter_files(dirnames, ('.c', '.h')) + cache = {} + for var in _iter_vars(filenames, + perfilecache=cache, + preprocessed=preprocessed, + ): + yield var + _remove_cached(cache, var) diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_parser/declarations.py index 19fa3ff4e66bb1..963c8413d05a45 100644 --- a/Tools/c-analyzer/c_parser/declarations.py +++ b/Tools/c-analyzer/c_parser/declarations.py @@ -228,6 +228,7 @@ def parse_compound(stmt, blocks): def iter_variables(filename, *, + preprocessed=False, _iter_source_lines=source.iter_lines, _iter_global=iter_global_declarations, _iter_local=iter_local_statements, @@ -236,6 +237,8 @@ def iter_variables(filename, *, _parse_compound=parse_compound, ): """Yield (funcname, name, vartype) for every variable in the given file.""" + if preprocessed: + raise NotImplementedError lines = _iter_source_lines(filename) for stmt, body in _iter_global(lines): # At the file top-level we only have to worry about vars & funcs. diff --git a/Tools/c-analyzer/c_parser/find.py b/Tools/c-analyzer/c_parser/find.py index 524843f76f4502..8c2a3f1902fb07 100644 --- a/Tools/c-analyzer/c_parser/find.py +++ b/Tools/c-analyzer/c_parser/find.py @@ -8,21 +8,29 @@ # * variable_from_id -def _iter_vars(filenames, +def _iter_vars(filenames, preprocessed, *, _iter_variables=declarations.iter_variables, ): for filename in filenames or (): - for funcname, name, decl in _iter_variables(filename): + for funcname, name, decl in _iter_variables(filename, + preprocessed=preprocessed, + ): yield Variable.from_parts(filename, funcname, name, decl) def variables(*filenames, perfilecache=None, + preprocessed=False, _iter_vars=_iter_vars, ): - """Yield a Variable for each one found in the given files.""" + """Yield a Variable for each one found in the given files. + + If "preprocessed" is provided (and not False/None) then it is used + to decide which tool to use to parse the source code after it runs + through the C preprocessor. Otherwise the raw + """ if perfilecache is None: - yield from _iter_vars(filenames) + yield from _iter_vars(filenames, preprocessed) else: # XXX Cache per-file variables (e.g. `{filename: [Variable]}`). raise NotImplementedError @@ -31,6 +39,7 @@ def variables(*filenames, def variable(name, filenames, *, local=False, perfilecache=None, + preprocessed=False, _iter_variables=variables, ): """Return the first found Variable that matches. @@ -39,7 +48,10 @@ def variable(name, filenames, *, file will always be returned. To avoid that, pass perfilecache and pop each variable from the cache after using it. """ - for var in _iter_variables(filenames, perfilecache=perfilecache): + for var in _iter_variables(filenames, + perfilecache=perfilecache, + preprocessed=preprocessed, + ): if var.name != name: continue if local: @@ -55,6 +67,7 @@ def variable(name, filenames, *, def variable_from_id(id, filenames, *, perfilecache=None, + preprocessed=False, _get_var=variable, ): """Return the first found Variable that matches.""" @@ -73,4 +86,5 @@ def variable_from_id(id, filenames, *, return _get_var(id, filenames, local=local, perfilecache=perfilecache, + preprocessed=preprocessed, ) diff --git a/Tools/c-analyzer/c_symbols/find.py b/Tools/c-analyzer/c_symbols/find.py index d854862b085618..6666d04585bce8 100644 --- a/Tools/c-analyzer/c_symbols/find.py +++ b/Tools/c-analyzer/c_symbols/find.py @@ -28,8 +28,8 @@ def _resolve_known(symbol, knownvars): def get_resolver(known=None, dirnames=(), *, filenames=None, perfilecache=None, + preprocessed=False, _iter_files=files.iter_files_by_suffix, -# _look_up_known=known.look_up_variable, _from_source=p_find.variable_from_id, ): """Return a "resolver" func for the given known vars/types and filenames. @@ -67,6 +67,7 @@ def resolve(symbol): #return None found = _from_source(symbol, filenames, perfilecache=perfilecache, + preprocessed=preprocessed, ) return found else: @@ -76,6 +77,7 @@ def resolve(symbol): def resolve(symbol): return _from_source(symbol, filenames, perfilecache=perfilecache, + preprocessed=preprocessed, ) else: def resolve(symbol): @@ -85,6 +87,7 @@ def resolve(symbol): def symbol(symbol, filenames, known=None, *, perfilecache=None, + preprocessed=False, _get_resolver=get_resolver, ): """Return the Variable matching the given symbol. @@ -102,6 +105,7 @@ def symbol(symbol, filenames, known=None, *, """ resolve = _get_resolver(known, filenames, perfilecache=perfilecache, + preprocessed=preprocessed, ) return resolve(symbol) From 80d299c1bece376a6182d873c3b3c82d594caeb0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 13:20:58 -0600 Subject: [PATCH 14/27] Add a TODO. --- Tools/c-analyzer/c_globals/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index e9b6fbdb0ea03a..7c82ea3410706d 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -67,7 +67,7 @@ def _match_unused_global(variable): def _find_globals(dirnames, known, ignored): if dirnames == SOURCE_DIRS: dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames] - # For now we only use known variables (no source lookup). + # XXX For now we only use known variables (no source lookup). dirnames = None knownvars = (known or {}).get('variables') From 15c122603558d0a2f717f4e69e96bcc39763b69a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Sep 2019 13:34:06 -0600 Subject: [PATCH 15/27] Use declarations.iter_all() instead of iter_variables(). --- Tools/c-analyzer/c_parser/declarations.py | 17 +++++++---------- Tools/c-analyzer/c_parser/find.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_parser/declarations.py index 963c8413d05a45..d116c40ed55e31 100644 --- a/Tools/c-analyzer/c_parser/declarations.py +++ b/Tools/c-analyzer/c_parser/declarations.py @@ -281,18 +281,15 @@ def _iter_locals(lines, *, compound.extend(bodies) -def iter_all(dirnames): +def iter_all(filename, *, + preprocessed=False, + ): """Yield a Declaration for each one found. If there are duplicates, due to preprocessor conditionals, then they are checked to make sure they are the same. """ - raise NotImplementedError - - -def iter_preprocessed(dirnames): - """Yield a Declaration for each one found. - - All source files are run through the preprocessor first. - """ - raise NotImplementedError + # XXX For the moment we cheat. + for funcname, name, decl in iter_variables(filename, + proprocessed=preprocessed): + yield 'variable', funcname, name, decl diff --git a/Tools/c-analyzer/c_parser/find.py b/Tools/c-analyzer/c_parser/find.py index 8c2a3f1902fb07..a99b33fda6244a 100644 --- a/Tools/c-analyzer/c_parser/find.py +++ b/Tools/c-analyzer/c_parser/find.py @@ -9,12 +9,14 @@ def _iter_vars(filenames, preprocessed, *, - _iter_variables=declarations.iter_variables, + _iter_decls=declarations.iter_all, ): for filename in filenames or (): - for funcname, name, decl in _iter_variables(filename, - preprocessed=preprocessed, - ): + for kind, funcname, name, decl in _iter_decls(filename, + preprocessed=preprocessed, + ): + if kind != 'variable': + continue yield Variable.from_parts(filename, funcname, name, decl) @@ -29,6 +31,9 @@ def variables(*filenames, to decide which tool to use to parse the source code after it runs through the C preprocessor. Otherwise the raw """ + if len(filenames) == 1 and not (filenames[0], str): + filenames, = filenames + if perfilecache is None: yield from _iter_vars(filenames, preprocessed) else: From e08bad54e14d656d2462a6d936724ccd5824eae8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 1 Oct 2019 09:32:09 -0600 Subject: [PATCH 16/27] Make sure "resolve()" is used properly. --- Tools/c-analyzer/c_symbols/find.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/c_symbols/find.py b/Tools/c-analyzer/c_symbols/find.py index 6666d04585bce8..f937f7a4001a5f 100644 --- a/Tools/c-analyzer/c_symbols/find.py +++ b/Tools/c-analyzer/c_symbols/find.py @@ -133,14 +133,14 @@ def symbols(binfile=PYTHON, *, def variables(binfile=PYTHON, *, - resolve=(lambda s: symbol(s, ())), + resolve, _iter_symbols=symbols, ): """Yield (Variable, Symbol) for each found symbol.""" for symbol in _iter_symbols(binfile): if symbol.kind != Symbol.KIND.VARIABLE: continue - var = resolve(symbol) + var = resolve(symbol) or None #if var is None: # var = Variable(symbol.id, UNKNOWN) yield var, symbol From fa6f1b50deffc81f6482b914a8dccaee4f2d8118 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2019 11:41:58 -0600 Subject: [PATCH 17/27] Move extract_storage() to c_parser.declarations. --- .../test_c_parser/test_declarations.py | 4 +-- Tools/c-analyzer/c_analyzer_common/info.py | 24 ++------------- Tools/c-analyzer/c_parser/declarations.py | 29 +++++++++++++++++-- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py index b68744ef0aba49..d3d8d4eae25a1b 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py @@ -5,7 +5,7 @@ with tool_imports_for_tests(): from c_parser.declarations import ( iter_global_declarations, iter_local_statements, - parse_func, parse_var, parse_compound, + parse_func, _parse_var, parse_compound, iter_variables, ) @@ -515,7 +515,7 @@ def test_typical(self): ]) for stmt, expected in tests: with self.subTest(stmt): - name, vartype = parse_var(stmt) + name, vartype = _parse_var(stmt) self.assertEqual((name, vartype), expected) diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer_common/info.py index 9248b39c90ec50..941a0c82e5b35f 100644 --- a/Tools/c-analyzer/c_analyzer_common/info.py +++ b/Tools/c-analyzer/c_analyzer_common/info.py @@ -152,26 +152,7 @@ def normalize_vartype(vartype): return str(vartype) -def extract_storage(decl, *, isfunc=False): - """Return (storage, vartype) based on the given declaration. - - The default storage is "implicit" or "local". - """ - if decl == UNKNOWN: - return decl, decl - if decl.startswith('static '): - return 'static', decl - #return 'static', decl.partition(' ')[2].strip() - elif decl.startswith('extern '): - return 'extern', decl - #return 'extern', decl.partition(' ')[2].strip() - elif re.match('.*\b(static|extern)\b', decl): - raise NotImplementedError - elif isfunc: - return 'local', decl - else: - return 'implicit', decl - +# XXX Variable.vartype -> decl (Declaration). class Variable(_NTBase, namedtuple('Variable', 'id storage vartype')): @@ -189,7 +170,8 @@ class Variable(_NTBase, @classonly def from_parts(cls, filename, funcname, name, decl, storage=None): if storage is None: - storage, decl = extract_storage(decl, isfunc=funcname) + from c_parser.declarations import extract_storage + storage = extract_storage(decl, infunc=funcname) id = ID(filename, funcname, name) self = cls(id, storage, decl) return self diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_parser/declarations.py index d116c40ed55e31..f81f4eee34a77f 100644 --- a/Tools/c-analyzer/c_parser/declarations.py +++ b/Tools/c-analyzer/c_parser/declarations.py @@ -2,6 +2,8 @@ import shlex import subprocess +from c_analyzer_common.info import UNKNOWN + from . import source @@ -194,7 +196,7 @@ def parse_func(stmt, body): return name, signature -def parse_var(stmt): +def _parse_var(stmt): """Return (name, vartype) for the given variable declaration.""" stmt = stmt.rstrip(';') m = LOCAL_STMT_START_RE.match(stmt) @@ -220,6 +222,27 @@ def parse_var(stmt): return name, vartype +def extract_storage(decl, *, infunc=None): + """Return (storage, vartype) based on the given declaration. + + The default storage is "implicit" (or "local" if infunc is True). + """ + if decl == UNKNOWN: + return decl + if decl.startswith('static '): + return 'static' + #return 'static', decl.partition(' ')[2].strip() + elif decl.startswith('extern '): + return 'extern' + #return 'extern', decl.partition(' ')[2].strip() + elif re.match('.*\b(static|extern)\b', decl): + raise NotImplementedError + elif infunc: + return 'local' + else: + return 'implicit' + + def parse_compound(stmt, blocks): """Return (headers, bodies) for the given compound statement.""" # XXX Identify declarations inside compound statements @@ -233,7 +256,7 @@ def iter_variables(filename, *, _iter_global=iter_global_declarations, _iter_local=iter_local_statements, _parse_func=parse_func, - _parse_var=parse_var, + _parse_var=_parse_var, _parse_compound=parse_compound, ): """Yield (funcname, name, vartype) for every variable in the given file.""" @@ -259,7 +282,7 @@ def iter_variables(filename, *, def _iter_locals(lines, *, _iter_statements=iter_local_statements, - _parse_var=parse_var, + _parse_var=_parse_var, _parse_compound=parse_compound, ): compound = [lines] From 653101411c8cdcc294b038e606d6bcdd4f1e4cba Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2019 13:52:03 -0600 Subject: [PATCH 18/27] Move the generic code under a top-level c_analyzer package. --- .../test_c_globals/test___main__.py | 4 ++-- .../test_c_globals/test_supported.py | 2 +- .../__init__.py | 0 .../test_files.py | 2 +- .../test_find.py | 4 ++-- .../test_info.py | 2 +- .../test_known.py | 4 ++-- .../test_show.py | 4 ++-- .../__init__.py | 0 .../test_declarations.py | 2 +- .../test_preprocessor.py | 2 +- .../__init__.py | 0 .../test_info.py | 4 ++-- .../{c_parser => c_analyzer}/__init__.py | 0 .../common}/__init__.py | 12 ++++++---- .../common}/_generate.py | 0 .../common}/files.py | 0 .../common}/find.py | 12 +++++----- .../common}/info.py | 2 +- .../common}/known.py | 0 .../common}/show.py | 0 .../common}/util.py | 0 .../parser}/__init__.py | 0 .../parser}/declarations.py | 23 ++++++++++++++++++- .../{c_parser => c_analyzer/parser}/find.py | 2 +- .../{c_parser => c_analyzer/parser}/naive.py | 2 +- .../parser}/preprocessor.py | 2 +- .../{c_parser => c_analyzer/parser}/source.py | 0 .../c-analyzer/c_analyzer/symbols/__init__.py | 0 .../{c_symbols => c_analyzer/symbols}/_nm.py | 2 +- .../{c_symbols => c_analyzer/symbols}/find.py | 6 ++--- .../{c_symbols => c_analyzer/symbols}/info.py | 4 ++-- Tools/c-analyzer/c_globals/__main__.py | 9 ++++---- Tools/c-analyzer/c_globals/supported.py | 14 +++++++---- 34 files changed, 75 insertions(+), 45 deletions(-) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/__init__.py (100%) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/test_files.py (99%) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/test_find.py (98%) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/test_info.py (99%) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/test_known.py (95%) rename Lib/test/test_tools/test_c_analyzer/{test_c_analyzer_common => test_common}/test_show.py (96%) rename Lib/test/test_tools/test_c_analyzer/{test_c_parser => test_parser}/__init__.py (100%) rename Lib/test/test_tools/test_c_analyzer/{test_c_parser => test_parser}/test_declarations.py (99%) rename Lib/test/test_tools/test_c_analyzer/{test_c_parser => test_parser}/test_preprocessor.py (99%) rename Lib/test/test_tools/test_c_analyzer/{test_c_symbols => test_symbols}/__init__.py (100%) rename Lib/test/test_tools/test_c_analyzer/{test_c_symbols => test_symbols}/test_info.py (98%) rename Tools/c-analyzer/{c_parser => c_analyzer}/__init__.py (100%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/__init__.py (52%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/_generate.py (100%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/files.py (100%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/find.py (92%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/info.py (99%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/known.py (100%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/show.py (100%) rename Tools/c-analyzer/{c_analyzer_common => c_analyzer/common}/util.py (100%) rename Tools/c-analyzer/{c_symbols => c_analyzer/parser}/__init__.py (100%) rename Tools/c-analyzer/{c_parser => c_analyzer/parser}/declarations.py (94%) rename Tools/c-analyzer/{c_parser => c_analyzer/parser}/find.py (98%) rename Tools/c-analyzer/{c_parser => c_analyzer/parser}/naive.py (98%) rename Tools/c-analyzer/{c_parser => c_analyzer/parser}/preprocessor.py (99%) rename Tools/c-analyzer/{c_parser => c_analyzer/parser}/source.py (100%) create mode 100644 Tools/c-analyzer/c_analyzer/symbols/__init__.py rename Tools/c-analyzer/{c_symbols => c_analyzer/symbols}/_nm.py (98%) rename Tools/c-analyzer/{c_symbols => c_analyzer/symbols}/find.py (97%) rename Tools/c-analyzer/{c_symbols => c_analyzer/symbols}/info.py (93%) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py index b604e1c00fd082..8069314939e24d 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py @@ -3,8 +3,8 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common import SOURCE_DIRS, info - from c_analyzer_common.known import DATA_FILE as KNOWN_FILE + from c_analyzer.common import SOURCE_DIRS, info + from c_analyzer.common.known import DATA_FILE as KNOWN_FILE import c_globals as cg from c_globals.supported import IGNORED_FILE from c_globals.__main__ import cmd_check, cmd_show, parse_args, main diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py index 696b5612cccb43..68d06b189fe73a 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py @@ -4,7 +4,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID, Variable + from c_analyzer.common.info import ID, Variable from c_globals.supported import is_supported, ignored_from_file diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_common/__init__.py similarity index 100% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py rename to Lib/test/test_tools/test_c_analyzer/test_common/__init__.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_files.py similarity index 99% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py rename to Lib/test/test_tools/test_c_analyzer/test_common/test_files.py index 6d14aea78a486e..0c97d2a0bbf9ad 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_files.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_files.py @@ -3,7 +3,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.files import ( + from c_analyzer.common.files import ( iter_files, _walk_tree, glob_tree, ) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_find.py similarity index 98% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py rename to Lib/test/test_tools/test_c_analyzer/test_common/test_find.py index da57e58603e585..c4fb85fd1b4f84 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_find.py @@ -2,8 +2,8 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common import info - from c_analyzer_common.find import vars_from_binary + from c_analyzer.common import info + from c_analyzer.common.find import vars_from_binary class _Base(unittest.TestCase): diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py similarity index 99% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py rename to Lib/test/test_tools/test_c_analyzer/test_common/test_info.py index 671cc511410fe6..b2dba58deb04b6 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py @@ -4,7 +4,7 @@ from ..util import PseudoStr, StrProxy, Object from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ( + from c_analyzer.common.info import ( UNKNOWN, ID, normalize_vartype, Variable diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py similarity index 95% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py rename to Lib/test/test_tools/test_c_analyzer/test_common/test_known.py index 99753ff6d6023e..bbd6cc5a9bed9c 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py @@ -4,8 +4,8 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID, Variable - from c_analyzer_common.known import from_file + from c_analyzer.common.info import Variable + from c_analyzer.common.known import from_file class FromFileTests(unittest.TestCase): diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py similarity index 96% rename from Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py rename to Lib/test/test_tools/test_c_analyzer/test_common/test_show.py index b19656961fc3a5..5d0b986ff7e2a8 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_show.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py @@ -2,8 +2,8 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common import info - from c_analyzer_common.show import basic + from c_analyzer.common import info + from c_analyzer.common.show import basic TYPICAL = [ diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_parser/__init__.py similarity index 100% rename from Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py rename to Lib/test/test_tools/test_c_analyzer/test_parser/__init__.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py b/Lib/test/test_tools/test_c_analyzer/test_parser/test_declarations.py similarity index 99% rename from Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py rename to Lib/test/test_tools/test_c_analyzer/test_parser/test_declarations.py index d3d8d4eae25a1b..674fcb1af1c7ad 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_declarations.py +++ b/Lib/test/test_tools/test_c_analyzer/test_parser/test_declarations.py @@ -3,7 +3,7 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_parser.declarations import ( + from c_analyzer.parser.declarations import ( iter_global_declarations, iter_local_statements, parse_func, _parse_var, parse_compound, iter_variables, diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py b/Lib/test/test_tools/test_c_analyzer/test_parser/test_preprocessor.py similarity index 99% rename from Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py rename to Lib/test/test_tools/test_c_analyzer/test_parser/test_preprocessor.py index 89e15570d65309..56a1c9c612f726 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_preprocessor.py +++ b/Lib/test/test_tools/test_c_analyzer/test_parser/test_preprocessor.py @@ -6,7 +6,7 @@ from ..util import wrapped_arg_combos, StrProxy from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_parser.preprocessor import ( + from c_analyzer.parser.preprocessor import ( iter_lines, # directives parse_directive, PreprocessorDirective, diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_symbols/__init__.py similarity index 100% rename from Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py rename to Lib/test/test_tools/test_c_analyzer/test_symbols/__init__.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_symbols/test_info.py similarity index 98% rename from Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py rename to Lib/test/test_tools/test_c_analyzer/test_symbols/test_info.py index e029dcf66124dc..1282a89718c820 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py +++ b/Lib/test/test_tools/test_c_analyzer/test_symbols/test_info.py @@ -4,8 +4,8 @@ from ..util import PseudoStr, StrProxy, Object from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID - from c_symbols.info import Symbol + from c_analyzer.common.info import ID + from c_analyzer.symbols.info import Symbol class SymbolTests(unittest.TestCase): diff --git a/Tools/c-analyzer/c_parser/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py similarity index 100% rename from Tools/c-analyzer/c_parser/__init__.py rename to Tools/c-analyzer/c_analyzer/__init__.py diff --git a/Tools/c-analyzer/c_analyzer_common/__init__.py b/Tools/c-analyzer/c_analyzer/common/__init__.py similarity index 52% rename from Tools/c-analyzer/c_analyzer_common/__init__.py rename to Tools/c-analyzer/c_analyzer/common/__init__.py index 25fa5e1afc246f..431af0fe365bdc 100644 --- a/Tools/c-analyzer/c_analyzer_common/__init__.py +++ b/Tools/c-analyzer/c_analyzer/common/__init__.py @@ -2,10 +2,14 @@ import sys -PKG_ROOT = os.path.dirname(__file__) -DATA_DIR = os.path.dirname(PKG_ROOT) -REPO_ROOT = os.path.dirname( - os.path.dirname(DATA_DIR)) +TOOL_ROOT = ( + os.path.dirname( # c-analyzer/ + os.path.dirname( # c_analyzer/ + os.path.dirname(__file__)))) # common/ +DATA_DIR = TOOL_ROOT +REPO_ROOT = ( + os.path.dirname( # .. + os.path.dirname(TOOL_ROOT))) # Tools/ SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ 'Include', diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer/common/_generate.py similarity index 100% rename from Tools/c-analyzer/c_analyzer_common/_generate.py rename to Tools/c-analyzer/c_analyzer/common/_generate.py diff --git a/Tools/c-analyzer/c_analyzer_common/files.py b/Tools/c-analyzer/c_analyzer/common/files.py similarity index 100% rename from Tools/c-analyzer/c_analyzer_common/files.py rename to Tools/c-analyzer/c_analyzer/common/files.py diff --git a/Tools/c-analyzer/c_analyzer_common/find.py b/Tools/c-analyzer/c_analyzer/common/find.py similarity index 92% rename from Tools/c-analyzer/c_analyzer_common/find.py rename to Tools/c-analyzer/c_analyzer/common/find.py index 212f0047fb556e..46cb0749a16166 100644 --- a/Tools/c-analyzer/c_analyzer_common/find.py +++ b/Tools/c-analyzer/c_analyzer/common/find.py @@ -1,12 +1,12 @@ -from c_analyzer_common import SOURCE_DIRS, PYTHON, files -from c_analyzer_common.info import UNKNOWN, Variable -from c_symbols import ( +from ..common import SOURCE_DIRS, PYTHON, files +from ..common.info import UNKNOWN, Variable +from ..parser import ( + find as p_find, + ) +from ..symbols import ( info as s_info, find as s_find, ) -from c_parser import ( - find as p_find, - ) # XXX need tests: # * vars_from_source diff --git a/Tools/c-analyzer/c_analyzer_common/info.py b/Tools/c-analyzer/c_analyzer/common/info.py similarity index 99% rename from Tools/c-analyzer/c_analyzer_common/info.py rename to Tools/c-analyzer/c_analyzer/common/info.py index 941a0c82e5b35f..354a8513ebcb76 100644 --- a/Tools/c-analyzer/c_analyzer_common/info.py +++ b/Tools/c-analyzer/c_analyzer/common/info.py @@ -170,7 +170,7 @@ class Variable(_NTBase, @classonly def from_parts(cls, filename, funcname, name, decl, storage=None): if storage is None: - from c_parser.declarations import extract_storage + from ..parser.declarations import extract_storage storage = extract_storage(decl, infunc=funcname) id = ID(filename, funcname, name) self = cls(id, storage, decl) diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer/common/known.py similarity index 100% rename from Tools/c-analyzer/c_analyzer_common/known.py rename to Tools/c-analyzer/c_analyzer/common/known.py diff --git a/Tools/c-analyzer/c_analyzer_common/show.py b/Tools/c-analyzer/c_analyzer/common/show.py similarity index 100% rename from Tools/c-analyzer/c_analyzer_common/show.py rename to Tools/c-analyzer/c_analyzer/common/show.py diff --git a/Tools/c-analyzer/c_analyzer_common/util.py b/Tools/c-analyzer/c_analyzer/common/util.py similarity index 100% rename from Tools/c-analyzer/c_analyzer_common/util.py rename to Tools/c-analyzer/c_analyzer/common/util.py diff --git a/Tools/c-analyzer/c_symbols/__init__.py b/Tools/c-analyzer/c_analyzer/parser/__init__.py similarity index 100% rename from Tools/c-analyzer/c_symbols/__init__.py rename to Tools/c-analyzer/c_analyzer/parser/__init__.py diff --git a/Tools/c-analyzer/c_parser/declarations.py b/Tools/c-analyzer/c_analyzer/parser/declarations.py similarity index 94% rename from Tools/c-analyzer/c_parser/declarations.py rename to Tools/c-analyzer/c_analyzer/parser/declarations.py index f81f4eee34a77f..7216ca3b1b6cda 100644 --- a/Tools/c-analyzer/c_parser/declarations.py +++ b/Tools/c-analyzer/c_analyzer/parser/declarations.py @@ -2,7 +2,7 @@ import shlex import subprocess -from c_analyzer_common.info import UNKNOWN +from ..common.info import UNKNOWN from . import source @@ -196,6 +196,27 @@ def parse_func(stmt, body): return name, signature +#TYPE_SPEC = rf'''(?: +# )''' +#VAR_DECLARATOR = rf'''(?: +# )''' +#VAR_DECL = rf'''(?: +# {TYPE_SPEC}+ +# {VAR_DECLARATOR} +# \s* +# )''' +#VAR_DECLARATION = rf'''(?: +# {VAR_DECL} +# (?: = [^=] [^;]* )? +# ; +# )''' +# +# +#def parse_variable(decl, *, inFunc=False): +# """Return [(name, storage, vartype)] for the given variable declaration.""" +# ... + + def _parse_var(stmt): """Return (name, vartype) for the given variable declaration.""" stmt = stmt.rstrip(';') diff --git a/Tools/c-analyzer/c_parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py similarity index 98% rename from Tools/c-analyzer/c_parser/find.py rename to Tools/c-analyzer/c_analyzer/parser/find.py index a99b33fda6244a..eedb469eda7f5f 100644 --- a/Tools/c-analyzer/c_parser/find.py +++ b/Tools/c-analyzer/c_analyzer/parser/find.py @@ -1,4 +1,4 @@ -from c_analyzer_common.info import Variable, UNKNOWN +from ..common.info import Variable, UNKNOWN from . import declarations diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_analyzer/parser/naive.py similarity index 98% rename from Tools/c-analyzer/c_parser/naive.py rename to Tools/c-analyzer/c_analyzer/parser/naive.py index dc4e0278d12ed3..c5dabca2f54103 100644 --- a/Tools/c-analyzer/c_parser/naive.py +++ b/Tools/c-analyzer/c_analyzer/parser/naive.py @@ -1,6 +1,6 @@ import re -from c_analyzer_common.info import UNKNOWN, Variable +from ..common.info import UNKNOWN, Variable from .preprocessor import _iter_clean_lines diff --git a/Tools/c-analyzer/c_parser/preprocessor.py b/Tools/c-analyzer/c_analyzer/parser/preprocessor.py similarity index 99% rename from Tools/c-analyzer/c_parser/preprocessor.py rename to Tools/c-analyzer/c_analyzer/parser/preprocessor.py index 35b807a6c62978..41f306e5f8022b 100644 --- a/Tools/c-analyzer/c_parser/preprocessor.py +++ b/Tools/c-analyzer/c_analyzer/parser/preprocessor.py @@ -3,7 +3,7 @@ import os import re -from c_analyzer_common import util, info +from ..common import util, info CONTINUATION = '\\' + os.linesep diff --git a/Tools/c-analyzer/c_parser/source.py b/Tools/c-analyzer/c_analyzer/parser/source.py similarity index 100% rename from Tools/c-analyzer/c_parser/source.py rename to Tools/c-analyzer/c_analyzer/parser/source.py diff --git a/Tools/c-analyzer/c_analyzer/symbols/__init__.py b/Tools/c-analyzer/c_analyzer/symbols/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Tools/c-analyzer/c_symbols/_nm.py b/Tools/c-analyzer/c_analyzer/symbols/_nm.py similarity index 98% rename from Tools/c-analyzer/c_symbols/_nm.py rename to Tools/c-analyzer/c_analyzer/symbols/_nm.py index 20dff5a4ebe070..bee9a4b6076cb1 100644 --- a/Tools/c-analyzer/c_symbols/_nm.py +++ b/Tools/c-analyzer/c_analyzer/symbols/_nm.py @@ -1,7 +1,7 @@ import os.path import shutil -from c_analyzer_common import util, info +from c_analyzer.common import util, info from .info import Symbol diff --git a/Tools/c-analyzer/c_symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py similarity index 97% rename from Tools/c-analyzer/c_symbols/find.py rename to Tools/c-analyzer/c_analyzer/symbols/find.py index f937f7a4001a5f..966c57ca4f5caa 100644 --- a/Tools/c-analyzer/c_symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -2,9 +2,9 @@ import os.path import shutil -from c_analyzer_common import PYTHON, known, files -from c_analyzer_common.info import UNKNOWN, Variable -from c_parser import find as p_find +from ..common import PYTHON, known, files +from ..common.info import UNKNOWN, Variable +from ..parser import find as p_find from . import _nm from .info import Symbol diff --git a/Tools/c-analyzer/c_symbols/info.py b/Tools/c-analyzer/c_analyzer/symbols/info.py similarity index 93% rename from Tools/c-analyzer/c_symbols/info.py rename to Tools/c-analyzer/c_analyzer/symbols/info.py index f6ed52c8f07142..96a251abb7c7fd 100644 --- a/Tools/c-analyzer/c_symbols/info.py +++ b/Tools/c-analyzer/c_analyzer/symbols/info.py @@ -1,7 +1,7 @@ from collections import namedtuple -from c_analyzer_common.info import ID -from c_analyzer_common.util import classonly, _NTBase +from c_analyzer.common.info import ID +from c_analyzer.common.util import classonly, _NTBase class Symbol(_NTBase, namedtuple('Symbol', 'id kind external')): diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index 7c82ea3410706d..9653b466895b19 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -3,13 +3,14 @@ import re import sys -from c_analyzer_common import SOURCE_DIRS, REPO_ROOT, show -from c_analyzer_common.find import vars_from_binary -from c_analyzer_common.info import UNKNOWN -from c_analyzer_common.known import ( +from c_analyzer.common import SOURCE_DIRS, REPO_ROOT, show +from c_analyzer.common.find import vars_from_binary +from c_analyzer.common.info import UNKNOWN +from c_analyzer.common.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) + from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/c_globals/supported.py index d185daa2463bb2..5bc523a2d58970 100644 --- a/Tools/c-analyzer/c_globals/supported.py +++ b/Tools/c-analyzer/c_globals/supported.py @@ -1,9 +1,12 @@ import os.path import re -from c_analyzer_common import DATA_DIR -from c_analyzer_common.info import ID -from c_analyzer_common.util import read_tsv, write_tsv +from c_analyzer.common import DATA_DIR +from c_analyzer.common.info import ID +from c_analyzer.common.util import read_tsv, write_tsv + +# XXX need tests: +# * generate / script IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv') @@ -379,11 +382,12 @@ def _generate_ignored_file(variables, filename=None, *, if __name__ == '__main__': - from c_analyzer_common import SOURCE_DIRS - from c_analyzer_common.known import ( + from c_analyzer.common import SOURCE_DIRS + from c_analyzer.common.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) + # XXX This is wrong! from . import find known = known_from_file(KNOWN_FILE) knownvars = (known or {}).get('variables') From 15f010bfd8b0c1a7288c307be74276e0f3e86c3b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2019 14:12:57 -0600 Subject: [PATCH 19/27] Move some code to a new c_analyzer.variables package. --- .../test_c_globals/test___main__.py | 5 +- .../test_c_globals/test_supported.py | 7 +- .../test_c_analyzer/test_common/test_info.py | 235 ----------------- .../test_c_analyzer/test_common/test_show.py | 6 +- .../test_variables/__init__.py | 6 + .../test_find.py | 6 +- .../test_variables/test_info.py | 244 ++++++++++++++++++ .../test_known.py | 6 +- Tools/c-analyzer/c_analyzer/common/info.py | 86 ------ Tools/c-analyzer/c_analyzer/parser/find.py | 3 +- Tools/c-analyzer/c_analyzer/parser/naive.py | 3 +- Tools/c-analyzer/c_analyzer/symbols/find.py | 6 +- .../c_analyzer/variables/__init__.py | 0 .../c_analyzer/{common => variables}/find.py | 3 +- Tools/c-analyzer/c_analyzer/variables/info.py | 87 +++++++ .../c_analyzer/{common => variables}/known.py | 7 +- Tools/c-analyzer/c_globals/__main__.py | 4 +- 17 files changed, 373 insertions(+), 341 deletions(-) create mode 100644 Lib/test/test_tools/test_c_analyzer/test_variables/__init__.py rename Lib/test/test_tools/test_c_analyzer/{test_common => test_variables}/test_find.py (97%) create mode 100644 Lib/test/test_tools/test_c_analyzer/test_variables/test_info.py rename Lib/test/test_tools/test_c_analyzer/{test_common => test_variables}/test_known.py (94%) create mode 100644 Tools/c-analyzer/c_analyzer/variables/__init__.py rename Tools/c-analyzer/c_analyzer/{common => variables}/find.py (96%) create mode 100644 Tools/c-analyzer/c_analyzer/variables/info.py rename Tools/c-analyzer/c_analyzer/{common => variables}/known.py (96%) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py index 8069314939e24d..6e6c401f1b5794 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py @@ -3,8 +3,9 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common import SOURCE_DIRS, info - from c_analyzer.common.known import DATA_FILE as KNOWN_FILE + from c_analyzer.common import SOURCE_DIRS + from c_analyzer.variables import info + from c_analyzer.variables.known import DATA_FILE as KNOWN_FILE import c_globals as cg from c_globals.supported import IGNORED_FILE from c_globals.__main__ import cmd_check, cmd_show, parse_args, main diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py index 68d06b189fe73a..15a0cbb089d17a 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py @@ -4,8 +4,11 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common.info import ID, Variable - from c_globals.supported import is_supported, ignored_from_file + from c_analyzer.common.info import ID + from c_analyzer.variables.info import Variable + from c_globals.supported import ( + is_supported, ignored_from_file, + ) class IsSupportedTests(unittest.TestCase): diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py index b2dba58deb04b6..69dbb582c6b684 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_info.py @@ -7,7 +7,6 @@ from c_analyzer.common.info import ( UNKNOWN, ID, - normalize_vartype, Variable ) @@ -196,237 +195,3 @@ def test_validate_bad_field(self): id = id._replace(**{field: value}) id.validate() # This does not fail. - - -class NormalizeVartypeTests(unittest.TestCase): - - def test_basic(self): - tests = [ - (None, None), - ('', ''), - ('int', 'int'), - (PseudoStr('int'), 'int'), - (StrProxy('int'), 'int'), - ] - for vartype, expected in tests: - with self.subTest(vartype): - normalized = normalize_vartype(vartype) - - self.assertEqual(normalized, expected) - - -class VariableTests(unittest.TestCase): - - VALID_ARGS = ( - ('x/y/z/spam.c', 'func', 'eggs'), - 'static', - 'int', - ) - VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS)) - VALID_EXPECTED = VALID_ARGS - - def test_init_typical_global(self): - for storage in ('static', 'extern', 'implicit'): - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname=None, - name='eggs', - ), - storage=storage, - vartype='int', - ) - - self.assertEqual(static, ( - ('x/y/z/spam.c', None, 'eggs'), - storage, - 'int', - )) - - def test_init_typical_local(self): - for storage in ('static', 'local'): - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - storage=storage, - vartype='int', - ) - - self.assertEqual(static, ( - ('x/y/z/spam.c', 'func', 'eggs'), - storage, - 'int', - )) - - def test_init_all_missing(self): - for value in ('', None): - with self.subTest(repr(value)): - static = Variable( - id=value, - storage=value, - vartype=value, - ) - - self.assertEqual(static, ( - None, - None, - None, - )) - - def test_init_all_coerced(self): - id = ID('x/y/z/spam.c', 'func', 'spam') - tests = [ - ('str subclass', - dict( - id=( - PseudoStr('x/y/z/spam.c'), - PseudoStr('func'), - PseudoStr('spam'), - ), - storage=PseudoStr('static'), - vartype=PseudoStr('int'), - ), - (id, - 'static', - 'int', - )), - ('non-str 1', - dict( - id=id, - storage=Object(), - vartype=Object(), - ), - (id, - '', - '', - )), - ('non-str 2', - dict( - id=id, - storage=StrProxy('static'), - vartype=StrProxy('variable'), - ), - (id, - 'static', - 'variable', - )), - ('non-str', - dict( - id=id, - storage=('a', 'b', 'c'), - vartype=('x', 'y', 'z'), - ), - (id, - "('a', 'b', 'c')", - "('x', 'y', 'z')", - )), - ] - for summary, kwargs, expected in tests: - with self.subTest(summary): - static = Variable(**kwargs) - - for field in Variable._fields: - value = getattr(static, field) - if field == 'id': - self.assertIs(type(value), ID) - else: - self.assertIs(type(value), str) - self.assertEqual(tuple(static), expected) - - def test_iterable(self): - static = Variable(**self.VALID_KWARGS) - - id, storage, vartype = static - - values = (id, storage, vartype) - for value, expected in zip(values, self.VALID_EXPECTED): - self.assertEqual(value, expected) - - def test_fields(self): - static = Variable(('a', 'b', 'z'), 'x', 'y') - - self.assertEqual(static.id, ('a', 'b', 'z')) - self.assertEqual(static.storage, 'x') - self.assertEqual(static.vartype, 'y') - - def test___getattr__(self): - static = Variable(('a', 'b', 'z'), 'x', 'y') - - self.assertEqual(static.filename, 'a') - self.assertEqual(static.funcname, 'b') - self.assertEqual(static.name, 'z') - - def test_validate_typical(self): - validstorage = ('static', 'extern', 'implicit', 'local') - self.assertEqual(set(validstorage), set(Variable.STORAGE)) - - for storage in validstorage: - with self.subTest(storage): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - storage=storage, - vartype='int', - ) - - static.validate() # This does not fail. - - def test_validate_missing_field(self): - for field in Variable._fields: - with self.subTest(field): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: None}) - - with self.assertRaises(TypeError): - static.validate() - for field in ('storage', 'vartype'): - with self.subTest(field): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: UNKNOWN}) - - with self.assertRaises(TypeError): - static.validate() - - def test_validate_bad_field(self): - badch = tuple(c for c in string.punctuation + string.digits) - notnames = ( - '1a', - 'a.b', - 'a-b', - '&a', - 'a++', - ) + badch - tests = [ - ('id', ()), # Any non-empty str is okay. - ('storage', ('external', 'global') + notnames), - ('vartype', ()), # Any non-empty str is okay. - ] - seen = set() - for field, invalid in tests: - for value in invalid: - seen.add(value) - with self.subTest(f'{field}={value!r}'): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: value}) - - with self.assertRaises(ValueError): - static.validate() - - for field, invalid in tests: - if field == 'id': - continue - valid = seen - set(invalid) - for value in valid: - with self.subTest(f'{field}={value!r}'): - static = Variable(**self.VALID_KWARGS) - static = static._replace(**{field: value}) - - static.validate() # This does not fail. diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py index 5d0b986ff7e2a8..91ca2f3b344dd3 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_show.py @@ -2,8 +2,10 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common import info - from c_analyzer.common.show import basic + from c_analyzer.variables import info + from c_analyzer.common.show import ( + basic, + ) TYPICAL = [ diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_variables/__init__.py new file mode 100644 index 00000000000000..bc502ef32d2916 --- /dev/null +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py similarity index 97% rename from Lib/test/test_tools/test_c_analyzer/test_common/test_find.py rename to Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py index c4fb85fd1b4f84..b6c92db72f69d4 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_common/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py @@ -2,8 +2,10 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common import info - from c_analyzer.common.find import vars_from_binary + from c_analyzer.variables import info + from c_analyzer.variables.find import ( + vars_from_binary, + ) class _Base(unittest.TestCase): diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_info.py new file mode 100644 index 00000000000000..d424d8eebb8111 --- /dev/null +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_info.py @@ -0,0 +1,244 @@ +import string +import unittest + +from ..util import PseudoStr, StrProxy, Object +from .. import tool_imports_for_tests +with tool_imports_for_tests(): + from c_analyzer.common.info import UNKNOWN, ID + from c_analyzer.variables.info import ( + normalize_vartype, Variable + ) + + +class NormalizeVartypeTests(unittest.TestCase): + + def test_basic(self): + tests = [ + (None, None), + ('', ''), + ('int', 'int'), + (PseudoStr('int'), 'int'), + (StrProxy('int'), 'int'), + ] + for vartype, expected in tests: + with self.subTest(vartype): + normalized = normalize_vartype(vartype) + + self.assertEqual(normalized, expected) + + +class VariableTests(unittest.TestCase): + + VALID_ARGS = ( + ('x/y/z/spam.c', 'func', 'eggs'), + 'static', + 'int', + ) + VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS)) + VALID_EXPECTED = VALID_ARGS + + def test_init_typical_global(self): + for storage in ('static', 'extern', 'implicit'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname=None, + name='eggs', + ), + storage=storage, + vartype='int', + ) + + self.assertEqual(static, ( + ('x/y/z/spam.c', None, 'eggs'), + storage, + 'int', + )) + + def test_init_typical_local(self): + for storage in ('static', 'local'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) + + self.assertEqual(static, ( + ('x/y/z/spam.c', 'func', 'eggs'), + storage, + 'int', + )) + + def test_init_all_missing(self): + for value in ('', None): + with self.subTest(repr(value)): + static = Variable( + id=value, + storage=value, + vartype=value, + ) + + self.assertEqual(static, ( + None, + None, + None, + )) + + def test_init_all_coerced(self): + id = ID('x/y/z/spam.c', 'func', 'spam') + tests = [ + ('str subclass', + dict( + id=( + PseudoStr('x/y/z/spam.c'), + PseudoStr('func'), + PseudoStr('spam'), + ), + storage=PseudoStr('static'), + vartype=PseudoStr('int'), + ), + (id, + 'static', + 'int', + )), + ('non-str 1', + dict( + id=id, + storage=Object(), + vartype=Object(), + ), + (id, + '', + '', + )), + ('non-str 2', + dict( + id=id, + storage=StrProxy('static'), + vartype=StrProxy('variable'), + ), + (id, + 'static', + 'variable', + )), + ('non-str', + dict( + id=id, + storage=('a', 'b', 'c'), + vartype=('x', 'y', 'z'), + ), + (id, + "('a', 'b', 'c')", + "('x', 'y', 'z')", + )), + ] + for summary, kwargs, expected in tests: + with self.subTest(summary): + static = Variable(**kwargs) + + for field in Variable._fields: + value = getattr(static, field) + if field == 'id': + self.assertIs(type(value), ID) + else: + self.assertIs(type(value), str) + self.assertEqual(tuple(static), expected) + + def test_iterable(self): + static = Variable(**self.VALID_KWARGS) + + id, storage, vartype = static + + values = (id, storage, vartype) + for value, expected in zip(values, self.VALID_EXPECTED): + self.assertEqual(value, expected) + + def test_fields(self): + static = Variable(('a', 'b', 'z'), 'x', 'y') + + self.assertEqual(static.id, ('a', 'b', 'z')) + self.assertEqual(static.storage, 'x') + self.assertEqual(static.vartype, 'y') + + def test___getattr__(self): + static = Variable(('a', 'b', 'z'), 'x', 'y') + + self.assertEqual(static.filename, 'a') + self.assertEqual(static.funcname, 'b') + self.assertEqual(static.name, 'z') + + def test_validate_typical(self): + validstorage = ('static', 'extern', 'implicit', 'local') + self.assertEqual(set(validstorage), set(Variable.STORAGE)) + + for storage in validstorage: + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) + + static.validate() # This does not fail. + + def test_validate_missing_field(self): + for field in Variable._fields: + with self.subTest(field): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: None}) + + with self.assertRaises(TypeError): + static.validate() + for field in ('storage', 'vartype'): + with self.subTest(field): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: UNKNOWN}) + + with self.assertRaises(TypeError): + static.validate() + + def test_validate_bad_field(self): + badch = tuple(c for c in string.punctuation + string.digits) + notnames = ( + '1a', + 'a.b', + 'a-b', + '&a', + 'a++', + ) + badch + tests = [ + ('id', ()), # Any non-empty str is okay. + ('storage', ('external', 'global') + notnames), + ('vartype', ()), # Any non-empty str is okay. + ] + seen = set() + for field, invalid in tests: + for value in invalid: + seen.add(value) + with self.subTest(f'{field}={value!r}'): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: value}) + + with self.assertRaises(ValueError): + static.validate() + + for field, invalid in tests: + if field == 'id': + continue + valid = seen - set(invalid) + for value in valid: + with self.subTest(f'{field}={value!r}'): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: value}) + + static.validate() # This does not fail. diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py similarity index 94% rename from Lib/test/test_tools/test_c_analyzer/test_common/test_known.py rename to Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py index bbd6cc5a9bed9c..24a9501f136afe 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py @@ -4,8 +4,10 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common.info import Variable - from c_analyzer.common.known import from_file + from c_analyzer.variables.info import Variable + from c_analyzer.variables.known import ( + from_file, + ) class FromFileTests(unittest.TestCase): diff --git a/Tools/c-analyzer/c_analyzer/common/info.py b/Tools/c-analyzer/c_analyzer/common/info.py index 354a8513ebcb76..34b7a3e66fd67b 100644 --- a/Tools/c-analyzer/c_analyzer/common/info.py +++ b/Tools/c-analyzer/c_analyzer/common/info.py @@ -136,89 +136,3 @@ def match(self, other, *, return True return False - - -############################# -# variables - -def normalize_vartype(vartype): - """Return the canonical form for a variable type (or func signature).""" - # We allow empty strring through for semantic reasons. - if vartype is None: - return None - - # XXX finish! - # XXX Return (modifiers, type, pointer)? - return str(vartype) - - -# XXX Variable.vartype -> decl (Declaration). - -class Variable(_NTBase, - namedtuple('Variable', 'id storage vartype')): - """Information about a single variable declaration.""" - - __slots__ = () - - STORAGE = ( - 'static', - 'extern', - 'implicit', - 'local', - ) - - @classonly - def from_parts(cls, filename, funcname, name, decl, storage=None): - if storage is None: - from ..parser.declarations import extract_storage - storage = extract_storage(decl, infunc=funcname) - id = ID(filename, funcname, name) - self = cls(id, storage, decl) - return self - - def __new__(cls, id, storage, vartype): - self = super().__new__( - cls, - id=ID.from_raw(id), - storage=str(storage) if storage else None, - vartype=normalize_vartype(vartype) if vartype else None, - ) - return self - - def __hash__(self): - return hash(self.id) - - def __getattr__(self, name): - return getattr(self.id, name) - - def _validate_id(self): - if not self.id: - raise TypeError('missing id') - - if not self.filename or self.filename == UNKNOWN: - raise TypeError(f'id missing filename ({self.id})') - - if self.funcname and self.funcname == UNKNOWN: - raise TypeError(f'id missing funcname ({self.id})') - - self.id.validate() - - def validate(self): - """Fail if the object is invalid (i.e. init with bad data).""" - self._validate_id() - - if self.storage is None or self.storage == UNKNOWN: - raise TypeError('missing storage') - elif self.storage not in self.STORAGE: - raise ValueError(f'unsupported storage {self.storage:r}') - - if self.vartype is None or self.vartype == UNKNOWN: - raise TypeError('missing vartype') - - @property - def isglobal(self): - return self.storage != 'local' - - @property - def isconst(self): - return 'const' in self.vartype.split() diff --git a/Tools/c-analyzer/c_analyzer/parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py index eedb469eda7f5f..be56e70a19e98a 100644 --- a/Tools/c-analyzer/c_analyzer/parser/find.py +++ b/Tools/c-analyzer/c_analyzer/parser/find.py @@ -1,4 +1,5 @@ -from ..common.info import Variable, UNKNOWN +from ..common.info import UNKNOWN +from ..variables.info import Variable from . import declarations diff --git a/Tools/c-analyzer/c_analyzer/parser/naive.py b/Tools/c-analyzer/c_analyzer/parser/naive.py index c5dabca2f54103..0a2fcf00a39cae 100644 --- a/Tools/c-analyzer/c_analyzer/parser/naive.py +++ b/Tools/c-analyzer/c_analyzer/parser/naive.py @@ -1,6 +1,7 @@ import re -from ..common.info import UNKNOWN, Variable +from ..common.info import UNKNOWN +from ..variables.info import Variable from .preprocessor import _iter_clean_lines diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py index 966c57ca4f5caa..48237d4f95c876 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -2,9 +2,11 @@ import os.path import shutil -from ..common import PYTHON, known, files -from ..common.info import UNKNOWN, Variable +from ..common import PYTHON, files +from ..common.info import UNKNOWN from ..parser import find as p_find +from ..variables import known +from ..variables.info import Variable from . import _nm from .info import Symbol diff --git a/Tools/c-analyzer/c_analyzer/variables/__init__.py b/Tools/c-analyzer/c_analyzer/variables/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Tools/c-analyzer/c_analyzer/common/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py similarity index 96% rename from Tools/c-analyzer/c_analyzer/common/find.py rename to Tools/c-analyzer/c_analyzer/variables/find.py index 46cb0749a16166..44c537fb3e66d0 100644 --- a/Tools/c-analyzer/c_analyzer/common/find.py +++ b/Tools/c-analyzer/c_analyzer/variables/find.py @@ -1,5 +1,5 @@ from ..common import SOURCE_DIRS, PYTHON, files -from ..common.info import UNKNOWN, Variable +from ..common.info import UNKNOWN from ..parser import ( find as p_find, ) @@ -7,6 +7,7 @@ info as s_info, find as s_find, ) +from .info import Variable # XXX need tests: # * vars_from_source diff --git a/Tools/c-analyzer/c_analyzer/variables/info.py b/Tools/c-analyzer/c_analyzer/variables/info.py new file mode 100644 index 00000000000000..e5910bc15ba85f --- /dev/null +++ b/Tools/c-analyzer/c_analyzer/variables/info.py @@ -0,0 +1,87 @@ +from collections import namedtuple + +from ..common.info import ID, UNKNOWN +from ..common.util import classonly, _NTBase + + +def normalize_vartype(vartype): + """Return the canonical form for a variable type (or func signature).""" + # We allow empty strring through for semantic reasons. + if vartype is None: + return None + + # XXX finish! + # XXX Return (modifiers, type, pointer)? + return str(vartype) + + +# XXX Variable.vartype -> decl (Declaration). + +class Variable(_NTBase, + namedtuple('Variable', 'id storage vartype')): + """Information about a single variable declaration.""" + + __slots__ = () + + STORAGE = ( + 'static', + 'extern', + 'implicit', + 'local', + ) + + @classonly + def from_parts(cls, filename, funcname, name, decl, storage=None): + if storage is None: + from ..parser.declarations import extract_storage + storage = extract_storage(decl, infunc=funcname) + id = ID(filename, funcname, name) + self = cls(id, storage, decl) + return self + + def __new__(cls, id, storage, vartype): + self = super().__new__( + cls, + id=ID.from_raw(id), + storage=str(storage) if storage else None, + vartype=normalize_vartype(vartype) if vartype else None, + ) + return self + + def __hash__(self): + return hash(self.id) + + def __getattr__(self, name): + return getattr(self.id, name) + + def _validate_id(self): + if not self.id: + raise TypeError('missing id') + + if not self.filename or self.filename == UNKNOWN: + raise TypeError(f'id missing filename ({self.id})') + + if self.funcname and self.funcname == UNKNOWN: + raise TypeError(f'id missing funcname ({self.id})') + + self.id.validate() + + def validate(self): + """Fail if the object is invalid (i.e. init with bad data).""" + self._validate_id() + + if self.storage is None or self.storage == UNKNOWN: + raise TypeError('missing storage') + elif self.storage not in self.STORAGE: + raise ValueError(f'unsupported storage {self.storage:r}') + + if self.vartype is None or self.vartype == UNKNOWN: + raise TypeError('missing vartype') + + @property + def isglobal(self): + return self.storage != 'local' + + @property + def isconst(self): + return 'const' in self.vartype.split() diff --git a/Tools/c-analyzer/c_analyzer/common/known.py b/Tools/c-analyzer/c_analyzer/variables/known.py similarity index 96% rename from Tools/c-analyzer/c_analyzer/common/known.py rename to Tools/c-analyzer/c_analyzer/variables/known.py index 0c1f3fbc5c4d63..d265a5364f9e2a 100644 --- a/Tools/c-analyzer/c_analyzer/common/known.py +++ b/Tools/c-analyzer/c_analyzer/variables/known.py @@ -1,9 +1,10 @@ import csv import os.path -from . import DATA_DIR -from .info import ID, UNKNOWN, Variable -from .util import read_tsv +from ..common import DATA_DIR +from ..common.info import ID, UNKNOWN +from ..common.util import read_tsv +from .info import Variable # XXX need tests: # * from_file() diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/c_globals/__main__.py index 9653b466895b19..71c04b3731f7d1 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/c_globals/__main__.py @@ -4,9 +4,9 @@ import sys from c_analyzer.common import SOURCE_DIRS, REPO_ROOT, show -from c_analyzer.common.find import vars_from_binary from c_analyzer.common.info import UNKNOWN -from c_analyzer.common.known import ( +from c_analyzer.variables.find import vars_from_binary +from c_analyzer.variables.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) From c399dc036b1010fa9cd80ac5b7063b579038906f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2019 17:08:43 -0600 Subject: [PATCH 20/27] c_global -> cpython. --- .../test_c_analyzer/test_common/test_known.py | 139 ++++++++++++++++++ .../__init__.py | 0 .../test___main__.py | 83 ++++------- .../test_functional.py | 0 .../test_supported.py | 2 +- .../test_variables/test_known.py | 69 --------- Tools/c-analyzer/c-globals.py | 2 +- .../c-analyzer/c_analyzer/common/__init__.py | 28 ---- Tools/c-analyzer/c_analyzer/common/files.py | 26 +--- .../c_analyzer/{variables => common}/known.py | 85 +++++------ Tools/c-analyzer/c_analyzer/symbols/find.py | 7 +- Tools/c-analyzer/c_analyzer/variables/find.py | 8 +- Tools/c-analyzer/c_globals/__init__.py | 0 .../c-analyzer/{c_globals => cpython}/README | 0 Tools/c-analyzer/cpython/__init__.py | 27 ++++ .../{c_globals => cpython}/__main__.py | 75 ++++------ .../common => cpython}/_generate.py | 12 +- Tools/c-analyzer/cpython/files.py | 29 ++++ Tools/c-analyzer/cpython/find.py | 80 ++++++++++ Tools/c-analyzer/cpython/known.py | 66 +++++++++ .../{c_globals => cpython}/supported.py | 7 +- 21 files changed, 458 insertions(+), 287 deletions(-) create mode 100644 Lib/test/test_tools/test_c_analyzer/test_common/test_known.py rename Lib/test/test_tools/test_c_analyzer/{test_c_globals => test_cpython}/__init__.py (100%) rename Lib/test/test_tools/test_c_analyzer/{test_c_globals => test_cpython}/test___main__.py (72%) rename Lib/test/test_tools/test_c_analyzer/{test_c_globals => test_cpython}/test_functional.py (100%) rename Lib/test/test_tools/test_c_analyzer/{test_c_globals => test_cpython}/test_supported.py (98%) delete mode 100644 Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py rename Tools/c-analyzer/c_analyzer/{variables => common}/known.py (60%) delete mode 100644 Tools/c-analyzer/c_globals/__init__.py rename Tools/c-analyzer/{c_globals => cpython}/README (100%) create mode 100644 Tools/c-analyzer/cpython/__init__.py rename Tools/c-analyzer/{c_globals => cpython}/__main__.py (77%) rename Tools/c-analyzer/{c_analyzer/common => cpython}/_generate.py (97%) create mode 100644 Tools/c-analyzer/cpython/files.py create mode 100644 Tools/c-analyzer/cpython/find.py create mode 100644 Tools/c-analyzer/cpython/known.py rename Tools/c-analyzer/{c_globals => cpython}/supported.py (98%) diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py new file mode 100644 index 00000000000000..cb6761d63d7fac --- /dev/null +++ b/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py @@ -0,0 +1,139 @@ +import re +import textwrap +import unittest + +from .. import tool_imports_for_tests +with tool_imports_for_tests(): + from c_analyzer.variables.info import Variable + from c_analyzer.common.info import ID + from c_analyzer.common.known import ( + read_file, + from_file, + ) + +class _BaseTests(unittest.TestCase): + + maxDiff = None + + @property + def calls(self): + try: + return self._calls + except AttributeError: + self._calls = [] + return self._calls + + +class ReadFileTests(_BaseTests): + + _return_read_tsv = () + + def _read_tsv(self, *args): + self.calls.append(('_read_tsv', args)) + return self._return_read_tsv + + def test_typical(self): + lines = textwrap.dedent(''' + filename funcname name kind declaration + file1.c - var1 variable static int + file1.c func1 local1 variable static int + file1.c - var2 variable int + file1.c func2 local2 variable char * + file2.c - var1 variable char * + ''').strip().splitlines() + lines = [re.sub(r'\s+', '\t', line, 4) for line in lines] + self._return_read_tsv = [tuple(v.strip() for v in line.split('\t')) + for line in lines[1:]] + + known = list(read_file('known.tsv', _read_tsv=self._read_tsv)) + + self.assertEqual(known, [ + ('variable', ID('file1.c', '', 'var1'), 'static int'), + ('variable', ID('file1.c', 'func1', 'local1'), 'static int'), + ('variable', ID('file1.c', '', 'var2'), 'int'), + ('variable', ID('file1.c', 'func2', 'local2'), 'char *'), + ('variable', ID('file2.c', '', 'var1'), 'char *'), + ]) + self.assertEqual(self.calls, [ + ('_read_tsv', + ('known.tsv', 'filename\tfuncname\tname\tkind\tdeclaration')), + ]) + + def test_empty(self): + self._return_read_tsv = [] + + known = list(read_file('known.tsv', _read_tsv=self._read_tsv)) + + self.assertEqual(known, []) + self.assertEqual(self.calls, [ + ('_read_tsv', ('known.tsv', 'filename\tfuncname\tname\tkind\tdeclaration')), + ]) + + +class FromFileTests(_BaseTests): + + _return_read_file = () + _return_handle_var = () + + def _read_file(self, infile): + self.calls.append(('_read_file', (infile,))) + return iter(self._return_read_file) + + def _handle_var(self, varid, decl): + self.calls.append(('_handle_var', (varid, decl))) + var = self._return_handle_var.pop(0) + return var + + def test_typical(self): + expected = [ + Variable.from_parts('file1.c', '', 'var1', 'static int'), + Variable.from_parts('file1.c', 'func1', 'local1', 'static int'), + Variable.from_parts('file1.c', '', 'var2', 'int'), + Variable.from_parts('file1.c', 'func2', 'local2', 'char *'), + Variable.from_parts('file2.c', '', 'var1', 'char *'), + ] + self._return_read_file = [('variable', v.id, v.vartype) + for v in expected] +# ('variable', ID('file1.c', '', 'var1'), 'static int'), +# ('variable', ID('file1.c', 'func1', 'local1'), 'static int'), +# ('variable', ID('file1.c', '', 'var2'), 'int'), +# ('variable', ID('file1.c', 'func2', 'local2'), 'char *'), +# ('variable', ID('file2.c', '', 'var1'), 'char *'), +# ] + self._return_handle_var = list(expected) # a copy + + known = from_file('known.tsv', + handle_var=self._handle_var, + _read_file=self._read_file, + ) + + self.assertEqual(known, { + 'variables': {v.id: v for v in expected}, + }) +# Variable.from_parts('file1.c', '', 'var1', 'static int'), +# Variable.from_parts('file1.c', 'func1', 'local1', 'static int'), +# Variable.from_parts('file1.c', '', 'var2', 'int'), +# Variable.from_parts('file1.c', 'func2', 'local2', 'char *'), +# Variable.from_parts('file2.c', '', 'var1', 'char *'), +# ]}, +# }) + self.assertEqual(self.calls, [ + ('_read_file', ('known.tsv',)), + *[('_handle_var', (v.id, v.vartype)) + for v in expected], + ]) + + def test_empty(self): + self._return_read_file = [] + + known = from_file('known.tsv', + handle_var=self._handle_var, + _read_file=self._read_file, + ) + + self.assertEqual(known, { + 'variables': {}, + }) + self.assertEqual(self.calls, [ + ('_read_file', ('known.tsv',)), + ]) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_cpython/__init__.py similarity index 100% rename from Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py rename to Lib/test/test_tools/test_c_analyzer/test_cpython/__init__.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py similarity index 72% rename from Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py rename to Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py index 6e6c401f1b5794..e1a9691260fbf7 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py @@ -3,12 +3,13 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.common import SOURCE_DIRS from c_analyzer.variables import info - from c_analyzer.variables.known import DATA_FILE as KNOWN_FILE - import c_globals as cg - from c_globals.supported import IGNORED_FILE - from c_globals.__main__ import cmd_check, cmd_show, parse_args, main + from cpython import SOURCE_DIRS + from cpython.supported import IGNORED_FILE + from cpython.known import DATA_FILE as KNOWN_FILE + from cpython.__main__ import ( + cmd_check, cmd_show, parse_args, main, + ) TYPICAL = [ @@ -46,8 +47,8 @@ class CMDBase(unittest.TestCase): maxDiff = None - _return_known_from_file = None - _return_ignored_from_file = None +# _return_known_from_file = None +# _return_ignored_from_file = None _return_find = () @property @@ -58,16 +59,16 @@ def calls(self): self._calls = [] return self._calls - def _known_from_file(self, *args): - self.calls.append(('_known_from_file', args)) - return self._return_known_from_file or {} +# def _known_from_file(self, *args): +# self.calls.append(('_known_from_file', args)) +# return self._return_known_from_file or {} +# +# def _ignored_from_file(self, *args): +# self.calls.append(('_ignored_from_file', args)) +# return self._return_ignored_from_file or {} - def _ignored_from_file(self, *args): - self.calls.append(('_ignored_from_file', args)) - return self._return_ignored_from_file or {} - - def _find(self, *args): - self.calls.append(('_find', args)) + def _find(self, dirs, known, ignored, skip_objects=False): + self.calls.append(('_find', (dirs, known, ignored, skip_objects))) return self._return_find def _show(self, *args): @@ -80,27 +81,20 @@ def _print(self, *args): class CheckTests(CMDBase): def test_defaults(self): - known = self._return_known_from_file = object() - ignored = self._return_ignored_from_file = object() self._return_find = [] cmd_check('check', - _known_from_file=self._known_from_file, - _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, ) - self.assertEqual(self.calls[:3], [ - ('_known_from_file', (KNOWN_FILE,)), - ('_ignored_from_file', (IGNORED_FILE,)), - ('_find', (SOURCE_DIRS, known, ignored)), - ]) + self.assertEqual( + self.calls[0], + ('_find', (SOURCE_DIRS, KNOWN_FILE, IGNORED_FILE, False)), + ) def test_all_supported(self): - known = self._return_known_from_file = object() - ignored = self._return_ignored_from_file = object() self._return_find = [(v, s) for v, s in TYPICAL if s] dirs = ['src1', 'src2', 'Include'] @@ -108,23 +102,17 @@ def test_all_supported(self): dirs, known='known.tsv', ignored='ignored.tsv', - _known_from_file=self._known_from_file, - _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, ) self.assertEqual(self.calls, [ - ('_known_from_file', ('known.tsv',)), - ('_ignored_from_file', ('ignored.tsv',)), - ('_find', (dirs, known, ignored)), + ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), #('_print', ('okay',)), ]) def test_some_unsupported(self): - known = self._return_known_from_file = object() - ignored = self._return_ignored_from_file = object() self._return_find = TYPICAL dirs = ['src1', 'src2', 'Include'] @@ -133,8 +121,6 @@ def test_some_unsupported(self): dirs, known='known.tsv', ignored='ignored.tsv', - _known_from_file=self._known_from_file, - _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, @@ -142,9 +128,7 @@ def test_some_unsupported(self): unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_known_from_file', ('known.tsv',)), - ('_ignored_from_file', ('ignored.tsv',)), - ('_find', (dirs, known, ignored)), + ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), ('_print', ('ERROR: found unsupported global variables',)), ('_print', ()), ('_show', (sorted(unsupported),)), @@ -156,27 +140,20 @@ def test_some_unsupported(self): class ShowTests(CMDBase): def test_defaults(self): - known = self._return_known_from_file = object() - ignored = self._return_ignored_from_file = object() self._return_find = [] cmd_show('show', - _known_from_file=self._known_from_file, - _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, ) - self.assertEqual(self.calls[:3], [ - ('_known_from_file', (KNOWN_FILE,)), - ('_ignored_from_file', (IGNORED_FILE,)), - ('_find', (SOURCE_DIRS, known, ignored)), - ]) + self.assertEqual( + self.calls[0], + ('_find', (SOURCE_DIRS, KNOWN_FILE, IGNORED_FILE, False)), + ) def test_typical(self): - known = self._return_known_from_file = object() - ignored = self._return_ignored_from_file = object() self._return_find = TYPICAL dirs = ['src1', 'src2', 'Include'] @@ -184,8 +161,6 @@ def test_typical(self): dirs, known='known.tsv', ignored='ignored.tsv', - _known_from_file=self._known_from_file, - _ignored_from_file=self._ignored_from_file, _find=self._find, _show=self._show, _print=self._print, @@ -194,9 +169,7 @@ def test_typical(self): supported = [v for v, s in TYPICAL if s] unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_known_from_file', ('known.tsv',)), - ('_ignored_from_file', ('ignored.tsv',)), - ('_find', (dirs, known, ignored)), + ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), ('_print', ('supported:',)), ('_print', ('----------',)), ('_show', (sorted(supported),)), diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py b/Lib/test/test_tools/test_c_analyzer/test_cpython/test_functional.py similarity index 100% rename from Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py rename to Lib/test/test_tools/test_c_analyzer/test_cpython/test_functional.py diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py b/Lib/test/test_tools/test_c_analyzer/test_cpython/test_supported.py similarity index 98% rename from Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py rename to Lib/test/test_tools/test_c_analyzer/test_cpython/test_supported.py index 15a0cbb089d17a..a244b97e1fc7c7 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py +++ b/Lib/test/test_tools/test_c_analyzer/test_cpython/test_supported.py @@ -6,7 +6,7 @@ with tool_imports_for_tests(): from c_analyzer.common.info import ID from c_analyzer.variables.info import Variable - from c_globals.supported import ( + from cpython.supported import ( is_supported, ignored_from_file, ) diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py deleted file mode 100644 index 24a9501f136afe..00000000000000 --- a/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py +++ /dev/null @@ -1,69 +0,0 @@ -import re -import textwrap -import unittest - -from .. import tool_imports_for_tests -with tool_imports_for_tests(): - from c_analyzer.variables.info import Variable - from c_analyzer.variables.known import ( - from_file, - ) - - -class FromFileTests(unittest.TestCase): - - maxDiff = None - - _return_read_tsv = () - - @property - def calls(self): - try: - return self._calls - except AttributeError: - self._calls = [] - return self._calls - - def _read_tsv(self, *args): - self.calls.append(('_read_tsv', args)) - return self._return_read_tsv - - def test_typical(self): - lines = textwrap.dedent(''' - filename funcname name kind declaration - file1.c - var1 variable static int - file1.c func1 local1 variable static int - file1.c - var2 variable int - file1.c func2 local2 variable char * - file2.c - var1 variable char * - ''').strip().splitlines() - lines = [re.sub(r'\s+', '\t', line, 4) for line in lines] - self._return_read_tsv = [tuple(v.strip() for v in line.split('\t')) - for line in lines[1:]] - - known = from_file('spam.c', _read_tsv=self._read_tsv) - - self.assertEqual(known, { - 'variables': {v.id: v for v in [ - Variable.from_parts('file1.c', '', 'var1', 'static int'), - Variable.from_parts('file1.c', 'func1', 'local1', 'static int'), - Variable.from_parts('file1.c', '', 'var2', 'int'), - Variable.from_parts('file1.c', 'func2', 'local2', 'char *'), - Variable.from_parts('file2.c', '', 'var1', 'char *'), - ]}, - }) - self.assertEqual(self.calls, [ - ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')), - ]) - - def test_empty(self): - self._return_read_tsv = [] - - known = from_file('spam.c', _read_tsv=self._read_tsv) - - self.assertEqual(known, { - 'variables': {}, - }) - self.assertEqual(self.calls, [ - ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')), - ]) diff --git a/Tools/c-analyzer/c-globals.py b/Tools/c-analyzer/c-globals.py index 9afe059b28c664..b36b791241d539 100644 --- a/Tools/c-analyzer/c-globals.py +++ b/Tools/c-analyzer/c-globals.py @@ -1,6 +1,6 @@ # This is a script equivalent of running "python -m test.test_c_globals.cg". -from c_globals.__main__ import parse_args, main +from cpython.__main__ import parse_args, main # This is effectively copied from cg/__main__.py: diff --git a/Tools/c-analyzer/c_analyzer/common/__init__.py b/Tools/c-analyzer/c_analyzer/common/__init__.py index 431af0fe365bdc..e69de29bb2d1d6 100644 --- a/Tools/c-analyzer/c_analyzer/common/__init__.py +++ b/Tools/c-analyzer/c_analyzer/common/__init__.py @@ -1,28 +0,0 @@ -import os.path -import sys - - -TOOL_ROOT = ( - os.path.dirname( # c-analyzer/ - os.path.dirname( # c_analyzer/ - os.path.dirname(__file__)))) # common/ -DATA_DIR = TOOL_ROOT -REPO_ROOT = ( - os.path.dirname( # .. - os.path.dirname(TOOL_ROOT))) # Tools/ - -SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ - 'Include', - 'Python', - 'Parser', - 'Objects', - 'Modules', - ]] - -#PYTHON = os.path.join(REPO_ROOT, 'python') -PYTHON = sys.executable - - -# Clean up the namespace. -del sys -del os diff --git a/Tools/c-analyzer/c_analyzer/common/files.py b/Tools/c-analyzer/c_analyzer/common/files.py index b3cd16c8dc0080..ab551a84bad15d 100644 --- a/Tools/c-analyzer/c_analyzer/common/files.py +++ b/Tools/c-analyzer/c_analyzer/common/files.py @@ -2,7 +2,10 @@ import os import os.path -from . import SOURCE_DIRS, REPO_ROOT +# XXX need tests: +# * walk_tree() +# * glob_tree() +# * iter_files_by_suffix() C_SOURCE_SUFFIXES = ('.c', '.h') @@ -115,24 +118,3 @@ def iter_files_by_suffix(root, suffixes, relparent=None, *, # XXX Ignore repeated suffixes? for suffix in suffixes: yield from _iter_files(root, suffix, relparent) - - -def iter_cpython_files(*, - walk=walk_tree, - _files=iter_files_by_suffix, - ): - """Yield each file in the tree for each of the given directory names.""" - excludedtrees = [ - os.path.join('Include', 'cpython', ''), - ] - def is_excluded(filename): - for root in excludedtrees: - if filename.startswith(root): - return True - return False - for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT, - walk=walk, - ): - if is_excluded(filename): - continue - yield filename diff --git a/Tools/c-analyzer/c_analyzer/variables/known.py b/Tools/c-analyzer/c_analyzer/common/known.py similarity index 60% rename from Tools/c-analyzer/c_analyzer/variables/known.py rename to Tools/c-analyzer/c_analyzer/common/known.py index d265a5364f9e2a..8fdeba675a5757 100644 --- a/Tools/c-analyzer/c_analyzer/variables/known.py +++ b/Tools/c-analyzer/c_analyzer/common/known.py @@ -1,83 +1,70 @@ import csv -import os.path -from ..common import DATA_DIR -from ..common.info import ID, UNKNOWN -from ..common.util import read_tsv -from .info import Variable +from c_analyzer.common.info import ID, UNKNOWN +from c_analyzer.common.util import read_tsv + # XXX need tests: -# * from_file() +# * read_file() # * look_up_variable() -DATA_FILE = os.path.join(DATA_DIR, 'known.tsv') - COLUMNS = ('filename', 'funcname', 'name', 'kind', 'declaration') HEADER = '\t'.join(COLUMNS) -def from_file(infile, *, +def read_file(infile, *, _read_tsv=read_tsv, ): - """Return the info for known declarations in the given file.""" + """Yield (kind, id, decl) for each row in the data file. + + The caller is responsible for validating each row. + """ + for row in _read_tsv(infile, HEADER): + filename, funcname, name, kind, declaration = row + if not funcname or funcname == '-': + funcname = None + id = ID(filename, funcname, name) + yield kind, id, declaration + + +def _handle_var(varid, decl): + from ..parser.declarations import extract_storage + from ..variables.info import Variable + storage = extract_storage(decl, infunc=varid.funcname) + return Variable(id, storage, decl) + + +def from_file(infile, *, + handle_var=_handle_var, + _read_file=read_file, + ): + """Return the info for known declarations in the given file. + + "handle_var" is a function that takes (varid, decl) and returns + a corresponding Variable. + """ known = { 'variables': {}, #'types': {}, #'constants': {}, #'macros': {}, } - for row in _read_tsv(infile, HEADER): - filename, funcname, name, kind, declaration = row - if not funcname or funcname == '-': - funcname = None - id = ID(filename, funcname, name) + for kind, id, decl in _read_file(infile): if kind == 'variable': values = known['variables'] - if funcname: - storage = _get_storage(declaration) or 'local' - else: - storage = _get_storage(declaration) or 'implicit' - value = Variable(id, storage, declaration) + value = handle_var(id, decl) else: raise ValueError(f'unsupported kind in row {row}') value.validate() -# if value.name == 'id' and declaration == UNKNOWN: -# # None of these are variables. -# declaration = 'int id'; -# else: -# value.validate() values[id] = value return known -def _get_storage(decl): - # statics - if decl.startswith('static '): - return 'static' - if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')): - return 'static' - if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')): - return 'static' - if decl.startswith('PyDoc_VAR('): - return 'static' - if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')): - return 'static' - if decl.startswith('WRAP_METHOD('): - return 'static' - # public extern - if decl.startswith('extern '): - return 'extern' - if decl.startswith('PyAPI_DATA('): - return 'extern' - # implicit or local - return None - - def look_up_variable(varid, knownvars, *, match_files=(lambda f1, f2: f1 == f2), ): - """Return the known variable matching the given ID. + """Return (the known variable matching the given ID. "knownvars" is a mapping of ID to Variable. diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py index 48237d4f95c876..4f98074c51520f 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -2,10 +2,9 @@ import os.path import shutil -from ..common import PYTHON, files +from ..common import files from ..common.info import UNKNOWN from ..parser import find as p_find -from ..variables import known from ..variables.info import Variable from . import _nm @@ -122,7 +121,7 @@ def _get_platform_tool(): raise NotImplementedError -def symbols(binfile=PYTHON, *, +def symbols(binfile, *, _file_exists=os.path.exists, _get_platform_tool=_get_platform_tool, ): @@ -134,7 +133,7 @@ def symbols(binfile=PYTHON, *, yield from _iter_symbols(binfile) -def variables(binfile=PYTHON, *, +def variables(binfile, *, resolve, _iter_symbols=symbols, ): diff --git a/Tools/c-analyzer/c_analyzer/variables/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py index 44c537fb3e66d0..05ea2fbc8d7f0c 100644 --- a/Tools/c-analyzer/c_analyzer/variables/find.py +++ b/Tools/c-analyzer/c_analyzer/variables/find.py @@ -1,4 +1,4 @@ -from ..common import SOURCE_DIRS, PYTHON, files +from ..common import files from ..common.info import UNKNOWN from ..parser import ( find as p_find, @@ -23,9 +23,9 @@ def _remove_cached(cache, var): pass -def vars_from_binary(binfile=PYTHON, *, +def vars_from_binary(binfile, *, known=None, - dirnames=SOURCE_DIRS, + dirnames=None, _iter_vars=s_find.variables, _get_symbol_resolver=s_find.get_resolver, ): @@ -48,7 +48,7 @@ def vars_from_source(filenames=None, *, iter_vars=p_find.variables, preprocessed=None, known=None, # for types - dirnames=SOURCE_DIRS, + dirnames=None, _iter_files=files.iter_files_by_suffix, ): """Yield a Variable for each declaration in the raw source code. diff --git a/Tools/c-analyzer/c_globals/__init__.py b/Tools/c-analyzer/c_globals/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Tools/c-analyzer/c_globals/README b/Tools/c-analyzer/cpython/README similarity index 100% rename from Tools/c-analyzer/c_globals/README rename to Tools/c-analyzer/cpython/README diff --git a/Tools/c-analyzer/cpython/__init__.py b/Tools/c-analyzer/cpython/__init__.py new file mode 100644 index 00000000000000..d2aafb57dc9c1d --- /dev/null +++ b/Tools/c-analyzer/cpython/__init__.py @@ -0,0 +1,27 @@ +import os.path +import sys + + +TOOL_ROOT = ( + os.path.dirname( # c-analyzer/ + os.path.dirname(__file__))) # cpython/ +DATA_DIR = TOOL_ROOT +REPO_ROOT = ( + os.path.dirname( # .. + os.path.dirname(TOOL_ROOT))) # Tools/ + +SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ + 'Include', + 'Python', + 'Parser', + 'Objects', + 'Modules', + ]] + +#PYTHON = os.path.join(REPO_ROOT, 'python') +PYTHON = sys.executable + + +# Clean up the namespace. +del sys +del os diff --git a/Tools/c-analyzer/c_globals/__main__.py b/Tools/c-analyzer/cpython/__main__.py similarity index 77% rename from Tools/c-analyzer/c_globals/__main__.py rename to Tools/c-analyzer/cpython/__main__.py index 71c04b3731f7d1..517217197821bb 100644 --- a/Tools/c-analyzer/c_globals/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -1,17 +1,17 @@ import argparse -import os.path import re import sys -from c_analyzer.common import SOURCE_DIRS, REPO_ROOT, show +from c_analyzer.common import show from c_analyzer.common.info import UNKNOWN -from c_analyzer.variables.find import vars_from_binary -from c_analyzer.variables.known import ( + +from . import SOURCE_DIRS +from .find import supported_vars +from .known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) - -from .supported import is_supported, ignored_from_file, IGNORED_FILE, _is_object +from .supported import IGNORED_FILE def _check_results(unknown, knownvars, used): @@ -65,29 +65,29 @@ def _match_unused_global(variable): raise Exception('could not find all symbols') -def _find_globals(dirnames, known, ignored): - if dirnames == SOURCE_DIRS: - dirnames = [os.path.relpath(d, REPO_ROOT) for d in dirnames] - # XXX For now we only use known variables (no source lookup). - dirnames = None +# XXX Move this check to its own command. +def cmd_check_cache(cmd, dirs=SOURCE_DIRS, *, + known=KNOWN_FILE, + ignored=IGNORED_FILE, + _known_from_file=known_from_file, + _find=supported_vars, + ): + known = _known_from_file(known) - knownvars = (known or {}).get('variables') - for var in vars_from_binary(known=known, dirnames=dirnames): - if not var.isglobal: + used = set() + unknown = set() + for var, supported in _find(dirs, known=known, ignored=ignored): + if supported is None: + unknown.add(var) continue - elif var.vartype == UNKNOWN: - yield var, None - else: - yield var, is_supported(var, ignored, known) + used.add(var.id) + _check_results(unknown, known['variables'], used) def cmd_check(cmd, dirs=SOURCE_DIRS, *, known=KNOWN_FILE, ignored=IGNORED_FILE, - checkfound=False, - _known_from_file=known_from_file, - _ignored_from_file=ignored_from_file, - _find=_find_globals, + _find=supported_vars, _show=show.basic, _print=print, ): @@ -97,24 +97,11 @@ def cmd_check(cmd, dirs=SOURCE_DIRS, *, In the failure case, the list of unsupported variables will be printed out. """ - known = _known_from_file(known) - ignored = _ignored_from_file(ignored) - - used = set() - unknown = set() unsupported = [] - for var, supported in _find(dirs, known, ignored): - if supported is None: - unknown.add(var) - continue - used.add(var.id) + for var, supported in _find(dirs, known=known, ignored=ignored): if not supported: unsupported.append(var) - if checkfound: - # XXX Move this check to its own command. - _check_results(unknown, known['variables'], used) - if not unsupported: #_print('okay') return @@ -130,9 +117,7 @@ def cmd_show(cmd, dirs=SOURCE_DIRS, *, known=KNOWN_FILE, ignored=IGNORED_FILE, skip_objects=False, - _known_from_file=known_from_file, - _ignored_from_file=ignored_from_file, - _find=_find_globals, + _find=supported_vars, _show=show.basic, _print=print, ): @@ -141,17 +126,15 @@ def cmd_show(cmd, dirs=SOURCE_DIRS, *, The variables will be distinguished as "supported" or "unsupported". """ - known = _known_from_file(known) - ignored = _ignored_from_file(ignored) - allsupported = [] allunsupported = [] - for found, supported in _find(dirs, known, ignored): + for found, supported in _find(dirs, + known=known, + ignored=ignored, + skip_objects=skip_objects, + ): if supported is None: continue - if skip_objects: # XXX Support proper filters instead. - if _is_object(found.vartype): - continue (allsupported if supported else allunsupported ).append(found) diff --git a/Tools/c-analyzer/c_analyzer/common/_generate.py b/Tools/c-analyzer/cpython/_generate.py similarity index 97% rename from Tools/c-analyzer/c_analyzer/common/_generate.py rename to Tools/c-analyzer/cpython/_generate.py index 8843af0b37c82e..4c340acf99e1c3 100644 --- a/Tools/c-analyzer/c_analyzer/common/_generate.py +++ b/Tools/c-analyzer/cpython/_generate.py @@ -1,14 +1,16 @@ # The code here consists of hacks for pre-populating the known.tsv file. -from c_parser.preprocessor import _iter_clean_lines -from c_parser.naive import ( +from c_analyzer.parser.preprocessor import _iter_clean_lines +from c_analyzer.parser.naive import ( iter_variables, parse_variable_declaration, find_variables, ) +from c_analyzer.common.known import HEADER as KNOWN_HEADER +from c_analyzer.common.info import UNKNOWN, ID +from c_analyzer.variables import Variable +from c_analyzer.util import write_tsv from . import SOURCE_DIRS, REPO_ROOT -from .known import DATA_FILE as KNOWN_FILE, HEADER as KNOWN_HEADER -from .info import UNKNOWN, ID, Variable -from .util import write_tsv +from .known import DATA_FILE as KNOWN_FILE from .files import iter_cpython_files diff --git a/Tools/c-analyzer/cpython/files.py b/Tools/c-analyzer/cpython/files.py new file mode 100644 index 00000000000000..543097af7bcd50 --- /dev/null +++ b/Tools/c-analyzer/cpython/files.py @@ -0,0 +1,29 @@ +from c_analyzer.common.files import ( + C_SOURCE_SUFFIXES, walk_tree, iter_files_by_suffix, + ) + +from . import SOURCE_DIRS, REPO_ROOT + +# XXX need tests: +# * iter_files() + + +def iter_files(*, + walk=walk_tree, + _files=iter_files_by_suffix, + ): + """Yield each file in the tree for each of the given directory names.""" + excludedtrees = [ + os.path.join('Include', 'cpython', ''), + ] + def is_excluded(filename): + for root in excludedtrees: + if filename.startswith(root): + return True + return False + for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT, + walk=walk, + ): + if is_excluded(filename): + continue + yield filename diff --git a/Tools/c-analyzer/cpython/find.py b/Tools/c-analyzer/cpython/find.py new file mode 100644 index 00000000000000..1da0585fe1a057 --- /dev/null +++ b/Tools/c-analyzer/cpython/find.py @@ -0,0 +1,80 @@ +import os.path + +from c_analyzer.common.info import UNKNOWN +from c_analyzer.variables import find as _common + +from . import SOURCE_DIRS, PYTHON, REPO_ROOT +from .known import ( + from_file as known_from_file, + DATA_FILE as KNOWN_FILE, + ) +from .supported import ( + ignored_from_file, IGNORED_FILE, is_supported, _is_object, + ) + +# XXX need tests: +# * vars_from_binary() +# * vars_from_source() +# * supported_vars() + + +def vars_from_binary(*, + dirnames=SOURCE_DIRS, + known=None, + _iter_vars=_common.vars_from_binary, + ): + """Yield a Variable for each found Symbol. + + Details are filled in from the given "known" variables and types. + """ + yield from _iter_vars(PYTHON, known=known, dirnames=dirnames) + + +def vars_from_source(*, + preprocessed=None, + known=None, # for types + _iter_vars=_common.vars_from_source, + ): + """Yield a Variable for each declaration in the raw source code. + + Details are filled in from the given "known" variables and types. + """ + yield from _iter_vars( + dirnames=SOURCE_DIRS, + preprocessed=preprocessed, + known=known, + ) + + +def supported_vars(dirnames=SOURCE_DIRS, *, + known=KNOWN_FILE, + ignored=IGNORED_FILE, + skip_objects=False, + _known_from_file=known_from_file, + _ignored_from_file=ignored_from_file, + _relpath=os.path.relpath, + _iter_vars=vars_from_binary, + _is_supported=is_supported, + ): + """Yield (var, is supported) for each found variable.""" + if isinstance(known, str): + known = _known_from_file(known) + if isinstance(ignored, str): + ignored = _ignored_from_file(ignored) + + if dirnames == SOURCE_DIRS: + dirnames = [_relpath(d, REPO_ROOT) for d in dirnames] + # XXX For now we only use known variables (no source lookup). + dirnames = None + + knownvars = (known or {}).get('variables') + for var in _iter_vars(known=known, dirnames=dirnames): + if not var.isglobal: + continue + elif var.vartype == UNKNOWN: + yield var, None + # XXX Support proper filters instead. + elif skip_objects and _is_object(found.vartype): + continue + else: + yield var, _is_supported(var, ignored, known) diff --git a/Tools/c-analyzer/cpython/known.py b/Tools/c-analyzer/cpython/known.py new file mode 100644 index 00000000000000..6fe1e005daf6d3 --- /dev/null +++ b/Tools/c-analyzer/cpython/known.py @@ -0,0 +1,66 @@ +import csv +import os.path + +from c_analyzer.common import known as _common +from c_analyzer.parser.declarations import extract_storage +from c_analyzer.variables.info import Variable + +from . import DATA_DIR + + +# XXX need tests: +# * from_file() +# * look_up_variable() + + +DATA_FILE = os.path.join(DATA_DIR, 'known.tsv') + + +def _get_storage(decl, infunc): + # statics + if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')): + return 'static' + if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')): + return 'static' + if decl.startswith('PyDoc_VAR('): + return 'static' + if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')): + return 'static' + if decl.startswith('WRAP_METHOD('): + return 'static' + # public extern + if decl.startswith('PyAPI_DATA('): + return 'extern' + # Fall back to the normal handler. + return extract_storage(decl, infunc=infunc) + + +def _handle_var(varid, decl): +# if varid.name == 'id' and decl == UNKNOWN: +# # None of these are variables. +# decl = 'int id'; + storage = _get_storage(decl, varid.funcname) + return Variable(varid, storage, decl) + + +def from_file(infile=DATA_FILE, *, + _from_file=_common.from_file, + _handle_var=_handle_var, + ): + """Return the info for known declarations in the given file.""" + return _from_file(infile, handle_var=_handle_var) + + +def look_up_variable(varid, knownvars, *, + _lookup=_common.look_up_variable, + ): + """Return the known variable matching the given ID. + + "knownvars" is a mapping of ID to Variable. + + "match_files" is used to verify if two filenames point to + the same file. + + If no match is found then None is returned. + """ + return _lookup(varid, knownvars) diff --git a/Tools/c-analyzer/c_globals/supported.py b/Tools/c-analyzer/cpython/supported.py similarity index 98% rename from Tools/c-analyzer/c_globals/supported.py rename to Tools/c-analyzer/cpython/supported.py index 5bc523a2d58970..18786eefd8dedc 100644 --- a/Tools/c-analyzer/c_globals/supported.py +++ b/Tools/c-analyzer/cpython/supported.py @@ -1,10 +1,11 @@ import os.path import re -from c_analyzer.common import DATA_DIR from c_analyzer.common.info import ID from c_analyzer.common.util import read_tsv, write_tsv +from . import DATA_DIR + # XXX need tests: # * generate / script @@ -382,8 +383,8 @@ def _generate_ignored_file(variables, filename=None, *, if __name__ == '__main__': - from c_analyzer.common import SOURCE_DIRS - from c_analyzer.common.known import ( + from cpython import SOURCE_DIRS + from cpython.known import ( from_file as known_from_file, DATA_FILE as KNOWN_FILE, ) From 025e8c3a310ceb497cd2e926a5ae3e0a17a33982 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 15 Oct 2019 12:26:24 -0600 Subject: [PATCH 21/27] Do not use Variable in symbols or parser packages. --- .../test_variables/test_find.py | 14 ++++--- .../test_known.py | 4 +- Tools/c-analyzer/c_analyzer/parser/find.py | 39 +++++++++--------- Tools/c-analyzer/c_analyzer/parser/naive.py | 41 +++++++++---------- Tools/c-analyzer/c_analyzer/symbols/find.py | 24 +++++------ Tools/c-analyzer/c_analyzer/variables/find.py | 10 +++-- Tools/c-analyzer/c_analyzer/variables/info.py | 14 +++++-- .../c_analyzer/{common => variables}/known.py | 22 +++------- Tools/c-analyzer/cpython/known.py | 2 +- 9 files changed, 86 insertions(+), 84 deletions(-) rename Lib/test/test_tools/test_c_analyzer/{test_common => test_variables}/test_known.py (99%) rename Tools/c-analyzer/c_analyzer/{common => variables}/known.py (82%) diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py index b6c92db72f69d4..491076de0f7993 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py @@ -36,10 +36,14 @@ def setUp(self): def _iter_vars(self, binfile, resolve): self.calls.append(('_iter_vars', (binfile, resolve))) - return [(v, v) for v in self._return_iter_vars] - - def _get_symbol_resolver(self, known, dirnames=None, perfilecache=None): - self.calls.append(('_get_symbol_resolver', (known, dirnames, perfilecache))) + return [(v, v.id) for v in self._return_iter_vars] + + def _get_symbol_resolver(self, known=None, dirnames=(), *, + handle_var, + perfilecache=None, + ): + self.calls.append(('_get_symbol_resolver', + (known, dirnames, handle_var, perfilecache))) return self._return_get_symbol_resolver def test_typical(self): @@ -69,7 +73,7 @@ def test_typical(self): info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), ]) self.assertEqual(self.calls, [ - ('_get_symbol_resolver', (known, dirnames, {})), + ('_get_symbol_resolver', (known, dirnames, info.Variable.from_id, {})), ('_iter_vars', ('python', resolver)), ]) diff --git a/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py similarity index 99% rename from Lib/test/test_tools/test_c_analyzer/test_common/test_known.py rename to Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py index cb6761d63d7fac..49ff45c6d1b2cf 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_common/test_known.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_known.py @@ -4,9 +4,9 @@ from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer.variables.info import Variable from c_analyzer.common.info import ID - from c_analyzer.common.known import ( + from c_analyzer.variables.info import Variable + from c_analyzer.variables.known import ( read_file, from_file, ) diff --git a/Tools/c-analyzer/c_analyzer/parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py index be56e70a19e98a..fd143c02c89b47 100644 --- a/Tools/c-analyzer/c_analyzer/parser/find.py +++ b/Tools/c-analyzer/c_analyzer/parser/find.py @@ -1,5 +1,4 @@ -from ..common.info import UNKNOWN -from ..variables.info import Variable +from ..common.info import UNKNOWN, ID from . import declarations @@ -18,15 +17,17 @@ def _iter_vars(filenames, preprocessed, *, ): if kind != 'variable': continue - yield Variable.from_parts(filename, funcname, name, decl) + yield ID(filename, funcname, name), decl +# XXX Add a "handle_var" arg like we did for get_resolver()? + def variables(*filenames, perfilecache=None, preprocessed=False, _iter_vars=_iter_vars, ): - """Yield a Variable for each one found in the given files. + """Yield (varid, decl) for each variable found in the given files. If "preprocessed" is provided (and not False/None) then it is used to decide which tool to use to parse the source code after it runs @@ -38,7 +39,7 @@ def variables(*filenames, if perfilecache is None: yield from _iter_vars(filenames, preprocessed) else: - # XXX Cache per-file variables (e.g. `{filename: [Variable]}`). + # XXX Cache per-file variables (e.g. `{filename: [(varid, decl)]}`). raise NotImplementedError @@ -46,29 +47,29 @@ def variable(name, filenames, *, local=False, perfilecache=None, preprocessed=False, - _iter_variables=variables, + _iter_vars=variables, ): - """Return the first found Variable that matches. + """Return (varid, decl) for the first found variable that matches. If "local" is True then the first matching local variable in the file will always be returned. To avoid that, pass perfilecache and pop each variable from the cache after using it. """ - for var in _iter_variables(filenames, - perfilecache=perfilecache, - preprocessed=preprocessed, - ): - if var.name != name: + for varid, decl in _iter_vars(filenames, + perfilecache=perfilecache, + preprocessed=preprocessed, + ): + if varid.name != name: continue if local: - if var.funcname: - if var.funcname == UNKNOWN: + if varid.funcname: + if varid.funcname == UNKNOWN: raise NotImplementedError - return var - elif not var.funcname: - return var + return varid, decl + elif not varid.funcname: + return varid, decl else: - return None # No matching variable was found. + return None, None # No matching variable was found. def variable_from_id(id, filenames, *, @@ -76,7 +77,7 @@ def variable_from_id(id, filenames, *, preprocessed=False, _get_var=variable, ): - """Return the first found Variable that matches.""" + """Return (varid, decl) for the first found variable that matches.""" local = False if isinstance(id, str): name = id diff --git a/Tools/c-analyzer/c_analyzer/parser/naive.py b/Tools/c-analyzer/c_analyzer/parser/naive.py index 0a2fcf00a39cae..4a4822d84ff54d 100644 --- a/Tools/c-analyzer/c_analyzer/parser/naive.py +++ b/Tools/c-analyzer/c_analyzer/parser/naive.py @@ -1,7 +1,6 @@ import re -from ..common.info import UNKNOWN -from ..variables.info import Variable +from ..common.info import UNKNOWN, ID from .preprocessor import _iter_clean_lines @@ -55,7 +54,7 @@ def parse_variable_declaration(srcline): def parse_variable(srcline, funcname=None): - """Return a Variable for the variable declared on the line (or None).""" + """Return (varid, decl) for the variable declared on the line (or None).""" line = srcline.strip() # XXX Handle more than just static variables. @@ -74,7 +73,7 @@ def iter_variables(filename, *, _get_srclines=get_srclines, _default_parse_variable=parse_variable, ): - """Yield a Variable for each in the given source file.""" + """Yield (varid, decl) for each variable in the given source file.""" if parse_variable is None: parse_variable = _default_parse_variable @@ -99,13 +98,13 @@ def iter_variables(filename, *, info = parse_variable(line, funcname) if isinstance(info, list): for name, _funcname, decl in info: - yield Variable.from_parts(filename, _funcname, name, decl) + yield ID(filename, _funcname, name), decl continue name, decl = info if name is None: continue - yield Variable.from_parts(filename, funcname, name, decl) + yield ID(filename, funcname, name), decl def _match_varid(variable, name, funcname, ignored=None): @@ -134,12 +133,12 @@ def find_variable(filename, funcname, name, *, Return None if the variable is not found. """ - for variable in _iter_variables(filename, + for varid, decl in _iter_variables(filename, srccache=srccache, parse_variable=parse_variable, ): - if _match_varid(variable, name, funcname, ignored): - return variable + if _match_varid(varid, name, funcname, ignored): + return varid, decl else: return None @@ -149,10 +148,10 @@ def find_variables(varids, filenames=None, *, parse_variable=None, _find_symbol=find_variable, ): - """Yield a Variable for each ID. + """Yield (varid, decl) for each ID. If the variable is not found then its decl will be UNKNOWN. That - way there will be one resulting Variable per given ID. + way there will be one resulting variable per given ID. """ if srccache is _NOT_SET: srccache = {} @@ -163,18 +162,18 @@ def find_variables(varids, filenames=None, *, srcfiles = [varid.filename] else: if not filenames: - yield Variable(varid, UNKNOWN, UNKNOWN) + yield varid, UNKNOWN continue srcfiles = filenames for filename in srcfiles: - found = _find_varid(filename, varid.funcname, varid.name, - ignored=used, - srccache=srccache, - parse_variable=parse_variable, - ) - if found: - yield found - used.add(found) + varid, decl = _find_varid(filename, varid.funcname, varid.name, + ignored=used, + srccache=srccache, + parse_variable=parse_variable, + ) + if varid: + yield varid, decl + used.add(varid) break else: - yield Variable(varid, UNKNOWN, UNKNOWN) + yield varid, UNKNOWN diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py index 4f98074c51520f..402028459231b5 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -5,7 +5,6 @@ from ..common import files from ..common.info import UNKNOWN from ..parser import find as p_find -from ..variables.info import Variable from . import _nm from .info import Symbol @@ -27,6 +26,7 @@ def _resolve_known(symbol, knownvars): def get_resolver(known=None, dirnames=(), *, + handle_var, filenames=None, perfilecache=None, preprocessed=False, @@ -66,20 +66,22 @@ def resolve(symbol): found = _resolve_known(symbol, knownvars) if found is None: #return None - found = _from_source(symbol, filenames, - perfilecache=perfilecache, - preprocessed=preprocessed, - ) + varid, decl = _from_source(symbol, filenames, + perfilecache=perfilecache, + preprocessed=preprocessed, + ) + found = handle_var(varid, decl) return found else: def resolve(symbol): return _resolve_known(symbol, knownvars) elif filenames: def resolve(symbol): - return _from_source(symbol, filenames, - perfilecache=perfilecache, - preprocessed=preprocessed, - ) + varid, decl = _from_source(symbol, filenames, + perfilecache=perfilecache, + preprocessed=preprocessed, + ) + return handle_var(varid, decl) else: def resolve(symbol): return None @@ -91,7 +93,7 @@ def symbol(symbol, filenames, known=None, *, preprocessed=False, _get_resolver=get_resolver, ): - """Return the Variable matching the given symbol. + """Return a Variable for the one matching the given symbol. "symbol" can be one of several objects: @@ -142,6 +144,4 @@ def variables(binfile, *, if symbol.kind != Symbol.KIND.VARIABLE: continue var = resolve(symbol) or None - #if var is None: - # var = Variable(symbol.id, UNKNOWN) yield var, symbol diff --git a/Tools/c-analyzer/c_analyzer/variables/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py index 05ea2fbc8d7f0c..6fb26d2f555c8e 100644 --- a/Tools/c-analyzer/c_analyzer/variables/find.py +++ b/Tools/c-analyzer/c_analyzer/variables/find.py @@ -35,6 +35,7 @@ def vars_from_binary(binfile, *, """ cache = {} resolve = _get_symbol_resolver(known, dirnames, + handle_var=Variable.from_id, perfilecache=cache, ) for var, symbol in _iter_vars(binfile, resolve=resolve): @@ -60,9 +61,10 @@ def vars_from_source(filenames=None, *, return filenames = _iter_files(dirnames, ('.c', '.h')) cache = {} - for var in _iter_vars(filenames, - perfilecache=cache, - preprocessed=preprocessed, - ): + for varid, decl in _iter_vars(filenames, + perfilecache=cache, + preprocessed=preprocessed, + ): + var = Variable.from_id(varid, decl) yield var _remove_cached(cache, var) diff --git a/Tools/c-analyzer/c_analyzer/variables/info.py b/Tools/c-analyzer/c_analyzer/variables/info.py index e5910bc15ba85f..336a523c7a2dba 100644 --- a/Tools/c-analyzer/c_analyzer/variables/info.py +++ b/Tools/c-analyzer/c_analyzer/variables/info.py @@ -32,13 +32,19 @@ class Variable(_NTBase, @classonly def from_parts(cls, filename, funcname, name, decl, storage=None): + varid = ID(filename, funcname, name) if storage is None: - from ..parser.declarations import extract_storage - storage = extract_storage(decl, infunc=funcname) - id = ID(filename, funcname, name) - self = cls(id, storage, decl) + self = cls.from_id(varid, decl) + else: + self = cls(varid, storage, decl) return self + @classonly + def from_id(cls, varid, decl): + from ..parser.declarations import extract_storage + storage = extract_storage(decl, infunc=varid.funcname) + return cls(varid, storage, decl) + def __new__(cls, id, storage, vartype): self = super().__new__( cls, diff --git a/Tools/c-analyzer/c_analyzer/common/known.py b/Tools/c-analyzer/c_analyzer/variables/known.py similarity index 82% rename from Tools/c-analyzer/c_analyzer/common/known.py rename to Tools/c-analyzer/c_analyzer/variables/known.py index 8fdeba675a5757..aa2934a069e16b 100644 --- a/Tools/c-analyzer/c_analyzer/common/known.py +++ b/Tools/c-analyzer/c_analyzer/variables/known.py @@ -1,7 +1,8 @@ import csv -from c_analyzer.common.info import ID, UNKNOWN -from c_analyzer.common.util import read_tsv +from ..common.info import ID, UNKNOWN +from ..common.util import read_tsv +from .info import Variable # XXX need tests: @@ -28,22 +29,11 @@ def read_file(infile, *, yield kind, id, declaration -def _handle_var(varid, decl): - from ..parser.declarations import extract_storage - from ..variables.info import Variable - storage = extract_storage(decl, infunc=varid.funcname) - return Variable(id, storage, decl) - - def from_file(infile, *, - handle_var=_handle_var, + handle_var=Variable.from_id, _read_file=read_file, ): - """Return the info for known declarations in the given file. - - "handle_var" is a function that takes (varid, decl) and returns - a corresponding Variable. - """ + """Return the info for known declarations in the given file.""" known = { 'variables': {}, #'types': {}, @@ -64,7 +54,7 @@ def from_file(infile, *, def look_up_variable(varid, knownvars, *, match_files=(lambda f1, f2: f1 == f2), ): - """Return (the known variable matching the given ID. + """Return the known Variable matching the given ID. "knownvars" is a mapping of ID to Variable. diff --git a/Tools/c-analyzer/cpython/known.py b/Tools/c-analyzer/cpython/known.py index 6fe1e005daf6d3..c3cc2c06026ce1 100644 --- a/Tools/c-analyzer/cpython/known.py +++ b/Tools/c-analyzer/cpython/known.py @@ -1,8 +1,8 @@ import csv import os.path -from c_analyzer.common import known as _common from c_analyzer.parser.declarations import extract_storage +from c_analyzer.variables import known as _common from c_analyzer.variables.info import Variable from . import DATA_DIR From f60551e1df774e2e1459b790afd1de53f5cf69d2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 12:40:50 -0600 Subject: [PATCH 22/27] Drop "INCLUDES" from SOURCE_DIRS. --- Tools/c-analyzer/cpython/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/__init__.py b/Tools/c-analyzer/cpython/__init__.py index d2aafb57dc9c1d..c0620207d3a464 100644 --- a/Tools/c-analyzer/cpython/__init__.py +++ b/Tools/c-analyzer/cpython/__init__.py @@ -10,8 +10,10 @@ os.path.dirname( # .. os.path.dirname(TOOL_ROOT))) # Tools/ -SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ +INCLUDE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ 'Include', + ]] +SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [ 'Python', 'Parser', 'Objects', From 4beba49a03726785e01512fb3bfb5c68caffe077 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 14:56:28 -0600 Subject: [PATCH 23/27] Process files more cleanly. --- .../test_cpython/test___main__.py | 39 ++++----- .../test_variables/test_find.py | 15 ++-- Tools/c-analyzer/c_analyzer/parser/find.py | 14 +++- Tools/c-analyzer/c_analyzer/symbols/_nm.py | 5 +- Tools/c-analyzer/c_analyzer/symbols/find.py | 80 +++++++++++++------ Tools/c-analyzer/c_analyzer/variables/find.py | 39 ++++----- Tools/c-analyzer/cpython/__init__.py | 2 +- Tools/c-analyzer/cpython/__main__.py | 19 +++-- Tools/c-analyzer/cpython/find.py | 58 +++++++++----- 9 files changed, 165 insertions(+), 106 deletions(-) diff --git a/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py b/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py index e1a9691260fbf7..6d69ed7525b595 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_cpython/test___main__.py @@ -67,8 +67,8 @@ def calls(self): # self.calls.append(('_ignored_from_file', args)) # return self._return_ignored_from_file or {} - def _find(self, dirs, known, ignored, skip_objects=False): - self.calls.append(('_find', (dirs, known, ignored, skip_objects))) + def _find(self, known, ignored, skip_objects=False): + self.calls.append(('_find', (known, ignored, skip_objects))) return self._return_find def _show(self, *args): @@ -91,7 +91,7 @@ def test_defaults(self): self.assertEqual( self.calls[0], - ('_find', (SOURCE_DIRS, KNOWN_FILE, IGNORED_FILE, False)), + ('_find', (KNOWN_FILE, IGNORED_FILE, False)), ) def test_all_supported(self): @@ -99,7 +99,6 @@ def test_all_supported(self): dirs = ['src1', 'src2', 'Include'] cmd_check('check', - dirs, known='known.tsv', ignored='ignored.tsv', _find=self._find, @@ -108,17 +107,15 @@ def test_all_supported(self): ) self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), + ('_find', ('known.tsv', 'ignored.tsv', False)), #('_print', ('okay',)), ]) def test_some_unsupported(self): self._return_find = TYPICAL - dirs = ['src1', 'src2', 'Include'] with self.assertRaises(SystemExit) as cm: cmd_check('check', - dirs, known='known.tsv', ignored='ignored.tsv', _find=self._find, @@ -128,7 +125,7 @@ def test_some_unsupported(self): unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), + ('_find', ('known.tsv', 'ignored.tsv', False)), ('_print', ('ERROR: found unsupported global variables',)), ('_print', ()), ('_show', (sorted(unsupported),)), @@ -150,15 +147,13 @@ def test_defaults(self): self.assertEqual( self.calls[0], - ('_find', (SOURCE_DIRS, KNOWN_FILE, IGNORED_FILE, False)), + ('_find', (KNOWN_FILE, IGNORED_FILE, False)), ) def test_typical(self): self._return_find = TYPICAL - dirs = ['src1', 'src2', 'Include'] cmd_show('show', - dirs, known='known.tsv', ignored='ignored.tsv', _find=self._find, @@ -169,7 +164,7 @@ def test_typical(self): supported = [v for v, s in TYPICAL if s] unsupported = [v for v, s in TYPICAL if not s] self.assertEqual(self.calls, [ - ('_find', (dirs, 'known.tsv', 'ignored.tsv', False)), + ('_find', ('known.tsv', 'ignored.tsv', False)), ('_print', ('supported:',)), ('_print', ('----------',)), ('_show', (sorted(supported),)), @@ -206,7 +201,7 @@ def test_check_no_args(self): self.assertEqual(cmdkwargs, { 'ignored': IGNORED_FILE, 'known': KNOWN_FILE, - 'dirs': SOURCE_DIRS, + #'dirs': SOURCE_DIRS, }) def test_check_full_args(self): @@ -214,16 +209,16 @@ def test_check_full_args(self): 'check', '--ignored', 'spam.tsv', '--known', 'eggs.tsv', - 'dir1', - 'dir2', - 'dir3', + #'dir1', + #'dir2', + #'dir3', ]) self.assertEqual(cmd, 'check') self.assertEqual(cmdkwargs, { 'ignored': 'spam.tsv', 'known': 'eggs.tsv', - 'dirs': ['dir1', 'dir2', 'dir3'] + #'dirs': ['dir1', 'dir2', 'dir3'] }) def test_show_no_args(self): @@ -235,7 +230,7 @@ def test_show_no_args(self): self.assertEqual(cmdkwargs, { 'ignored': IGNORED_FILE, 'known': KNOWN_FILE, - 'dirs': SOURCE_DIRS, + #'dirs': SOURCE_DIRS, 'skip_objects': False, }) @@ -244,16 +239,16 @@ def test_show_full_args(self): 'show', '--ignored', 'spam.tsv', '--known', 'eggs.tsv', - 'dir1', - 'dir2', - 'dir3', + #'dir1', + #'dir2', + #'dir3', ]) self.assertEqual(cmd, 'show') self.assertEqual(cmdkwargs, { 'ignored': 'spam.tsv', 'known': 'eggs.tsv', - 'dirs': ['dir1', 'dir2', 'dir3'], + #'dirs': ['dir1', 'dir2', 'dir3'], 'skip_objects': False, }) diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py index 491076de0f7993..e82b2a1cf756c0 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py @@ -34,16 +34,17 @@ def setUp(self): _get_symbol_resolver=self._get_symbol_resolver, ) - def _iter_vars(self, binfile, resolve): - self.calls.append(('_iter_vars', (binfile, resolve))) + def _iter_vars(self, binfile, resolve, handle_id): + self.calls.append(('_iter_vars', (binfile, resolve, handle_id))) return [(v, v.id) for v in self._return_iter_vars] def _get_symbol_resolver(self, known=None, dirnames=(), *, handle_var, + filenames=None, perfilecache=None, ): self.calls.append(('_get_symbol_resolver', - (known, dirnames, handle_var, perfilecache))) + (known, dirnames, handle_var, filenames, perfilecache))) return self._return_get_symbol_resolver def test_typical(self): @@ -57,11 +58,11 @@ def test_typical(self): info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), ] known = object() - dirnames = object() + filenames = object() found = list(vars_from_binary('python', known=known, - dirnames=dirnames, + filenames=filenames, **self.kwargs)) self.assertEqual(found, [ @@ -73,8 +74,8 @@ def test_typical(self): info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), ]) self.assertEqual(self.calls, [ - ('_get_symbol_resolver', (known, dirnames, info.Variable.from_id, {})), - ('_iter_vars', ('python', resolver)), + ('_get_symbol_resolver', (filenames, known, info.Variable.from_id, None, {})), + ('_iter_vars', ('python', resolver, None)), ]) # self._return_iter_symbols = [ diff --git a/Tools/c-analyzer/c_analyzer/parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py index fd143c02c89b47..9e422c3e279d99 100644 --- a/Tools/c-analyzer/c_analyzer/parser/find.py +++ b/Tools/c-analyzer/c_analyzer/parser/find.py @@ -9,15 +9,20 @@ def _iter_vars(filenames, preprocessed, *, + handle_id=None, _iter_decls=declarations.iter_all, ): + if handle_id is None: + handle_id = ID + for filename in filenames or (): for kind, funcname, name, decl in _iter_decls(filename, preprocessed=preprocessed, ): if kind != 'variable': continue - yield ID(filename, funcname, name), decl + varid = handle_id(filename, funcname, name) + yield varid, decl # XXX Add a "handle_var" arg like we did for get_resolver()? @@ -25,6 +30,8 @@ def _iter_vars(filenames, preprocessed, *, def variables(*filenames, perfilecache=None, preprocessed=False, + known=None, # for types + handle_id=None, _iter_vars=_iter_vars, ): """Yield (varid, decl) for each variable found in the given files. @@ -47,6 +54,7 @@ def variable(name, filenames, *, local=False, perfilecache=None, preprocessed=False, + handle_id=None, _iter_vars=variables, ): """Return (varid, decl) for the first found variable that matches. @@ -75,6 +83,7 @@ def variable(name, filenames, *, def variable_from_id(id, filenames, *, perfilecache=None, preprocessed=False, + handle_id=None, _get_var=variable, ): """Return (varid, decl) for the first found variable that matches.""" @@ -90,8 +99,9 @@ def variable_from_id(id, filenames, *, name = id.name if id.filename and id.filename != UNKNOWN: filenames = [id.filename] - return _get_var(id, filenames, + return _get_var(name, filenames, local=local, perfilecache=perfilecache, preprocessed=preprocessed, + handle_id=handle_id, ) diff --git a/Tools/c-analyzer/c_analyzer/symbols/_nm.py b/Tools/c-analyzer/c_analyzer/symbols/_nm.py index bee9a4b6076cb1..f3a75a6d4ba824 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/_nm.py +++ b/Tools/c-analyzer/c_analyzer/symbols/_nm.py @@ -43,6 +43,7 @@ def _is_special_symbol(name): def iter_symbols(binfile, *, nm=None, + handle_id=None, _which=shutil.which, _run=util.run_cmd, ): @@ -51,6 +52,8 @@ def iter_symbols(binfile, *, nm = _which('nm') if not nm: raise NotImplementedError + if handle_id is None: + handle_id = info.ID argv = [nm, '--line-numbers', @@ -71,7 +74,7 @@ def iter_symbols(binfile, *, elif _is_special_symbol(name): continue yield Symbol( - id=(filename, funcname, name), + id=handle_id(filename, funcname, name), kind=kind, external=external, ) diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py index 402028459231b5..a0b106fb951730 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -3,17 +3,18 @@ import shutil from ..common import files -from ..common.info import UNKNOWN +from ..common.info import UNKNOWN, ID from ..parser import find as p_find from . import _nm from .info import Symbol # XXX need tests: -# * get_resolver -# * symbol -# * symbols -# * variables +# * get_resolver() +# * get_resolver_from_dirs() +# * symbol() +# * symbols() +# * variables() def _resolve_known(symbol, knownvars): @@ -25,40 +26,31 @@ def _resolve_known(symbol, knownvars): return knownvars.pop(varid) -def get_resolver(known=None, dirnames=(), *, +def get_resolver(filenames=None, known=None, *, handle_var, - filenames=None, + check_filename=None, perfilecache=None, preprocessed=False, - _iter_files=files.iter_files_by_suffix, _from_source=p_find.variable_from_id, ): """Return a "resolver" func for the given known vars/types and filenames. + "handle_var" is a callable that takes (ID, decl) and returns a + Variable. Variable.from_id is a suitable callable. + The returned func takes a single Symbol and returns a corresponding Variable. If the symbol was located then the variable will be valid, populated with the corresponding information. Otherwise None is returned. """ knownvars = (known or {}).get('variables') - if filenames: - filenames = list(filenames) - def check_filename(filename): - return filename in filenames - elif dirnames: - dirnames = [d if d.endswith(os.path.sep) else d + os.path.sep - for d in dirnames] - filenames = list(_iter_files(dirnames, ('.c', '.h'))) - def check_filename(filename): - for dirname in dirnames: - if filename.startswith(dirname): - return True - else: - return False - if knownvars: knownvars = dict(knownvars) # a copy if filenames: + if check_filename is None: + filenames = list(filenames) + def check_filename(filename): + return filename in filenames def resolve(symbol): # XXX Check "found" instead? if not check_filename(symbol.filename): @@ -88,9 +80,42 @@ def resolve(symbol): return resolve +def get_resolver_from_dirs(dirnames, known=None, *, + handle_var, + suffixes=('.c',), + perfilecache=None, + preprocessed=False, + _iter_files=files.iter_files_by_suffix, + _get_resolver=get_resolver, + ): + """Return a "resolver" func for the given known vars/types and filenames. + + "dirnames" should be absolute paths. If not then they will be + resolved relative to CWD. + + See get_resolver(). + """ + dirnames = [d if d.endswith(os.path.sep) else d + os.path.sep + for d in dirnames] + filenames = _iter_files(dirnames, suffixes) + def check_filename(filename): + for dirname in dirnames: + if filename.startswith(dirname): + return True + else: + return False + return _get_resolver(filenames, known, + handle_var=handle_var, + check_filename=check_filename, + perfilecache=perfilecache, + preprocessed=preprocessed, + ) + + def symbol(symbol, filenames, known=None, *, perfilecache=None, preprocessed=False, + handle_id=None, _get_resolver=get_resolver, ): """Return a Variable for the one matching the given symbol. @@ -107,6 +132,7 @@ def symbol(symbol, filenames, known=None, *, "" or "UNKNOWN" then only local variables will be searched for. """ resolve = _get_resolver(known, filenames, + handle_id=handle_id, perfilecache=perfilecache, preprocessed=preprocessed, ) @@ -118,12 +144,13 @@ def _get_platform_tool(): # XXX Support this. raise NotImplementedError elif nm := shutil.which('nm'): - return lambda b: _nm.iter_symbols(b, nm=nm) + return lambda b, hi: _nm.iter_symbols(b, nm=nm, handle_id=hi) else: raise NotImplementedError def symbols(binfile, *, + handle_id=None, _file_exists=os.path.exists, _get_platform_tool=_get_platform_tool, ): @@ -132,15 +159,16 @@ def symbols(binfile, *, raise Exception('executable missing (need to build it first?)') _iter_symbols = _get_platform_tool() - yield from _iter_symbols(binfile) + yield from _iter_symbols(binfile, handle_id) def variables(binfile, *, resolve, + handle_id=None, _iter_symbols=symbols, ): """Yield (Variable, Symbol) for each found symbol.""" - for symbol in _iter_symbols(binfile): + for symbol in _iter_symbols(binfile, handle_id=handle_id): if symbol.kind != Symbol.KIND.VARIABLE: continue var = resolve(symbol) or None diff --git a/Tools/c-analyzer/c_analyzer/variables/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py index 6fb26d2f555c8e..5378026fc00135 100644 --- a/Tools/c-analyzer/c_analyzer/variables/find.py +++ b/Tools/c-analyzer/c_analyzer/variables/find.py @@ -25,7 +25,9 @@ def _remove_cached(cache, var): def vars_from_binary(binfile, *, known=None, - dirnames=None, + filenames=None, + handle_id=None, + handle_var=Variable.from_id, _iter_vars=s_find.variables, _get_symbol_resolver=s_find.get_resolver, ): @@ -34,37 +36,38 @@ def vars_from_binary(binfile, *, Details are filled in from the given "known" variables and types. """ cache = {} - resolve = _get_symbol_resolver(known, dirnames, - handle_var=Variable.from_id, + resolve = _get_symbol_resolver(filenames, known, + handle_var=handle_var, perfilecache=cache, ) - for var, symbol in _iter_vars(binfile, resolve=resolve): + for var, symbol in _iter_vars(binfile, + resolve=resolve, + handle_id=handle_id, + ): if var is None: var = Variable(symbol.id, UNKNOWN, UNKNOWN) yield var _remove_cached(cache, var) -def vars_from_source(filenames=None, *, - iter_vars=p_find.variables, +def vars_from_source(filenames, *, preprocessed=None, - known=None, # for types - dirnames=None, - _iter_files=files.iter_files_by_suffix, + known=None, + handle_id=None, + handle_var=Variable.from_id, + iter_vars=p_find.variables, ): """Yield a Variable for each declaration in the raw source code. Details are filled in from the given "known" variables and types. """ - if filenames is None: - if not dirnames: - return - filenames = _iter_files(dirnames, ('.c', '.h')) cache = {} - for varid, decl in _iter_vars(filenames, - perfilecache=cache, - preprocessed=preprocessed, - ): - var = Variable.from_id(varid, decl) + for varid, decl in iter_vars(filenames or (), + perfilecache=cache, + preprocessed=preprocessed, + known=known, + handle_id=handle_id, + ): + var = handle_var(varid, decl) yield var _remove_cached(cache, var) diff --git a/Tools/c-analyzer/cpython/__init__.py b/Tools/c-analyzer/cpython/__init__.py index c0620207d3a464..ae45b424e3cc8e 100644 --- a/Tools/c-analyzer/cpython/__init__.py +++ b/Tools/c-analyzer/cpython/__init__.py @@ -2,7 +2,7 @@ import sys -TOOL_ROOT = ( +TOOL_ROOT = os.path.abspath( os.path.dirname( # c-analyzer/ os.path.dirname(__file__))) # cpython/ DATA_DIR = TOOL_ROOT diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 517217197821bb..6b0f9bcb9687fb 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -66,7 +66,7 @@ def _match_unused_global(variable): # XXX Move this check to its own command. -def cmd_check_cache(cmd, dirs=SOURCE_DIRS, *, +def cmd_check_cache(cmd, *, known=KNOWN_FILE, ignored=IGNORED_FILE, _known_from_file=known_from_file, @@ -76,7 +76,7 @@ def cmd_check_cache(cmd, dirs=SOURCE_DIRS, *, used = set() unknown = set() - for var, supported in _find(dirs, known=known, ignored=ignored): + for var, supported in _find(known=known, ignored=ignored): if supported is None: unknown.add(var) continue @@ -84,7 +84,7 @@ def cmd_check_cache(cmd, dirs=SOURCE_DIRS, *, _check_results(unknown, known['variables'], used) -def cmd_check(cmd, dirs=SOURCE_DIRS, *, +def cmd_check(cmd, *, known=KNOWN_FILE, ignored=IGNORED_FILE, _find=supported_vars, @@ -98,7 +98,7 @@ def cmd_check(cmd, dirs=SOURCE_DIRS, *, will be printed out. """ unsupported = [] - for var, supported in _find(dirs, known=known, ignored=ignored): + for var, supported in _find(known=known, ignored=ignored): if not supported: unsupported.append(var) @@ -113,7 +113,7 @@ def cmd_check(cmd, dirs=SOURCE_DIRS, *, sys.exit(1) -def cmd_show(cmd, dirs=SOURCE_DIRS, *, +def cmd_show(cmd, *, known=KNOWN_FILE, ignored=IGNORED_FILE, skip_objects=False, @@ -128,8 +128,7 @@ def cmd_show(cmd, dirs=SOURCE_DIRS, *, """ allsupported = [] allunsupported = [] - for found, supported in _find(dirs, - known=known, + for found, supported in _find(known=known, ignored=ignored, skip_objects=skip_objects, ): @@ -169,9 +168,9 @@ def parse_args(prog=PROG, argv=sys.argv[1:], *, _fail=None): common.add_argument('--known', metavar='FILE', default=KNOWN_FILE, help='path to file that lists known types') - common.add_argument('dirs', metavar='DIR', nargs='*', - default=SOURCE_DIRS, - help='a directory to check') + #common.add_argument('dirs', metavar='DIR', nargs='*', + # default=SOURCE_DIRS, + # help='a directory to check') parser = argparse.ArgumentParser( prog=prog, diff --git a/Tools/c-analyzer/cpython/find.py b/Tools/c-analyzer/cpython/find.py index 1da0585fe1a057..706e998f0eceea 100644 --- a/Tools/c-analyzer/cpython/find.py +++ b/Tools/c-analyzer/cpython/find.py @@ -1,6 +1,7 @@ import os.path -from c_analyzer.common.info import UNKNOWN +from c_analyzer.common import files +from c_analyzer.common.info import UNKNOWN, ID from c_analyzer.variables import find as _common from . import SOURCE_DIRS, PYTHON, REPO_ROOT @@ -18,41 +19,66 @@ # * supported_vars() +def _handle_id(filename, funcname, name, *, + _relpath=os.path.relpath, + ): + filename = _relpath(filename, REPO_ROOT) + return ID(filename, funcname, name) + + def vars_from_binary(*, - dirnames=SOURCE_DIRS, - known=None, + known=KNOWN_FILE, + _known_from_file=known_from_file, + _iter_files=files.iter_files_by_suffix, _iter_vars=_common.vars_from_binary, ): """Yield a Variable for each found Symbol. Details are filled in from the given "known" variables and types. """ - yield from _iter_vars(PYTHON, known=known, dirnames=dirnames) + if isinstance(known, str): + known = _known_from_file(known) + dirnames = SOURCE_DIRS + suffixes = ('.c',) + filenames = _iter_files(dirnames, suffixes) + # XXX For now we only use known variables (no source lookup). + filenames = None + yield from _iter_vars(PYTHON, + known=known, + filenames=filenames, + handle_id=_handle_id, + ) def vars_from_source(*, preprocessed=None, - known=None, # for types + known=KNOWN_FILE, + _known_from_file=known_from_file, + _iter_files=files.iter_files_by_suffix, _iter_vars=_common.vars_from_source, ): """Yield a Variable for each declaration in the raw source code. Details are filled in from the given "known" variables and types. """ - yield from _iter_vars( - dirnames=SOURCE_DIRS, - preprocessed=preprocessed, - known=known, - ) + if isinstance(known, str): + known = _known_from_file(known) + dirnames = SOURCE_DIRS + suffixes = ('.c',) + filenames = _iter_files(dirnames, suffixes) + yield from _iter_vars(filenames, + preprocessed=preprocessed, + known=known, + handle_id=_handle_id, + ) -def supported_vars(dirnames=SOURCE_DIRS, *, +def supported_vars(*, known=KNOWN_FILE, ignored=IGNORED_FILE, skip_objects=False, _known_from_file=known_from_file, _ignored_from_file=ignored_from_file, - _relpath=os.path.relpath, _iter_vars=vars_from_binary, _is_supported=is_supported, ): @@ -62,13 +88,7 @@ def supported_vars(dirnames=SOURCE_DIRS, *, if isinstance(ignored, str): ignored = _ignored_from_file(ignored) - if dirnames == SOURCE_DIRS: - dirnames = [_relpath(d, REPO_ROOT) for d in dirnames] - # XXX For now we only use known variables (no source lookup). - dirnames = None - - knownvars = (known or {}).get('variables') - for var in _iter_vars(known=known, dirnames=dirnames): + for var in _iter_vars(known=known): if not var.isglobal: continue elif var.vartype == UNKNOWN: From 9e5c09bc78370356f7791e5d779cde3fe1a5e3a1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 15:28:08 -0600 Subject: [PATCH 24/27] Fix a typo. --- Tools/c-analyzer/c_analyzer/parser/declarations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_analyzer/parser/declarations.py b/Tools/c-analyzer/c_analyzer/parser/declarations.py index 7216ca3b1b6cda..f37072cccad864 100644 --- a/Tools/c-analyzer/c_analyzer/parser/declarations.py +++ b/Tools/c-analyzer/c_analyzer/parser/declarations.py @@ -335,5 +335,5 @@ def iter_all(filename, *, """ # XXX For the moment we cheat. for funcname, name, decl in iter_variables(filename, - proprocessed=preprocessed): + preprocessed=preprocessed): yield 'variable', funcname, name, decl From 6e14276d795083d57c25179f4d40313907bb6238 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 15:32:50 -0600 Subject: [PATCH 25/27] Call check_filename() properly. --- .../test_tools/test_c_analyzer/test_variables/test_find.py | 6 ++++-- Tools/c-analyzer/c_analyzer/variables/find.py | 2 ++ Tools/c-analyzer/cpython/find.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py index e82b2a1cf756c0..7a13cf3f5bf56c 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_variables/test_find.py @@ -41,10 +41,12 @@ def _iter_vars(self, binfile, resolve, handle_id): def _get_symbol_resolver(self, known=None, dirnames=(), *, handle_var, filenames=None, + check_filename=None, perfilecache=None, ): self.calls.append(('_get_symbol_resolver', - (known, dirnames, handle_var, filenames, perfilecache))) + (known, dirnames, handle_var, filenames, + check_filename, perfilecache))) return self._return_get_symbol_resolver def test_typical(self): @@ -74,7 +76,7 @@ def test_typical(self): info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), ]) self.assertEqual(self.calls, [ - ('_get_symbol_resolver', (filenames, known, info.Variable.from_id, None, {})), + ('_get_symbol_resolver', (filenames, known, info.Variable.from_id, None, None, {})), ('_iter_vars', ('python', resolver, None)), ]) diff --git a/Tools/c-analyzer/c_analyzer/variables/find.py b/Tools/c-analyzer/c_analyzer/variables/find.py index 5378026fc00135..3fe7284fc00a79 100644 --- a/Tools/c-analyzer/c_analyzer/variables/find.py +++ b/Tools/c-analyzer/c_analyzer/variables/find.py @@ -27,6 +27,7 @@ def vars_from_binary(binfile, *, known=None, filenames=None, handle_id=None, + check_filename=None, handle_var=Variable.from_id, _iter_vars=s_find.variables, _get_symbol_resolver=s_find.get_resolver, @@ -38,6 +39,7 @@ def vars_from_binary(binfile, *, cache = {} resolve = _get_symbol_resolver(filenames, known, handle_var=handle_var, + check_filename=check_filename, perfilecache=cache, ) for var, symbol in _iter_vars(binfile, diff --git a/Tools/c-analyzer/cpython/find.py b/Tools/c-analyzer/cpython/find.py index 706e998f0eceea..a7bc0b477b839c 100644 --- a/Tools/c-analyzer/cpython/find.py +++ b/Tools/c-analyzer/cpython/find.py @@ -47,6 +47,7 @@ def vars_from_binary(*, known=known, filenames=filenames, handle_id=_handle_id, + check_filename=(lambda n: True), ) From 0ce0cd1dfb4d8b52cb3073e5dc849d1d31bbfb7d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 15:34:49 -0600 Subject: [PATCH 26/27] Fix one last import. --- Lib/test/test_check_c_globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py index a3925f0ca887ea..030debc452e409 100644 --- a/Lib/test/test_check_c_globals.py +++ b/Lib/test/test_check_c_globals.py @@ -3,7 +3,7 @@ test.test_tools.skip_if_missing('c-analyzer') with test.test_tools.imports_under_tool('c-analyzer'): - from c_globals.__main__ import main + from cpython.__main__ import main class ActualChecks(unittest.TestCase): From 264d698f031a05df9c57478c29f9b8df736b487d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 18 Oct 2019 16:17:29 -0600 Subject: [PATCH 27/27] Fix whitespace. --- Tools/c-analyzer/c_analyzer/common/info.py | 2 +- Tools/c-analyzer/c_analyzer/parser/find.py | 2 +- Tools/c-analyzer/c_analyzer/symbols/find.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer/common/info.py b/Tools/c-analyzer/c_analyzer/common/info.py index 34b7a3e66fd67b..3f3f8c5b05de59 100644 --- a/Tools/c-analyzer/c_analyzer/common/info.py +++ b/Tools/c-analyzer/c_analyzer/common/info.py @@ -75,7 +75,7 @@ def match(self, other, *, match_files=(lambda f1, f2: f1 == f2), ): """Return True if the two match. - + At least one of the two must be completely valid (no UNKNOWN anywhere). Otherwise False is returned. The remaining one *may* have UNKNOWN for both funcname and filename. It must diff --git a/Tools/c-analyzer/c_analyzer/parser/find.py b/Tools/c-analyzer/c_analyzer/parser/find.py index 9e422c3e279d99..3860d3d459b18d 100644 --- a/Tools/c-analyzer/c_analyzer/parser/find.py +++ b/Tools/c-analyzer/c_analyzer/parser/find.py @@ -77,7 +77,7 @@ def variable(name, filenames, *, elif not varid.funcname: return varid, decl else: - return None, None # No matching variable was found. + return None, None # No matching variable was found. def variable_from_id(id, filenames, *, diff --git a/Tools/c-analyzer/c_analyzer/symbols/find.py b/Tools/c-analyzer/c_analyzer/symbols/find.py index a0b106fb951730..85646523f7a60b 100644 --- a/Tools/c-analyzer/c_analyzer/symbols/find.py +++ b/Tools/c-analyzer/c_analyzer/symbols/find.py @@ -119,7 +119,7 @@ def symbol(symbol, filenames, known=None, *, _get_resolver=get_resolver, ): """Return a Variable for the one matching the given symbol. - + "symbol" can be one of several objects: * Symbol - use the contained info