From 3651ee7312e145c0731d9e0f446c518d6522b09e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:22:51 -0700 Subject: [PATCH 01/21] Print a helpful message when the globals check fails. --- Tools/c-analyzer/c_analyzer/__main__.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 5d89b29adf899e..59eea487fa7094 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -4,6 +4,7 @@ import os.path import re import sys +import textwrap from c_common import fsutil from c_common.logging import VERBOSITY, Printer @@ -357,6 +358,37 @@ def cmd_check(filenames, *, print(f'subtotal: {len(decls)}') if len(failed) > 0: + print(textwrap.dedent(''' + ------------------------- + + Non-constant global variables are generally not supported + in the CPython repo. We use a tool to analyze the C code + and report if any unsupported globals are found. The tool + may be run manually with: + + ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] + + Occasionally the tool is unable to parse updated code. + If this happens then add the file to the "EXCLUDED" list + in Tools/c-analyzer/cpython/_parser.py and create a new + issue for fixing the tool (and CC ericsnowcurrently + on the issue). + + If the tool reports an unsupported global variable and + it is actually const (and thus supported) then first try + fixing the declaration appropriately in the code. If that + doesn't work then add the variable to the "should be const" + section of Tools/c-analyzer/cpython/_parser.py. + + If the tool otherwise reports an unsupported global variable + then first try to make it non-global, possibly adding to + PyInterpreterState (for core code) or module state (for + extension modules). In an emergency, you can add the + variable to Tools/c-analyzer/cpython/globals-to-fix.tsv + to get CI passing, but doing so should be avoided. If + this course it taken, be sure to create an issue for + eliminating the global (and CC ericsnowcurrently). + ''')) sys.exit(len(failed)) From 1c1e0b0f915a5ea603d7de5588b73a9965d7e7b2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:45:50 -0700 Subject: [PATCH 02/21] Print the explanation even when the parser breaks. --- Tools/c-analyzer/c_analyzer/__main__.py | 131 +++++++++++++----------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 59eea487fa7094..443ad6f5f16e99 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -44,6 +44,39 @@ logger = logging.getLogger(__name__) +EXPLANATION = textwrap.dedent(''' + ------------------------- + + Non-constant global variables are generally not supported + in the CPython repo. We use a tool to analyze the C code + and report if any unsupported globals are found. The tool + may be run manually with: + + ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] + + Occasionally the tool is unable to parse updated code. + If this happens then add the file to the "EXCLUDED" list + in Tools/c-analyzer/cpython/_parser.py and create a new + issue for fixing the tool (and CC ericsnowcurrently + on the issue). + + If the tool reports an unsupported global variable and + it is actually const (and thus supported) then first try + fixing the declaration appropriately in the code. If that + doesn't work then add the variable to the "should be const" + section of Tools/c-analyzer/cpython/_parser.py. + + If the tool otherwise reports an unsupported global variable + then first try to make it non-global, possibly adding to + PyInterpreterState (for core code) or module state (for + extension modules). In an emergency, you can add the + variable to Tools/c-analyzer/cpython/globals-to-fix.tsv + to get CI passing, but doing so should be avoided. If + this course it taken, be sure to create an issue for + eliminating the global (and CC ericsnowcurrently). +''') + + ####################################### # table helpers @@ -324,71 +357,45 @@ def cmd_check(filenames, *, if track_progress: filenames = track_progress(filenames) - logger.info('analyzing files...') - analyzed = _analyze(filenames, **kwargs) - analyzed.fix_filenames(relroot, normalize=False) - decls = filter_forward(analyzed, markpublic=True) - - logger.info('checking analysis results...') - failed = [] - for data, failure in _check_all(decls, checks, failfast=failfast): - if data is None: - printer.info('stopping after one failure') - break - if div is not None and len(failed) > 0: - printer.info(div) - failed.append(data) - handle_failure(failure, data) - handle_after() - - printer.info('-------------------------') - logger.info(f'total failures: {len(failed)}') - logger.info('done checking') - - if fmt == 'summary': - print('Categorized by storage:') - print() - from .match import group_by_storage - grouped = group_by_storage(failed, ignore_non_match=False) - for group, decls in grouped.items(): + try: + logger.info('analyzing files...') + analyzed = _analyze(filenames, **kwargs) + analyzed.fix_filenames(relroot, normalize=False) + decls = filter_forward(analyzed, markpublic=True) + + logger.info('checking analysis results...') + failed = [] + for data, failure in _check_all(decls, checks, failfast=failfast): + if data is None: + printer.info('stopping after one failure') + break + if div is not None and len(failed) > 0: + printer.info(div) + failed.append(data) + handle_failure(failure, data) + handle_after() + + printer.info('-------------------------') + logger.info(f'total failures: {len(failed)}') + logger.info('done checking') + + if fmt == 'summary': + print('Categorized by storage:') print() - print(group) - for decl in decls: - print(' ', _fmt_one_summary(decl)) - print(f'subtotal: {len(decls)}') + from .match import group_by_storage + grouped = group_by_storage(failed, ignore_non_match=False) + for group, decls in grouped.items(): + print() + print(group) + for decl in decls: + print(' ', _fmt_one_summary(decl)) + print(f'subtotal: {len(decls)}') + except Exception: + print(EXPLANATION) + raise # re-raise if len(failed) > 0: - print(textwrap.dedent(''' - ------------------------- - - Non-constant global variables are generally not supported - in the CPython repo. We use a tool to analyze the C code - and report if any unsupported globals are found. The tool - may be run manually with: - - ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] - - Occasionally the tool is unable to parse updated code. - If this happens then add the file to the "EXCLUDED" list - in Tools/c-analyzer/cpython/_parser.py and create a new - issue for fixing the tool (and CC ericsnowcurrently - on the issue). - - If the tool reports an unsupported global variable and - it is actually const (and thus supported) then first try - fixing the declaration appropriately in the code. If that - doesn't work then add the variable to the "should be const" - section of Tools/c-analyzer/cpython/_parser.py. - - If the tool otherwise reports an unsupported global variable - then first try to make it non-global, possibly adding to - PyInterpreterState (for core code) or module state (for - extension modules). In an emergency, you can add the - variable to Tools/c-analyzer/cpython/globals-to-fix.tsv - to get CI passing, but doing so should be avoided. If - this course it taken, be sure to create an issue for - eliminating the global (and CC ericsnowcurrently). - ''')) + print(EXPLANATION) sys.exit(len(failed)) From aa08d9d05e81a06ae25cec9be5707b16d34ab29e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:46:51 -0700 Subject: [PATCH 03/21] Enable the CI globals check. --- Lib/test/test_check_c_globals.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py index 670be52422f799..ea0811da24d61c 100644 --- a/Lib/test/test_check_c_globals.py +++ b/Lib/test/test_check_c_globals.py @@ -1,12 +1,11 @@ +import os import unittest + import test.test_tools +import test.support from test.support.warnings_helper import save_restore_warnings_filters -# TODO: gh-92584: c-analyzer uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - - test.test_tools.skip_if_missing('c-analyzer') with test.test_tools.imports_under_tool('c-analyzer'): # gh-95349: Save/restore warnings filters to leave them unchanged. @@ -19,14 +18,15 @@ class ActualChecks(unittest.TestCase): # XXX Also run the check in "make check". - #@unittest.expectedFailure - # Failing on one of the buildbots (see https://bugs.python.org/issue36876). - @unittest.skip('activate this once all the globals have been resolved') def test_check_c_globals(self): + olddir = os.getcwd() + os.chdir(test.support.REPO_ROOT) try: main('check', {}) except NotImplementedError: raise unittest.SkipTest('not supported on this host') + finally: + os.chdir(olddir) if __name__ == '__main__': From 94db0e4e174225adcf0465a045e458de19920add Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 10:14:41 -0700 Subject: [PATCH 04/21] Add the globals check to CI. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2241b0b8aa409e..c0cede262dda5d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,6 +111,9 @@ jobs: run: make smelly - name: Check limited ABI symbols run: make check-limited-abi + - name: Check for Unsupported C Global Variables + if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME + run: python3 Tools/c-analyzer/check.py --format summary build_win32: name: 'Windows (x86)' From 68db07f9f258d32ef5ff88de6a32671048ab0bfb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 10:15:14 -0700 Subject: [PATCH 05/21] Drop the test. --- Lib/test/test_check_c_globals.py | 34 -------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 Lib/test/test_check_c_globals.py diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py deleted file mode 100644 index ea0811da24d61c..00000000000000 --- a/Lib/test/test_check_c_globals.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import unittest - -import test.test_tools -import test.support -from test.support.warnings_helper import save_restore_warnings_filters - - -test.test_tools.skip_if_missing('c-analyzer') -with test.test_tools.imports_under_tool('c-analyzer'): - # gh-95349: Save/restore warnings filters to leave them unchanged. - # Importing the c-analyzer imports docutils which imports pkg_resources - # which adds a warnings filter. - with save_restore_warnings_filters(): - from cpython.__main__ import main - - -class ActualChecks(unittest.TestCase): - - # XXX Also run the check in "make check". - def test_check_c_globals(self): - olddir = os.getcwd() - os.chdir(test.support.REPO_ROOT) - try: - main('check', {}) - except NotImplementedError: - raise unittest.SkipTest('not supported on this host') - finally: - os.chdir(olddir) - - -if __name__ == '__main__': - # Test needs to be a package, so we can do relative imports. - unittest.main() From 9ed998a7f935ae48657233656cf7028e69bd7a57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 10:55:39 -0700 Subject: [PATCH 06/21] Fix the command. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0cede262dda5d..1dca0b3c1e9b94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: run: make check-limited-abi - name: Check for Unsupported C Global Variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME - run: python3 Tools/c-analyzer/check.py --format summary + run: python3 Tools/c-analyzer/check-c-globals.py --format summary build_win32: name: 'Windows (x86)' From 5ee7611bf7df0626cad464797f312d5e35b208cc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 15:57:34 -0700 Subject: [PATCH 07/21] Add some temporary debugging code. --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 7ef1a8afc3b135..a86ff47fda4b8d 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -82,6 +82,9 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): ]: line = next(lines) if line != expected: + print('--------------------') + print(text) + print('--------------------') raise NotImplementedError((line, expected)) # Do all the CLI-provided includes. From bb1463e5f37741b0828b540c66af2e4abef86756 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 16:16:08 -0700 Subject: [PATCH 08/21] Accommodate newer GCC versions. --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index a86ff47fda4b8d..257e047218376b 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -75,16 +75,17 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): # The first line is special. # The next two lines are consistent. - for expected in [ - f'# 1 "{reqfile}"', - '# 1 ""', - '# 1 ""', - ]: + firstlines = [ + f'# 0 "{reqfile}"', + '# 0 ""', + '# 0 ""', + ] + if text.startswith('# 1 '): + # Some preprocessors emit a lineno of 1 for line-less entries. + firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines] + for expected in firstlines: line = next(lines) if line != expected: - print('--------------------') - print(text) - print('--------------------') raise NotImplementedError((line, expected)) # Do all the CLI-provided includes. From d9f45a8ff317eb2b31fcccab74e564c78a2a82d4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 16:22:40 -0700 Subject: [PATCH 09/21] Fix the "ignored" filename. --- Tools/c-analyzer/c_analyzer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 443ad6f5f16e99..0436a1c77a99de 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -64,7 +64,7 @@ it is actually const (and thus supported) then first try fixing the declaration appropriately in the code. If that doesn't work then add the variable to the "should be const" - section of Tools/c-analyzer/cpython/_parser.py. + section of Tools/c-analyzer/cpython/ignored.tsv. If the tool otherwise reports an unsupported global variable then first try to make it non-global, possibly adding to From f1e5827d4b8464b47f2e985de37677676e18e605 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 16:33:43 -0700 Subject: [PATCH 10/21] Emit the message from the right script. --- Tools/c-analyzer/c_analyzer/__main__.py | 101 ++++++++---------------- Tools/c-analyzer/cpython/__main__.py | 60 ++++++++++++-- 2 files changed, 83 insertions(+), 78 deletions(-) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 0436a1c77a99de..5d89b29adf899e 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -4,7 +4,6 @@ import os.path import re import sys -import textwrap from c_common import fsutil from c_common.logging import VERBOSITY, Printer @@ -44,39 +43,6 @@ logger = logging.getLogger(__name__) -EXPLANATION = textwrap.dedent(''' - ------------------------- - - Non-constant global variables are generally not supported - in the CPython repo. We use a tool to analyze the C code - and report if any unsupported globals are found. The tool - may be run manually with: - - ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] - - Occasionally the tool is unable to parse updated code. - If this happens then add the file to the "EXCLUDED" list - in Tools/c-analyzer/cpython/_parser.py and create a new - issue for fixing the tool (and CC ericsnowcurrently - on the issue). - - If the tool reports an unsupported global variable and - it is actually const (and thus supported) then first try - fixing the declaration appropriately in the code. If that - doesn't work then add the variable to the "should be const" - section of Tools/c-analyzer/cpython/ignored.tsv. - - If the tool otherwise reports an unsupported global variable - then first try to make it non-global, possibly adding to - PyInterpreterState (for core code) or module state (for - extension modules). In an emergency, you can add the - variable to Tools/c-analyzer/cpython/globals-to-fix.tsv - to get CI passing, but doing so should be avoided. If - this course it taken, be sure to create an issue for - eliminating the global (and CC ericsnowcurrently). -''') - - ####################################### # table helpers @@ -357,45 +323,40 @@ def cmd_check(filenames, *, if track_progress: filenames = track_progress(filenames) - try: - logger.info('analyzing files...') - analyzed = _analyze(filenames, **kwargs) - analyzed.fix_filenames(relroot, normalize=False) - decls = filter_forward(analyzed, markpublic=True) - - logger.info('checking analysis results...') - failed = [] - for data, failure in _check_all(decls, checks, failfast=failfast): - if data is None: - printer.info('stopping after one failure') - break - if div is not None and len(failed) > 0: - printer.info(div) - failed.append(data) - handle_failure(failure, data) - handle_after() - - printer.info('-------------------------') - logger.info(f'total failures: {len(failed)}') - logger.info('done checking') - - if fmt == 'summary': - print('Categorized by storage:') + logger.info('analyzing files...') + analyzed = _analyze(filenames, **kwargs) + analyzed.fix_filenames(relroot, normalize=False) + decls = filter_forward(analyzed, markpublic=True) + + logger.info('checking analysis results...') + failed = [] + for data, failure in _check_all(decls, checks, failfast=failfast): + if data is None: + printer.info('stopping after one failure') + break + if div is not None and len(failed) > 0: + printer.info(div) + failed.append(data) + handle_failure(failure, data) + handle_after() + + printer.info('-------------------------') + logger.info(f'total failures: {len(failed)}') + logger.info('done checking') + + if fmt == 'summary': + print('Categorized by storage:') + print() + from .match import group_by_storage + grouped = group_by_storage(failed, ignore_non_match=False) + for group, decls in grouped.items(): print() - from .match import group_by_storage - grouped = group_by_storage(failed, ignore_non_match=False) - for group, decls in grouped.items(): - print() - print(group) - for decl in decls: - print(' ', _fmt_one_summary(decl)) - print(f'subtotal: {len(decls)}') - except Exception: - print(EXPLANATION) - raise # re-raise + print(group) + for decl in decls: + print(' ', _fmt_one_summary(decl)) + print(f'subtotal: {len(decls)}') if len(failed) > 0: - print(EXPLANATION) sys.exit(len(failed)) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 2b9e4233b95ac4..f9189c5818f8e4 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -1,5 +1,6 @@ import logging import sys +import textwrap from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_common.scriptutil import ( @@ -26,6 +27,39 @@ logger = logging.getLogger(__name__) +CHECK_EXPLANATION = textwrap.dedent(''' + ------------------------- + + Non-constant global variables are generally not supported + in the CPython repo. We use a tool to analyze the C code + and report if any unsupported globals are found. The tool + may be run manually with: + + ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] + + Occasionally the tool is unable to parse updated code. + If this happens then add the file to the "EXCLUDED" list + in Tools/c-analyzer/cpython/_parser.py and create a new + issue for fixing the tool (and CC ericsnowcurrently + on the issue). + + If the tool reports an unsupported global variable and + it is actually const (and thus supported) then first try + fixing the declaration appropriately in the code. If that + doesn't work then add the variable to the "should be const" + section of Tools/c-analyzer/cpython/ignored.tsv. + + If the tool otherwise reports an unsupported global variable + then first try to make it non-global, possibly adding to + PyInterpreterState (for core code) or module state (for + extension modules). In an emergency, you can add the + variable to Tools/c-analyzer/cpython/globals-to-fix.tsv + to get CI passing, but doing so should be avoided. If + this course it taken, be sure to create an issue for + eliminating the global (and CC ericsnowcurrently). +''') + + def _resolve_filenames(filenames): if filenames: resolved = (_files.resolve_filename(f) for f in filenames) @@ -123,14 +157,24 @@ def _cli_check(parser, **kwargs): def cmd_check(filenames=None, **kwargs): filenames = _resolve_filenames(filenames) kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) - c_analyzer.cmd_check( - filenames, - relroot=REPO_ROOT, - _analyze=_analyzer.analyze, - _CHECKS=CHECKS, - file_maxsizes=_parser.MAX_SIZES, - **kwargs - ) + try: + c_analyzer.cmd_check( + filenames, + relroot=REPO_ROOT, + _analyze=_analyzer.analyze, + _CHECKS=CHECKS, + file_maxsizes=_parser.MAX_SIZES, + **kwargs + ) + except SystemExit as exc: + num_failed = exc.args[0] if getattr(exc, 'args', None) else None + if isinstance(num_failed, int): + if num_failed > 0: + print(CHECK_EXPLANATION) + raise # re-raise + except Exception: + print(CHECK_EXPLANATION) + raise # re-raise def cmd_analyze(filenames=None, **kwargs): From d59e2af41f29c3d6a4b1828fa00be6baac256ca3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 16:35:44 -0700 Subject: [PATCH 11/21] Print the traceback. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1dca0b3c1e9b94..4ebdc78884614d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: run: make check-limited-abi - name: Check for Unsupported C Global Variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME - run: python3 Tools/c-analyzer/check-c-globals.py --format summary + run: python3 Tools/c-analyzer/check-c-globals.py --format summary --traceback build_win32: name: 'Windows (x86)' From a959ccd7dbf4b4b2802c2d19339d56f039907e3e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 16:53:01 -0700 Subject: [PATCH 12/21] Let lineno be 0 for meta files. --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 257e047218376b..7a893682da73a2 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -6,6 +6,11 @@ TOOL = 'gcc' +META_FILES = { + '', + '', +} + # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html # flags: # 1 start of a new file @@ -178,7 +183,8 @@ def _parse_marker_line(line, reqfile=None): return None, None, None lno, origfile, flags = m.groups() lno = int(lno) - assert lno > 0, (line, lno) + assert (lno in (0, 1) if origfile in META_FILES else lno > 0), (line, lno) + assert lno > 0 or (lno == 0 and origfile in META_FILES), (line, lno) assert origfile not in ('', ''), (line,) flags = set(int(f) for f in flags.split()) if flags else () From 33900143edb2c6890a31bc9361daff04c45675a2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 17:01:54 -0700 Subject: [PATCH 13/21] Allow meta files with flags. --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 7a893682da73a2..68ef80c562d441 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -184,8 +184,7 @@ def _parse_marker_line(line, reqfile=None): lno, origfile, flags = m.groups() lno = int(lno) assert (lno in (0, 1) if origfile in META_FILES else lno > 0), (line, lno) - assert lno > 0 or (lno == 0 and origfile in META_FILES), (line, lno) - assert origfile not in ('', ''), (line,) + assert flags or origfile not in META_FILES, (line,) flags = set(int(f) for f in flags.split()) if flags else () if 1 in flags: From 8f935509aa433e894c00b471e02a66e9ac0fc8bc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Mar 2023 17:11:45 -0700 Subject: [PATCH 14/21] Special-case . --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 68ef80c562d441..24c1b0e9b9d48c 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -130,7 +130,7 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 1 "" 2': + if line == '# 0 "" 2' or line == '# 1 "" 2': # We're done with this top-level include. return @@ -183,8 +183,8 @@ def _parse_marker_line(line, reqfile=None): return None, None, None lno, origfile, flags = m.groups() lno = int(lno) - assert (lno in (0, 1) if origfile in META_FILES else lno > 0), (line, lno) - assert flags or origfile not in META_FILES, (line,) + assert origfile not in META_FILES, (line,) + assert lno > 0, (line, lno) flags = set(int(f) for f in flags.split()) if flags else () if 1 in flags: From 76039c08d3b8f0efe25e0c71967069c712dd9eb4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 09:30:30 -0700 Subject: [PATCH 15/21] Flush stdout/stderr around the explanation. --- Tools/c-analyzer/cpython/__main__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index f9189c5818f8e4..fe7a16726f45a9 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -170,10 +170,12 @@ def cmd_check(filenames=None, **kwargs): num_failed = exc.args[0] if getattr(exc, 'args', None) else None if isinstance(num_failed, int): if num_failed > 0: - print(CHECK_EXPLANATION) + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) raise # re-raise except Exception: - print(CHECK_EXPLANATION) + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) raise # re-raise From dc4ee3ab27d42dc9f8b0ac07a9d4ef8349547f37 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 09:39:06 -0700 Subject: [PATCH 16/21] Print preprocessor errors by default. --- Tools/c-analyzer/c_parser/preprocessor/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/common.py b/Tools/c-analyzer/c_parser/preprocessor/common.py index 4291a066337545..dbe1edeef38527 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/common.py +++ b/Tools/c-analyzer/c_parser/preprocessor/common.py @@ -115,15 +115,15 @@ def converted_error(tool, argv, filename): def convert_error(tool, argv, filename, stderr, rc): error = (stderr.splitlines()[0], rc) if (_expected := is_os_mismatch(filename, stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise OSMismatchError(filename, _expected, argv, error, tool) elif (_missing := is_missing_dep(stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise MissingDependenciesError(filename, (_missing,), argv, error, tool) elif '#error' in stderr: # XXX Ignore incompatible files. error = (stderr.splitlines()[1], rc) - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise ErrorDirectiveError(filename, argv, error, tool) else: # Try one more time, with stderr written to the terminal. From 4c03fd04adeb3d1ab5cad12e3aad7ced70b22fe8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 09:46:36 -0700 Subject: [PATCH 17/21] Add a possible include dir for uuid. --- Tools/c-analyzer/cpython/_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index e7764165d36c4c..125e9b819ecbc2 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -109,6 +109,7 @@ def clean_lines(text): Modules/_hacl/*.c Modules/_hacl/include Modules/_hacl/*.h Modules/_hacl/include Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include From 863e1526af4592010906944f420128ddedf7b44e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 10:05:52 -0700 Subject: [PATCH 18/21] Add other possible include dirs. --- Tools/c-analyzer/cpython/_parser.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 125e9b819ecbc2..6da4fbbf4224f1 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -106,16 +106,20 @@ def clean_lines(text): * ./Include/internal Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_elementtree.c Modules/expat Modules/_hacl/*.c Modules/_hacl/include Modules/_hacl/*.h Modules/_hacl/include -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/_uuidmodule.c /usr/include/uuid Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include -Modules/tkappinit.c /usr/include/tcl Objects/stringlib/*.h Objects +# possible system-installed headers, just in case +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid +Modules/nismodule.c /usr/include/tirpc +Modules/tkappinit.c /usr/include/tcl + # @end=tsv@ ''')[1:] From f230d79fa5d80f0429e9c9e366f1d3a1c577cea4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 08:13:11 -0600 Subject: [PATCH 19/21] Use Same Capitalization as Other Jobs Co-authored-by: C.A.M. Gerlach --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ebdc78884614d..3cabc345374449 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,7 +111,7 @@ jobs: run: make smelly - name: Check limited ABI symbols run: make check-limited-abi - - name: Check for Unsupported C Global Variables + - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: python3 Tools/c-analyzer/check-c-globals.py --format summary --traceback From 08f92795adeafb55484b8966333503fe2c1b7236 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 08:25:59 -0600 Subject: [PATCH 20/21] Add a make target. --- .github/workflows/build.yml | 2 +- Makefile.pre.in | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3cabc345374449..4e5328282f1224 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: run: make check-limited-abi - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME - run: python3 Tools/c-analyzer/check-c-globals.py --format summary --traceback + run: make check-c-globals build_win32: name: 'Windows (x86)' diff --git a/Makefile.pre.in b/Makefile.pre.in index 1a1853bf3d7871..59762165c10036 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2560,6 +2560,12 @@ distclean: clobber docclean smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py +# Check if any unsupported C global variables have been added. +check-c-globals: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/c-analyzer/check-c-globals.py \ + --format summary \ + --traceback + # Find files with funny names funny: find $(SUBDIRS) $(SUBDIRSTOO) \ From 9cc170c1e86d0ca2f3a9d1701aa955558d6bee07 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 09:21:47 -0600 Subject: [PATCH 21/21] Ignore a global that snuck in. --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 700ddf2851839e..048112dd992555 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -228,6 +228,7 @@ Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - +Modules/_xxsubinterpretersmodule.c - no_exception - Modules/arraymodule.c - descriptors - Modules/arraymodule.c - emptybuf - Modules/cjkcodecs/_codecs_cn.c - _mapping_list -