From b04014fbb8e054eaf942e95c76ff7bb0588d15c0 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 12:37:40 +0300 Subject: [PATCH 1/9] gh-115421: Test that our Makefile has all needed test folders --- Lib/test/test_tools/test_makefile.py | 66 ++++++++++++++++++++++++++++ Makefile.pre.in | 6 +++ 2 files changed, 72 insertions(+) create mode 100644 Lib/test/test_tools/test_makefile.py diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 00000000000000..5cfd2b23f3737d --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,66 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support + +MAKEFILE = os.path.join(support.REPO_ROOT, 'Makefile') + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if support.MS_WINDOWS: + raise unittest.SkipTest('Windows does not support make') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + +try: + import _testcapi +except ImportError: + TEST_MODULES = False +else: + TEST_MODULES = True + +@support.requires_subprocess() +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + import subprocess + return subprocess.check_output( + ['make', '-C', support.REPO_ROOT, 'listtestmodules'], + universal_newlines=True, + encoding='utf-8', + ) + + def check_existing_test_modules(self, result): + test_dirs = result.split(' ') + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = [idle_test] + for dirpath, _, _ in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + if dirname == '__pycache__': + continue + with self.subTest(dirpath=dirpath): + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + self.assertIn(relpath, test_dirs) + used.append(relpath) + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertEqual( + unique_test_dirs.symmetric_difference(set(used)), + set(), + ) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) + + def check_missing_test_modules(self, result): + self.assertEqual(result, '') + + def test_makefile_test_folders(self): + result = self.list_test_dirs().strip() + if TEST_MODULES: + self.check_existing_test_modules(result) + else: + self.check_missing_test_modules(result) diff --git a/Makefile.pre.in b/Makefile.pre.in index 11d22d9a419ba7..71a89bb6966a33 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2373,6 +2373,12 @@ COMPILEALL_OPTS=-j0 TEST_MODULES=@TEST_MODULES@ +.PHONY: listtestmodules +listtestmodules: + @if test "$(TEST_MODULES)" = yes; then \ + echo "$(TESTSUBDIRS)"; \ + fi + .PHONY: libinstall libinstall: all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ From 5393647e95b0fba927ab8e68abca4fc431705949 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 13:47:48 +0300 Subject: [PATCH 2/9] Be silent --- Lib/test/test_tools/test_makefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 5cfd2b23f3737d..552dc79f50056a 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -27,7 +27,7 @@ class TestMakefile(unittest.TestCase): def list_test_dirs(self): import subprocess return subprocess.check_output( - ['make', '-C', support.REPO_ROOT, 'listtestmodules'], + ['make', '-s', '-C', support.REPO_ROOT, 'listtestmodules'], universal_newlines=True, encoding='utf-8', ) From f048fb509702b142d14938d73e47d51e63d27bfe Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 15:33:22 +0300 Subject: [PATCH 3/9] Test PR that a missing `TESTSUBDIR` will fail --- Makefile.pre.in | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 71a89bb6966a33..1d35017113b164 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2237,7 +2237,6 @@ LIBSUBDIRS= asyncio \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ - test/archivetestdata \ test/audiodata \ test/certdata \ test/certdata/capath \ From f6c86396b835e5ff2de2762bd5c5698927d7617f Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 16:59:13 +0300 Subject: [PATCH 4/9] Revert `test/archivetestdata` change --- Lib/test/test_tools/test_makefile.py | 21 ++++++++++++++++++--- Makefile.pre.in | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 552dc79f50056a..b3c7a0ea8ae403 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -6,6 +6,7 @@ import unittest from test import support +SPLITTER = '@-----@' MAKEFILE = os.path.join(support.REPO_ROOT, 'Makefile') if not support.check_impl_detail(cpython=True): @@ -22,6 +23,7 @@ else: TEST_MODULES = True + @support.requires_subprocess() class TestMakefile(unittest.TestCase): def list_test_dirs(self): @@ -44,7 +46,14 @@ def check_existing_test_modules(self, result): continue with self.subTest(dirpath=dirpath): relpath = os.path.relpath(dirpath, support.STDLIB_DIR) - self.assertIn(relpath, test_dirs) + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) used.append(relpath) # Check that there are no extra entries: @@ -60,7 +69,13 @@ def check_missing_test_modules(self, result): def test_makefile_test_folders(self): result = self.list_test_dirs().strip() + self.assertEqual( + result.count(SPLITTER), 1, + msg=f'{SPLITTER} is contained in result multiple times', + ) + _, actual_result = result.split(SPLITTER) + actual_result = actual_result.strip() if TEST_MODULES: - self.check_existing_test_modules(result) + self.check_existing_test_modules(actual_result) else: - self.check_missing_test_modules(result) + self.check_missing_test_modules(actual_result) diff --git a/Makefile.pre.in b/Makefile.pre.in index 1d35017113b164..7bb465baf698fa 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2237,6 +2237,7 @@ LIBSUBDIRS= asyncio \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ + test/archivetestdata \ test/audiodata \ test/certdata \ test/certdata/capath \ @@ -2374,6 +2375,7 @@ TEST_MODULES=@TEST_MODULES@ .PHONY: listtestmodules listtestmodules: + @echo "@-----@" @if test "$(TEST_MODULES)" = yes; then \ echo "$(TESTSUBDIRS)"; \ fi From 182822dd1f8c1eb36ca747a3690f5207cab27f80 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 17:02:16 +0300 Subject: [PATCH 5/9] Better wording --- Lib/test/test_tools/test_makefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index b3c7a0ea8ae403..27dbc36a0b221a 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -71,7 +71,7 @@ def test_makefile_test_folders(self): result = self.list_test_dirs().strip() self.assertEqual( result.count(SPLITTER), 1, - msg=f'{SPLITTER} is contained in result multiple times', + msg=f'{SPLITTER} should be contained in the output exactly once', ) _, actual_result = result.split(SPLITTER) actual_result = actual_result.strip() From d192fbe74826f5b267c002dd65f793dac6ffb460 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 22 Feb 2024 20:13:36 +0300 Subject: [PATCH 6/9] Fix buildbot failures --- Lib/test/test_tools/test_makefile.py | 7 ++++--- Makefile.pre.in | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 27dbc36a0b221a..91282dedb53217 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -35,7 +35,7 @@ def list_test_dirs(self): ) def check_existing_test_modules(self, result): - test_dirs = result.split(' ') + test_dirs = list(filter(None, result.split(' '))) idle_test = 'idlelib/idle_test' self.assertIn(idle_test, test_dirs) @@ -44,8 +44,9 @@ def check_existing_test_modules(self, result): dirname = os.path.basename(dirpath) if dirname == '__pycache__': continue - with self.subTest(dirpath=dirpath): - relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): self.assertIn( relpath, test_dirs, diff --git a/Makefile.pre.in b/Makefile.pre.in index 7bb465baf698fa..935901af3219f0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2373,6 +2373,7 @@ COMPILEALL_OPTS=-j0 TEST_MODULES=@TEST_MODULES@ +# This is needed for `test_makefile`: .PHONY: listtestmodules listtestmodules: @echo "@-----@" From a743ffec57b1c9d4ff5fef1265479a7ff1cbffaa Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 25 Feb 2024 11:00:08 +0300 Subject: [PATCH 7/9] Do not run `make`, read file by lines --- Lib/test/test_tools/test_makefile.py | 49 ++++++++-------------------- Makefile.pre.in | 11 ++----- 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 91282dedb53217..eae1c898f06b41 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -6,36 +6,31 @@ import unittest from test import support -SPLITTER = '@-----@' MAKEFILE = os.path.join(support.REPO_ROOT, 'Makefile') if not support.check_impl_detail(cpython=True): raise unittest.SkipTest('cpython only') -if support.MS_WINDOWS: - raise unittest.SkipTest('Windows does not support make') if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): raise unittest.SkipTest('Makefile could not be found') -try: - import _testcapi -except ImportError: - TEST_MODULES = False -else: - TEST_MODULES = True - -@support.requires_subprocess() class TestMakefile(unittest.TestCase): def list_test_dirs(self): - import subprocess - return subprocess.check_output( - ['make', '-s', '-C', support.REPO_ROOT, 'listtestmodules'], - universal_newlines=True, - encoding='utf-8', - ) + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result - def check_existing_test_modules(self, result): - test_dirs = list(filter(None, result.split(' '))) + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() idle_test = 'idlelib/idle_test' self.assertIn(idle_test, test_dirs) @@ -64,19 +59,3 @@ def check_existing_test_modules(self, result): set(), ) self.assertEqual(len(test_dirs), len(unique_test_dirs)) - - def check_missing_test_modules(self, result): - self.assertEqual(result, '') - - def test_makefile_test_folders(self): - result = self.list_test_dirs().strip() - self.assertEqual( - result.count(SPLITTER), 1, - msg=f'{SPLITTER} should be contained in the output exactly once', - ) - _, actual_result = result.split(SPLITTER) - actual_result = actual_result.strip() - if TEST_MODULES: - self.check_existing_test_modules(actual_result) - else: - self.check_missing_test_modules(actual_result) diff --git a/Makefile.pre.in b/Makefile.pre.in index f0fb6fcfbe5210..78171a2411fa98 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2235,7 +2235,8 @@ LIBSUBDIRS= asyncio \ zipfile zipfile/_path \ zoneinfo \ __phello__ -TESTSUBDIRS= idlelib/idle_test \ +TESTSUBDIRS=\ + idlelib/idle_test \ test \ test/archivetestdata \ test/audiodata \ @@ -2373,14 +2374,6 @@ COMPILEALL_OPTS=-j0 TEST_MODULES=@TEST_MODULES@ -# This is needed for `test_makefile`: -.PHONY: listtestmodules -listtestmodules: - @echo "@-----@" - @if test "$(TEST_MODULES)" = yes; then \ - echo "$(TESTSUBDIRS)"; \ - fi - .PHONY: libinstall libinstall: all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ From 001c14b58ba20945b5f2bb596c0c050b7dd295e0 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 25 Feb 2024 11:02:22 +0300 Subject: [PATCH 8/9] Do not touch makefile at all --- Lib/test/test_tools/test_makefile.py | 5 +++++ Makefile.pre.in | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index eae1c898f06b41..c4ade76009a978 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -22,6 +22,11 @@ def list_test_dirs(self): for line in f: if line.startswith('TESTSUBDIRS='): found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) continue if found_testsubdirs: if '\t' not in line: diff --git a/Makefile.pre.in b/Makefile.pre.in index 78171a2411fa98..4c1a18602b2d0b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2235,8 +2235,7 @@ LIBSUBDIRS= asyncio \ zipfile zipfile/_path \ zoneinfo \ __phello__ -TESTSUBDIRS=\ - idlelib/idle_test \ +TESTSUBDIRS= idlelib/idle_test \ test \ test/archivetestdata \ test/audiodata \ From 6caa8f9342a12b89054234ecb7f327306ef67dc5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 4 Mar 2024 16:47:34 +0300 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Petr Viktorin --- Lib/test/test_tools/test_makefile.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index c4ade76009a978..7222a054dcd61c 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -5,8 +5,9 @@ import os import unittest from test import support +import sysconfig -MAKEFILE = os.path.join(support.REPO_ROOT, 'Makefile') +MAKEFILE = sysconfig.get_makefile_filename() if not support.check_impl_detail(cpython=True): raise unittest.SkipTest('cpython only') @@ -59,8 +60,5 @@ def test_makefile_test_folders(self): # Check that there are no extra entries: unique_test_dirs = set(test_dirs) - self.assertEqual( - unique_test_dirs.symmetric_difference(set(used)), - set(), - ) + self.assertSetEqual(unique_test_dirs, set(used)) self.assertEqual(len(test_dirs), len(unique_test_dirs))