[3.11] gh-115421: List all test/ subdirs in Makefile, and test them (GH-115813)

encukou · sobolevn · web-flow · commit d78117981a15 · 2024-03-11T12:02:05.000+01:00
This backports: - GH-115813 - GH-115422 Unlike on the main branch, new directories are added to the end, so they're a bit easier to patch out if a redistributor needs to do so. On main & 3.12, there's a special case for `idlelib/idle_test`; on 3.11 TESTSUBDIRS has several more entries that are not in `test/`. This backport ignores all of them (including idlelib). (The alternative would be list them, as additions to TEST_HOME_DIR. But that's probably too invasive; people might split stdlib up in surprising ways.) Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py
@@ -0,0 +1,67 @@
+"""
+Tests for `Makefile`.
+"""
+
+import os
+import unittest
+from test import support
+import sysconfig
+
+MAKEFILE = sysconfig.get_makefile_filename()
+
+if not support.check_impl_detail(cpython=True):
+    raise unittest.SkipTest('cpython only')
+if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE):
+    raise unittest.SkipTest('Makefile could not be found')
+
+
+class TestMakefile(unittest.TestCase):
+    def list_test_dirs(self):
+        result = []
+        found_testsubdirs = False
+        with open(MAKEFILE, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.startswith('TESTSUBDIRS='):
+                    found_testsubdirs = True
+                    result.append(
+                        line.removeprefix('TESTSUBDIRS=').replace(
+                            '\\', '',
+                        ).strip(),
+                    )
+                    continue
+                if found_testsubdirs:
+                    if '\t' not in line:
+                        break
+                    result.append(line.replace('\\', '').strip())
+
+        # In Python 3.11 (and lower), many test modules are not in
+        # the tests/ directory. This check ignores them.
+        result = [d for d in result if d.startswith('test/') or d == 'test']
+
+        return result
+
+    def test_makefile_test_folders(self):
+        test_dirs = self.list_test_dirs()
+
+        used = []
+        for dirpath, _, _ in os.walk(support.TEST_HOME_DIR):
+            dirname = os.path.basename(dirpath)
+            if dirname == '__pycache__':
+                continue
+
+            relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
+            with self.subTest(relpath=relpath):
+                self.assertIn(
+                    relpath,
+                    test_dirs,
+                    msg=(
+                        f"{relpath!r} is not included in the Makefile's list "
+                        "of test directories to install"
+                    )
+                )
+                used.append(relpath)
+
+        # Check that there are no extra entries:
+        unique_test_dirs = set(test_dirs)
+        self.assertSetEqual(unique_test_dirs, set(used))
+        self.assertEqual(len(test_dirs), len(unique_test_dirs))
diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -2065,7 +2065,11 @@ TESTSUBDIRS=	ctypes/test \
 		tkinter/test/test_tkinter \
 		tkinter/test/test_ttk \
 		unittest/test \
-		unittest/test/testmock
+		unittest/test/testmock \
+		test/test_concurrent_futures \
+		test/test_multiprocessing_fork \
+		test/test_multiprocessing_forkserver \
+		test/test_multiprocessing_spawn
 
 TEST_MODULES=@TEST_MODULES@
 libinstall:	all $(srcdir)/Modules/xxmodule.c
