Add optional requires_python field to third-party stubs metadata (#10724)

hamdanal · AlexWaygood · web-flow · commit c5dde1e7208c · 2023-09-24T18:39:12.000+01:00
Co-authored-by: Alex Waygood &lt;Alex.Waygood@Gmail.com&gt;
diff --git a/pyproject.toml b/pyproject.toml
@@ -98,3 +98,4 @@ select = [
 
 [tool.typeshed]
 pyright_version = "1.1.328"
+oldest_supported_python = "3.7"
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
@@ -25,8 +25,9 @@
 
 import tomli
 
-from parse_metadata import PackageDependencies, get_recursive_requirements
+from parse_metadata import PackageDependencies, get_recursive_requirements, read_metadata
 from utils import (
+    PYTHON_VERSION,
     VERSIONS_RE as VERSION_LINE_RE,
     VenvInfo,
     colored,
@@ -307,6 +308,7 @@ def add_third_party_files(
 class TestResults(NamedTuple):
     exit_code: int
     files_checked: int
+    packages_skipped: int = 0
 
 
 def test_third_party_distribution(
@@ -393,6 +395,9 @@ def install_requirements_for_venv(venv_info: VenvInfo, args: TestConfig, externa
 
 def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None:
     """Logic necessary for testing stubs with non-types dependencies in isolated environments."""
+    if not distributions:
+        return  # hooray! Nothing to do
+
     # STAGE 1: Determine which (if any) stubs packages require virtual environments.
     # Group stubs packages according to their external-requirements sets
 
@@ -471,6 +476,7 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar
 def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults:
     print("Testing third-party packages...")
     files_checked = 0
+    packages_skipped = 0
     gitignore_spec = get_gitignore_spec()
     distributions_to_check: dict[str, PackageDependencies] = {}
 
@@ -480,47 +486,73 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe
         if spec_matches_path(gitignore_spec, distribution_path):
             continue
 
+        metadata = read_metadata(distribution)
+        if not metadata.requires_python.contains(PYTHON_VERSION):
+            msg = (
+                f"skipping {distribution!r} (requires Python {metadata.requires_python}; "
+                f"test is being run using Python {PYTHON_VERSION})"
+            )
+            print(colored(msg, "yellow"))
+            packages_skipped += 1
+            continue
+        if not metadata.requires_python.contains(args.version):
+            msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})"
+            print(colored(msg, "yellow"))
+            packages_skipped += 1
+            continue
+
         if (
             distribution_path in args.filter
             or Path("stubs") in args.filter
             or any(distribution_path in path.parents for path in args.filter)
         ):
             distributions_to_check[distribution] = get_recursive_requirements(distribution)
 
-    # If it's the first time test_third_party_stubs() has been called during this session,
-    # setup the necessary virtual environments for testing the third-party stubs.
-    # It should only be necessary to call setup_virtual_environments() once per session.
-    if not _DISTRIBUTION_TO_VENV_MAPPING:
-        setup_virtual_environments(distributions_to_check, args, tempdir)
-
-    assert _DISTRIBUTION_TO_VENV_MAPPING.keys() == distributions_to_check.keys()
-
-    for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items():
+    # Setup the necessary virtual environments for testing the third-party stubs.
+    # Note that some stubs may not be tested on all Python versions
+    # (due to version incompatibilities),
+    # so we can't guarantee that setup_virtual_environments()
+    # will only be called once per session.
+    distributions_without_venv = {
+        distribution: requirements
+        for distribution, requirements in distributions_to_check.items()
+        if distribution not in _DISTRIBUTION_TO_VENV_MAPPING
+    }
+    setup_virtual_environments(distributions_without_venv, args, tempdir)
+
+    # Check that there is a venv for every distribution we're testing.
+    # Some venvs may exist from previous runs but are skipped in this run.
+    assert _DISTRIBUTION_TO_VENV_MAPPING.keys() >= distributions_to_check.keys()
+
+    for distribution in distributions_to_check:
+        venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution]
         non_types_dependencies = venv_info.python_exe != sys.executable
-        this_code, checked = test_third_party_distribution(
+        this_code, checked, _ = test_third_party_distribution(
             distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies
         )
         code = max(code, this_code)
         files_checked += checked
 
-    return TestResults(code, files_checked)
+    return TestResults(code, files_checked, packages_skipped)
 
 
 def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults:
     print(f"*** Testing Python {args.version} on {args.platform}")
     files_checked_this_version = 0
+    packages_skipped_this_version = 0
     stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs")
     if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter):
-        code, stdlib_files_checked = test_stdlib(code, args)
+        code, stdlib_files_checked, _ = test_stdlib(code, args)
         files_checked_this_version += stdlib_files_checked
         print()
 
     if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter):
-        code, third_party_files_checked = test_third_party_stubs(code, args, tempdir)
+        code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir)
         files_checked_this_version += third_party_files_checked
+        packages_skipped_this_version = third_party_packages_skipped
         print()
 
-    return TestResults(code, files_checked_this_version)
+    return TestResults(code, files_checked_this_version, packages_skipped_this_version)
 
 
 def main() -> None:
@@ -531,19 +563,27 @@ def main() -> None:
     exclude = args.exclude or []
     code = 0
     total_files_checked = 0
+    total_packages_skipped = 0
     with tempfile.TemporaryDirectory() as td:
         td_path = Path(td)
         for version, platform in product(versions, platforms):
             config = TestConfig(args.verbose, filter, exclude, version, platform)
-            code, files_checked_this_version = test_typeshed(code, args=config, tempdir=td_path)
+            code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path)
             total_files_checked += files_checked_this_version
+            total_packages_skipped += packages_skipped_this_version
     if code:
-        print_error(f"--- exit status {code}, {total_files_checked} files checked ---")
+        plural = "" if total_files_checked == 1 else "s"
+        print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---")
         sys.exit(code)
-    if not total_files_checked:
+    if total_packages_skipped:
+        plural = "" if total_packages_skipped == 1 else "s"
+        print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow"))
+    if total_files_checked:
+        plural = "" if total_files_checked == 1 else "s"
+        print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green"))
+    else:
         print_error("--- nothing to do; exit 1 ---")
         sys.exit(1)
-    print(colored(f"--- success, {total_files_checked} files checked ---", "green"))
 
 
 if __name__ == "__main__":
diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py
@@ -15,6 +15,7 @@
 
 import tomli
 from packaging.requirements import Requirement
+from packaging.specifiers import Specifier
 from packaging.version import Version
 
 from utils import cache
@@ -40,6 +41,14 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]:
     return isinstance(obj, list) and all(isinstance(item, str) for item in obj)
 
 
+@cache
+def _get_oldest_supported_python() -> str:
+    with open("pyproject.toml", "rb") as config:
+        val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"]
+    assert type(val) is str
+    return val
+
+
 @final
 @dataclass(frozen=True)
 class StubtestSettings:
@@ -130,6 +139,7 @@ class StubMetadata:
     uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"]
     partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."]
     stubtest_settings: StubtestSettings
+    requires_python: Annotated[Specifier, "Versions of Python supported by the stub package"]
 
 
 _KNOWN_METADATA_FIELDS: Final = frozenset(
@@ -144,6 +154,7 @@ class StubMetadata:
         "upload",
         "tool",
         "partial_stub",
+        "requires_python",
     }
 )
 _KNOWN_METADATA_TOOL_FIELDS: Final = {
@@ -240,6 +251,20 @@ def read_metadata(distribution: str) -> StubMetadata:
     assert type(uploaded_to_pypi) is bool
     partial_stub: object = data.get("partial_stub", True)
     assert type(partial_stub) is bool
+    requires_python_str: object = data.get("requires_python")
+    oldest_supported_python = _get_oldest_supported_python()
+    oldest_supported_python_specifier = Specifier(f">={oldest_supported_python}")
+    if requires_python_str is None:
+        requires_python = oldest_supported_python_specifier
+    else:
+        assert type(requires_python_str) is str
+        requires_python = Specifier(requires_python_str)
+        assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant'
+        # Check minimum Python version is not less than the oldest version of Python supported by typeshed
+        assert oldest_supported_python_specifier.contains(
+            requires_python.version
+        ), f"'requires_python' contains versions lower than typeshed's oldest supported Python ({oldest_supported_python})"
+        assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'"
 
     empty_tools: dict[object, object] = {}
     tools_settings: object = data.get("tool", empty_tools)
@@ -262,6 +287,7 @@ def read_metadata(distribution: str) -> StubMetadata:
         uploaded_to_pypi=uploaded_to_pypi,
         partial_stub=partial_stub,
         stubtest_settings=read_stubtest_settings(distribution),
+        requires_python=requires_python,
     )
 
 
diff --git a/tests/regr_test.py b/tests/regr_test.py
@@ -13,15 +13,17 @@
 import sys
 import tempfile
 import threading
+from collections.abc import Callable
 from contextlib import ExitStack, suppress
 from dataclasses import dataclass
 from enum import IntEnum
-from itertools import product
+from functools import partial
 from pathlib import Path
 from typing_extensions import TypeAlias
 
-from parse_metadata import get_recursive_requirements
+from parse_metadata import get_recursive_requirements, read_metadata
 from utils import (
+    PYTHON_VERSION,
     PackageInfo,
     VenvInfo,
     colored,
@@ -274,6 +276,30 @@ def concurrently_run_testcases(
     packageinfo_to_tempdir = {
         package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories
     }
+    to_do: list[Callable[[], Result]] = []
+    for testcase_dir, tempdir in packageinfo_to_tempdir.items():
+        pkg = testcase_dir.name
+        requires_python = None
+        if not testcase_dir.is_stdlib:  # type: ignore[misc]  # mypy bug, already fixed on master
+            requires_python = read_metadata(pkg).requires_python
+            if not requires_python.contains(PYTHON_VERSION):
+                msg = f"skipping {pkg!r} (requires Python {requires_python}; test is being run using Python {PYTHON_VERSION})"
+                print(colored(msg, "yellow"))
+                continue
+        for version in versions_to_test:
+            if not testcase_dir.is_stdlib:  # type: ignore[misc]  # mypy bug, already fixed on master
+                assert requires_python is not None
+                if not requires_python.contains(version):
+                    msg = f"skipping {pkg!r} for target Python {version} (requires Python {requires_python})"
+                    print(colored(msg, "yellow"))
+                    continue
+            to_do.extend(
+                partial(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir)
+                for platform in platforms_to_test
+            )
+
+    if not to_do:
+        return []
 
     event = threading.Event()
     printer_thread = threading.Thread(target=print_queued_messages, args=(event,))
@@ -289,12 +315,7 @@ def concurrently_run_testcases(
         ]
         concurrent.futures.wait(testcase_futures)
 
-        mypy_futures = [
-            executor.submit(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir)
-            for (testcase_dir, tempdir), platform, version in product(
-                packageinfo_to_tempdir.items(), platforms_to_test, versions_to_test
-            )
-        ]
+        mypy_futures = [executor.submit(task) for task in to_do]
         results = [future.result() for future in mypy_futures]
 
     event.set()
@@ -315,15 +336,18 @@ def main() -> ReturnCode:
         platforms_to_test, versions_to_test = SUPPORTED_PLATFORMS, SUPPORTED_VERSIONS
     else:
         platforms_to_test = args.platforms_to_test or [sys.platform]
-        versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"]
+        versions_to_test = args.versions_to_test or [PYTHON_VERSION]
 
-    code = 0
     results: list[Result] | None = None
 
     with ExitStack() as stack:
         results = concurrently_run_testcases(stack, testcase_directories, verbosity, platforms_to_test, versions_to_test)
 
     assert results is not None
+    if not results:
+        print_error("All tests were skipped!")
+        return 1
+
     print()
 
     for result in results:
diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py
@@ -13,7 +13,7 @@
 from typing import NoReturn
 
 from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata
-from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg
+from utils import PYTHON_VERSION, colored, get_mypy_req, make_venv, print_error, print_success_msg
 
 
 def run_stubtest(
@@ -37,6 +37,10 @@ def run_stubtest(
             return True
         print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow"))
 
+    if not metadata.requires_python.contains(PYTHON_VERSION):
+        print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow"))
+        return True
+
     with tempfile.TemporaryDirectory() as tmp:
         venv_dir = Path(tmp)
         try:
diff --git a/tests/utils.py b/tests/utils.py
@@ -9,7 +9,7 @@
 import venv
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, NamedTuple
+from typing import Any, Final, NamedTuple
 from typing_extensions import Annotated
 
 import pathspec
@@ -22,6 +22,9 @@ def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type:
         return text
 
 
+PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+
 # A backport of functools.cache for Python <3.9
 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI
 cache = lru_cache(None)

Original file line number	Diff line number	Diff line change
`@@ -98,3 +98,4 @@ select = [`
`98`	`98`
`99`	`99`	`[tool.typeshed]`
`100`	`100`	`pyright_version = "1.1.328"`
	`101`	`+oldest_supported_python = "3.7"`