Refactor and merge requirements parsing (#11581)

srittau · web-flow · commit 417bdb9ac960 · 2024-03-12T16:34:47.000+01:00
diff --git a/tests/check_consistent.py b/tests/check_consistent.py
@@ -17,7 +17,15 @@
 from packaging.specifiers import SpecifierSet
 
 from parse_metadata import read_metadata
-from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments
+from utils import (
+    REQS_FILE,
+    VERSIONS_RE,
+    get_all_testcase_directories,
+    get_gitignore_spec,
+    parse_requirements,
+    spec_matches_path,
+    strip_comments,
+)
 
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
 
@@ -135,13 +143,6 @@ def check_metadata() -> None:
         read_metadata(distribution)
 
 
-def get_txt_requirements() -> dict[str, SpecifierSet]:
-    with open("requirements-tests.txt", encoding="UTF-8") as requirements_file:
-        stripped_lines = map(strip_comments, requirements_file)
-        requirements = map(Requirement, filter(None, stripped_lines))
-        return {requirement.name: requirement.specifier for requirement in requirements}
-
-
 class PreCommitConfigRepos(TypedDict):
     hooks: list[dict[str, str]]
     repo: str
@@ -172,30 +173,30 @@ def get_precommit_requirements() -> dict[str, SpecifierSet]:
 
 def check_requirement_pins() -> None:
     """Check that type checkers and linters are pinned to an exact version."""
-    requirements = get_txt_requirements()
+    requirements = parse_requirements()
     for package in linters:
-        assert package in requirements, f"type checker/linter '{package}' not found in requirements-tests.txt"
-        spec = requirements[package]
-        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in requirements-tests.txt"
-        msg = f"type checker/linter '{package}' is not pinned to an exact version in requirements-tests.txt"
+        assert package in requirements, f"type checker/linter '{package}' not found in {REQS_FILE}"
+        spec = requirements[package].specifier
+        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQS_FILE}"
+        msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQS_FILE}"
         assert str(spec).startswith("=="), msg
 
 
 def check_precommit_requirements() -> None:
-    """Check that the requirements in requirements-tests.txt and .pre-commit-config.yaml match."""
-    requirements_txt_requirements = get_txt_requirements()
+    """Check that the requirements in the requirements file and .pre-commit-config.yaml match."""
+    requirements_txt_requirements = parse_requirements()
     precommit_requirements = get_precommit_requirements()
-    no_txt_entry_msg = "All pre-commit requirements must also be listed in `requirements-tests.txt` (missing {requirement!r})"
+    no_txt_entry_msg = f"All pre-commit requirements must also be listed in `{REQS_FILE}` (missing {{requirement!r}})"
     for requirement, specifier in precommit_requirements.items():
-        # annoying: the Ruff and Black repos for pre-commit are different to the names in requirements-tests.txt
+        # annoying: the Ruff and Black repos for pre-commit are different to the names in the requirements file
         if requirement in {"ruff-pre-commit", "black-pre-commit-mirror"}:
             requirement = requirement.split("-")[0]
         assert requirement in requirements_txt_requirements, no_txt_entry_msg.format(requirement=requirement)
         specifier_mismatch = (
             f'Specifier "{specifier}" for {requirement!r} in `.pre-commit-config.yaml` '
-            f'does not match specifier "{requirements_txt_requirements[requirement]}" in `requirements-tests.txt`'
+            f'does not match specifier "{requirements_txt_requirements[requirement].specifier}" in `{REQS_FILE}`'
         )
-        assert specifier == requirements_txt_requirements[requirement], specifier_mismatch
+        assert specifier == requirements_txt_requirements[requirement].specifier, specifier_mismatch
 
 
 if __name__ == "__main__":
diff --git a/tests/utils.py b/tests/utils.py
@@ -5,11 +5,13 @@
 import os
 import re
 import sys
+from collections.abc import Mapping
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Final, NamedTuple
 
 import pathspec
+from packaging.requirements import Requirement
 
 try:
     from termcolor import colored as colored  # pyright: ignore[reportAssignmentType]
@@ -31,6 +33,11 @@ def strip_comments(text: str) -> str:
     return text.split("#")[0].strip()
 
 
+# ====================================================================
+# Printing utilities
+# ====================================================================
+
+
 def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None:
     error_split = error.split("\n")
     old, new = fix_path
@@ -55,10 +62,26 @@ def venv_python(venv_dir: Path) -> Path:
     return venv_dir / "bin" / "python"
 
 
+# ====================================================================
+# Parsing the requirements file
+# ====================================================================
+
+
+REQS_FILE: Final = "requirements-tests.txt"
+
+
 @cache
+def parse_requirements() -> Mapping[str, Requirement]:
+    """Return a dictionary of requirements from the requirements file."""
+
+    with open(REQS_FILE, encoding="UTF-8") as requirements_file:
+        stripped_lines = map(strip_comments, requirements_file)
+        requirements = map(Requirement, filter(None, stripped_lines))
+        return {requirement.name: requirement for requirement in requirements}
+
+
 def get_mypy_req() -> str:
-    with open("requirements-tests.txt", encoding="UTF-8") as requirements_file:
-        return next(strip_comments(line) for line in requirements_file if "mypy" in line)
+    return str(parse_requirements()["mypy"])
 
 
 # ====================================================================
