Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 6a6d2e8

Browse files
ikonstikonst
authored
meta tests: refactor run_pytest (#15481)
Factor `run_pytest` out of mypy/test/meta/test_*.py.
1 parent fb32db7 commit 6a6d2e8

3 files changed

Lines changed: 104 additions & 86 deletions

File tree

‎mypy/test/meta/_pytest.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import shlex
2+
import subprocess
3+
import sys
4+
import textwrap
5+
import uuid
6+
from dataclasses import dataclass
7+
from pathlib import Path
8+
from typing import Iterable
9+
10+
from mypy.test.config import test_data_prefix
11+
12+
13+
@dataclass
14+
class PytestResult:
15+
input: str
16+
input_updated: str # any updates made by --update-data
17+
stdout: str
18+
stderr: str
19+
20+
21+
def dedent_docstring(s: str) -> str:
22+
return textwrap.dedent(s).lstrip()
23+
24+
25+
def run_pytest_data_suite(
26+
data_suite: str,
27+
*,
28+
data_file_prefix: str = "check",
29+
pytest_node_prefix: str = "mypy/test/testcheck.py::TypeCheckSuite",
30+
extra_args: Iterable[str],
31+
max_attempts: int,
32+
) -> PytestResult:
33+
"""
34+
Runs a suite of data test cases through pytest until either tests pass
35+
or until a maximum number of attempts (needed for incremental tests).
36+
37+
:param data_suite: the actual "suite" i.e. the contents of a .test file
38+
"""
39+
p_test_data = Path(test_data_prefix)
40+
p_root = p_test_data.parent.parent
41+
p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test"
42+
assert not p.exists()
43+
data_suite = dedent_docstring(data_suite)
44+
try:
45+
p.write_text(data_suite)
46+
47+
test_nodeid = f"{pytest_node_prefix}::{p.name}"
48+
extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid]
49+
cmd = shlex.join(extra_args)
50+
for i in range(max_attempts - 1, -1, -1):
51+
print(f">> {cmd}")
52+
proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root)
53+
if proc.returncode == 0:
54+
break
55+
prefix = "NESTED PYTEST STDOUT"
56+
for line in proc.stdout.decode().splitlines():
57+
print(f"{prefix}: {line}")
58+
prefix = " " * len(prefix)
59+
prefix = "NESTED PYTEST STDERR"
60+
for line in proc.stderr.decode().splitlines():
61+
print(f"{prefix}: {line}")
62+
prefix = " " * len(prefix)
63+
print(f"Exit code {proc.returncode} ({i} attempts remaining)")
64+
65+
return PytestResult(
66+
input=data_suite,
67+
input_updated=p.read_text(),
68+
stdout=proc.stdout.decode(),
69+
stderr=proc.stderr.decode(),
70+
)
71+
finally:
72+
p.unlink()

‎mypy/test/meta/test_parse_data.py‎

Lines changed: 17 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2,37 +2,18 @@
22
A "meta test" which tests the parsing of .test files. This is not meant to become exhaustive
33
but to ensure we maintain a basic level of ergonomics for mypy contributors.
44
"""
5-
import subprocess
6-
import sys
7-
import textwrap
8-
import uuid
9-
from pathlib import Path
10-
11-
from mypy.test.config import test_data_prefix
125
from mypy.test.helpers import Suite
6+
from mypy.test.meta._pytest import PytestResult, run_pytest_data_suite
137

148

15-
class ParseTestDataSuite(Suite):
16-
def _dedent(self, s: str) -> str:
17-
return textwrap.dedent(s).lstrip()
9+
def _run_pytest(data_suite: str) -> PytestResult:
10+
return run_pytest_data_suite(data_suite, extra_args=[], max_attempts=1)
1811

19-
def _run_pytest(self, data_suite: str) -> str:
20-
p_test_data = Path(test_data_prefix)
21-
p_root = p_test_data.parent.parent
22-
p = p_test_data / f"check-meta-{uuid.uuid4()}.test"
23-
assert not p.exists()
24-
try:
25-
p.write_text(data_suite)
26-
test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
27-
args = [sys.executable, "-m", "pytest", "-n", "0", "-s", test_nodeid]
28-
proc = subprocess.run(args, cwd=p_root, capture_output=True, check=False)
29-
return proc.stdout.decode()
30-
finally:
31-
p.unlink()
3212

13+
class ParseTestDataSuite(Suite):
3314
def test_parse_invalid_case(self) -> None:
34-
# Arrange
35-
data = self._dedent(
15+
# Act
16+
result = _run_pytest(
3617
"""
3718
[case abc]
3819
s: str
@@ -41,15 +22,12 @@ def test_parse_invalid_case(self) -> None:
4122
"""
4223
)
4324

44-
# Act
45-
actual = self._run_pytest(data)
46-
4725
# Assert
48-
assert "Invalid testcase id 'foo-XFAIL'" in actual
26+
assert "Invalid testcase id 'foo-XFAIL'" in result.stdout
4927

5028
def test_parse_invalid_section(self) -> None:
51-
# Arrange
52-
data = self._dedent(
29+
# Act
30+
result = _run_pytest(
5331
"""
5432
[case abc]
5533
s: str
@@ -58,19 +36,16 @@ def test_parse_invalid_section(self) -> None:
5836
"""
5937
)
6038

61-
# Act
62-
actual = self._run_pytest(data)
63-
6439
# Assert
65-
expected_lineno = data.splitlines().index("[unknownsection]") + 1
40+
expected_lineno = result.input.splitlines().index("[unknownsection]") + 1
6641
expected = (
6742
f".test:{expected_lineno}: Invalid section header [unknownsection] in case 'abc'"
6843
)
69-
assert expected in actual
44+
assert expected in result.stdout
7045

7146
def test_bad_ge_version_check(self) -> None:
72-
# Arrange
73-
data = self._dedent(
47+
# Act
48+
actual = _run_pytest(
7449
"""
7550
[case abc]
7651
s: str
@@ -79,15 +54,12 @@ def test_bad_ge_version_check(self) -> None:
7954
"""
8055
)
8156

82-
# Act
83-
actual = self._run_pytest(data)
84-
8557
# Assert
86-
assert "version>=3.8 always true since minimum runtime version is (3, 8)" in actual
58+
assert "version>=3.8 always true since minimum runtime version is (3, 8)" in actual.stdout
8759

8860
def test_bad_eq_version_check(self) -> None:
89-
# Arrange
90-
data = self._dedent(
61+
# Act
62+
actual = _run_pytest(
9163
"""
9264
[case abc]
9365
s: str
@@ -96,8 +68,5 @@ def test_bad_eq_version_check(self) -> None:
9668
"""
9769
)
9870

99-
# Act
100-
actual = self._run_pytest(data)
101-
10271
# Assert
103-
assert "version==3.7 always false since minimum runtime version is (3, 8)" in actual
72+
assert "version==3.7 always false since minimum runtime version is (3, 8)" in actual.stdout

‎mypy/test/meta/test_update_data.py‎

Lines changed: 15 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,47 +3,23 @@
33
Updating the expected output, especially when it's in the form of inline (comment) assertions,
44
can be brittle, which is why we're "meta-testing" here.
55
"""
6-
import shlex
7-
import subprocess
8-
import sys
9-
import textwrap
10-
import uuid
11-
from pathlib import Path
12-
13-
from mypy.test.config import test_data_prefix
146
from mypy.test.helpers import Suite
7+
from mypy.test.meta._pytest import PytestResult, dedent_docstring, run_pytest_data_suite
158

169

17-
class UpdateDataSuite(Suite):
18-
def _run_pytest_update_data(self, data_suite: str, *, max_attempts: int) -> str:
19-
"""
20-
Runs a suite of data test cases through 'pytest --update-data' until either tests pass
21-
or until a maximum number of attempts (needed for incremental tests).
22-
"""
23-
p_test_data = Path(test_data_prefix)
24-
p_root = p_test_data.parent.parent
25-
p = p_test_data / f"check-meta-{uuid.uuid4()}.test"
26-
assert not p.exists()
27-
try:
28-
p.write_text(textwrap.dedent(data_suite).lstrip())
29-
30-
test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
31-
args = [sys.executable, "-m", "pytest", "-n", "0", "-s", "--update-data", test_nodeid]
32-
cmd = shlex.join(args)
33-
for i in range(max_attempts - 1, -1, -1):
34-
res = subprocess.run(args, cwd=p_root)
35-
if res.returncode == 0:
36-
break
37-
print(f"`{cmd}` returned {res.returncode}: {i} attempts remaining")
38-
39-
return p.read_text()
40-
finally:
41-
p.unlink()
10+
def _run_pytest_update_data(data_suite: str) -> PytestResult:
11+
"""
12+
Runs a suite of data test cases through 'pytest --update-data' until either tests pass
13+
or until a maximum number of attempts (needed for incremental tests).
14+
"""
15+
return run_pytest_data_suite(data_suite, extra_args=["--update-data"], max_attempts=3)
16+
4217

18+
class UpdateDataSuite(Suite):
4319
def test_update_data(self) -> None:
4420
# Note: We test multiple testcases rather than 'test case per test case'
4521
# so we could also exercise rewriting multiple testcases at once.
46-
actual = self._run_pytest_update_data(
22+
result = _run_pytest_update_data(
4723
"""
4824
[case testCorrect]
4925
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
@@ -97,12 +73,12 @@ def test_update_data(self) -> None:
9773
[file b.py]
9874
s2: str = 43 # E: baz
9975
[builtins fixtures/list.pyi]
100-
""",
101-
max_attempts=3,
76+
"""
10277
)
10378

10479
# Assert
105-
expected = """
80+
expected = dedent_docstring(
81+
"""
10682
[case testCorrect]
10783
s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
10884
@@ -154,4 +130,5 @@ def test_update_data(self) -> None:
154130
s2: str = 43 # E: Incompatible types in assignment (expression has type "int", variable has type "str")
155131
[builtins fixtures/list.pyi]
156132
"""
157-
assert actual == textwrap.dedent(expected).lstrip()
133+
)
134+
assert result.input_updated == expected

0 commit comments

Comments
 (0)