From d9ab6fa1d914b2484e47399e323501105c379451 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 13:18:32 -0700 Subject: [PATCH 01/15] wip pytest for testcheck -- sort of mostly works! --- conftest.py | 3 +++ mypy/test/collect.py | 28 ++++++++++++++++++++++++++++ mypy/test/data.py | 11 ++++++++++- mypy/test/testcheck.py | 23 ++++++++++++----------- pytest.ini | 5 +++++ test-requirements.txt | 1 + 6 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 conftest.py create mode 100644 mypy/test/collect.py create mode 100644 pytest.ini diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000000..987135e4d0020 --- /dev/null +++ b/conftest.py @@ -0,0 +1,3 @@ +pytest_plugins = [ + 'mypy.test.collect', +] diff --git a/mypy/test/collect.py b/mypy/test/collect.py new file mode 100644 index 0000000000000..3cdfc7f5fa805 --- /dev/null +++ b/mypy/test/collect.py @@ -0,0 +1,28 @@ +import os + +import pytest + +from mypy.test.data import DataSuite + + +def pytest_pycollect_makeitem(collector, name, obj): + if not isinstance(obj, type) or not issubclass(obj, DataSuite): + return None + #os.write(3, ('collecting: %r %r %r\n' % (collector, name, obj)).encode('utf-8')) + + return MypyDataSuite(name, parent=collector) + + +class MypyDataSuite(pytest.Class): + def collect(self): + for case in self.obj.cases(): + yield MypyDataCase(case.name, self, case) + + +class MypyDataCase(pytest.Item): + def __init__(self, name, parent, obj): + super().__init__(name, parent) + self.obj = obj + + def runtest(self): + self.parent.obj().run_case(self.obj) diff --git a/mypy/test/data.py b/mypy/test/data.py index d2626a8c1a3a7..f13a4df19ad90 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -13,7 +13,7 @@ def parse_test_cases( path: str, - perform: Callable[['DataDrivenTestCase'], None], + perform: Optional[Callable[['DataDrivenTestCase'], None]], base_path: str = '.', optional_out: bool = False, include_path: str = None, @@ -218,6 +218,15 @@ def __init__(self, id: str, arg: str, data: List[str], file: str, self.line = line +class DataSuite: + @classmethod + def cases(cls) -> List[DataDrivenTestCase]: + return [] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + raise NotImplementedError + + def parse_test_data(l: List[str], fnam: str) -> List[TestItem]: """Parse a list of lines that represent a sequence of test items.""" diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 1bc7f83834fb8..bedfbe8ff9ade 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -11,9 +11,9 @@ from mypy import build, defaults import mypy.myunit # for mutable globals (ick!) from mypy.build import BuildSource, find_module_clear_caches -from mypy.myunit import Suite, AssertionFailure +from mypy.myunit import AssertionFailure from mypy.test.config import test_temp_dir, test_data_prefix -from mypy.test.data import parse_test_cases, DataDrivenTestCase +from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite from mypy.test.helpers import ( assert_string_arrays_equal, normalize_error_messages, testcase_pyversion, update_testcase_output, @@ -67,16 +67,17 @@ ] -class TypeCheckSuite(Suite): +class TypeCheckSuite(DataSuite): - def cases(self) -> List[DataDrivenTestCase]: + @classmethod + def cases(cls) -> List[DataDrivenTestCase]: c = [] # type: List[DataDrivenTestCase] for f in files: c += parse_test_cases(os.path.join(test_data_prefix, f), - self.run_test, test_temp_dir, True) + None, test_temp_dir, True) return c - def run_test(self, testcase: DataDrivenTestCase) -> None: + def run_case(self, testcase: DataDrivenTestCase) -> None: incremental = 'incremental' in testcase.name.lower() or 'incremental' in testcase.file optional = 'optional' in testcase.file if incremental: @@ -84,17 +85,17 @@ def run_test(self, testcase: DataDrivenTestCase) -> None: # Expect success on first run, errors from testcase.output (if any) on second run. # We briefly sleep to make sure file timestamps are distinct. self.clear_cache() - self.run_test_once(testcase, 1) + self.run_case_once(testcase, 1) time.sleep(0.1) - self.run_test_once(testcase, 2) + self.run_case_once(testcase, 2) elif optional: try: experiments.STRICT_OPTIONAL = True - self.run_test_once(testcase) + self.run_case_once(testcase) finally: experiments.STRICT_OPTIONAL = False else: - self.run_test_once(testcase) + self.run_case_once(testcase) def clear_cache(self) -> None: dn = defaults.MYPY_CACHE @@ -102,7 +103,7 @@ def clear_cache(self) -> None: if os.path.exists(dn): shutil.rmtree(dn) - def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None: + def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None: find_module_clear_caches() program_text = '\n'.join(testcase.input) module_name, program_name, program_text = self.parse_module(program_text) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000000..4071a7cd5b55e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +minversion = 2.8 +testpaths = mypy/test +python_files = test*.py +python_functions = diff --git a/test-requirements.txt b/test-requirements.txt index 47744fe21aac1..a0f8e13a9e8d1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ flake8 typed-ast +pytest>=2.9 From e087d81cead987ddb71f14bb04056a8845595107 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 13:37:02 -0700 Subject: [PATCH 02/15] fix skipping --- mypy/test/collect.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypy/test/collect.py b/mypy/test/collect.py index 3cdfc7f5fa805..2ba279f8d8f82 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -21,8 +21,15 @@ def collect(self): class MypyDataCase(pytest.Item): def __init__(self, name, parent, obj): + self.skip = False + if name.endswith('-skip'): + self.skip = True + name = name[:-len('-skip')] + super().__init__(name, parent) self.obj = obj def runtest(self): + if self.skip: + pytest.skip() self.parent.obj().run_case(self.obj) From 494adb6c384abbd3dcfc69a7dd491d3f01094f0f Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 13:56:41 -0700 Subject: [PATCH 03/15] fix setup/teardown -- now all seems to work! --- mypy/test/collect.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/test/collect.py b/mypy/test/collect.py index 2ba279f8d8f82..06669c843655c 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -2,7 +2,7 @@ import pytest -from mypy.test.data import DataSuite +from mypy.test.data import DataSuite, DataDrivenTestCase def pytest_pycollect_makeitem(collector, name, obj): @@ -20,7 +20,7 @@ def collect(self): class MypyDataCase(pytest.Item): - def __init__(self, name, parent, obj): + def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase): self.skip = False if name.endswith('-skip'): self.skip = True @@ -33,3 +33,9 @@ def runtest(self): if self.skip: pytest.skip() self.parent.obj().run_case(self.obj) + + def setup(self): + self.obj.set_up() + + def teardown(self): + self.obj.tear_down() From c22efe2eb7af7ef3440c966baa8729d376e35b32 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 14:04:40 -0700 Subject: [PATCH 04/15] customize output to be a bit more like myunit --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 4071a7cd5b55e..c64d0cb126375 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] minversion = 2.8 +addopts = --capture=no --tb=line testpaths = mypy/test python_files = test*.py python_functions = From 186b9d03acb6843a3dab412881118fd835fa39f6 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 14:37:57 -0700 Subject: [PATCH 05/15] take more control of error reporting --- mypy/test/collect.py | 6 ++++++ pytest.ini | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/test/collect.py b/mypy/test/collect.py index 06669c843655c..770f39bc9ba9d 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -39,3 +39,9 @@ def setup(self): def teardown(self): self.obj.tear_down() + + def reportinfo(self): + return self.obj.file, self.obj.line, self.obj.name + + def repr_failure(self, excinfo): + return "data: {}:{}\n{}".format(self.obj.file, self.obj.line, excinfo.exconly()) diff --git a/pytest.ini b/pytest.ini index c64d0cb126375..4071a7cd5b55e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,5 @@ [pytest] minversion = 2.8 -addopts = --capture=no --tb=line testpaths = mypy/test python_files = test*.py python_functions = From a030509d8cadc318c0fabc77ca12f63c2242534c Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 15:03:24 -0700 Subject: [PATCH 06/15] print traceback, except on SystemExit --- mypy/test/collect.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy/test/collect.py b/mypy/test/collect.py index 770f39bc9ba9d..7932df4ac8f52 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -44,4 +44,14 @@ def reportinfo(self): return self.obj.file, self.obj.line, self.obj.name def repr_failure(self, excinfo): - return "data: {}:{}\n{}".format(self.obj.file, self.obj.line, excinfo.exconly()) + if excinfo.errisinstance(SystemExit): + # We assume that before doing exit() (which raises SystemExit) we've printed + # enough context about what happened so that a stack trace is not useful. + # In particular, uncaught exceptions during semantic analysis or type checking + # call exit() and they already print out a stack trace. + excrepr = excinfo.exconly() + else: + self.parent._prunetraceback(excinfo) + excrepr = excinfo.getrepr(style='short') + + return "data: {}:{}\n{}".format(self.obj.file, self.obj.line, excrepr) From 0e77a3876b35ccbcd51c4d77a46c54a58261149b Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 15:37:08 -0700 Subject: [PATCH 07/15] wip implement -i as --update-data --- mypy/myunit/__init__.py | 11 +---------- mypy/test/collect.py | 3 ++- mypy/test/helpers.py | 5 ++--- mypy/test/testcheck.py | 7 ++++--- mypy/test/update.py | 8 ++++++++ test-data/unit/check-classes.test | 1 + 6 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 mypy/test/update.py diff --git a/mypy/myunit/__init__.py b/mypy/myunit/__init__.py index b89542da662ec..591b6a5bac14a 100644 --- a/mypy/myunit/__init__.py +++ b/mypy/myunit/__init__.py @@ -14,8 +14,6 @@ is_quiet = False patterns = [] # type: List[str] times = [] # type: List[Tuple[float, str]] -APPEND_TESTCASES = '' -UPDATE_TESTCASES = False class AssertionFailure(Exception): @@ -199,7 +197,6 @@ def __init__(self, suites: List[Suite]) -> None: def main(args: List[str] = None) -> None: global patterns, is_verbose, is_quiet - global APPEND_TESTCASES, UPDATE_TESTCASES if not args: args = sys.argv[1:] is_verbose = False @@ -213,12 +210,6 @@ def main(args: List[str] = None) -> None: is_verbose = True elif a == '-q': is_quiet = True - elif a == '-u': - APPEND_TESTCASES = '.new' - UPDATE_TESTCASES = True - elif a == '-i': - APPEND_TESTCASES = '' - UPDATE_TESTCASES = True elif a == '-m': i += 1 if i == len(args): @@ -227,7 +218,7 @@ def main(args: List[str] = None) -> None: elif not a.startswith('-'): patterns.append(a) else: - sys.exit('Usage: python -m mypy.myunit [-v] [-q] [-u | -i]' + sys.exit('Usage: python -m mypy.myunit [-v] [-q]' + ' -m mypy.test.module [-m mypy.test.module ...] [filter ...]') i += 1 if len(patterns) == 0: diff --git a/mypy/test/collect.py b/mypy/test/collect.py index 7932df4ac8f52..c863caedcef79 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -32,7 +32,8 @@ def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase): def runtest(self): if self.skip: pytest.skip() - self.parent.obj().run_case(self.obj) + update_data = self.config.getoption('--update-data', False) + self.parent.obj(update_data=update_data).run_case(self.obj) def setup(self): self.obj.set_up() diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 95abef2501b91..6a56a6d13d9bb 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -85,9 +85,8 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str], raise AssertionFailure(msg) -def update_testcase_output(testcase: DataDrivenTestCase, output: List[str], append: str) -> None: +def update_testcase_output(testcase: DataDrivenTestCase, output: List[str]) -> None: testcase_path = os.path.join(testcase.old_cwd, testcase.file) - newfile = testcase_path + append data_lines = open(testcase_path).read().splitlines() test = '\n'.join(data_lines[testcase.line:testcase.lastline]) @@ -111,7 +110,7 @@ def update_testcase_output(testcase: DataDrivenTestCase, output: List[str], appe data_lines[testcase.line:testcase.lastline] = [test] data = '\n'.join(data_lines) - with open(newfile, 'w') as f: + with open(testcase_path, 'w') as f: print(data, file=f) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index bedfbe8ff9ade..07d73a4345c0c 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -9,7 +9,6 @@ from typing import Tuple, List, Dict, Set from mypy import build, defaults -import mypy.myunit # for mutable globals (ick!) from mypy.build import BuildSource, find_module_clear_caches from mypy.myunit import AssertionFailure from mypy.test.config import test_temp_dir, test_data_prefix @@ -68,6 +67,8 @@ class TypeCheckSuite(DataSuite): + def __init__(self, *, update_data=False): + self.update_data = update_data @classmethod def cases(cls) -> List[DataDrivenTestCase]: @@ -141,8 +142,8 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None: a = e.messages a = normalize_error_messages(a) - if output != a and mypy.myunit.UPDATE_TESTCASES: - update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES) + if output != a and self.update_data: + update_testcase_output(testcase, a) assert_string_arrays_equal( output, a, diff --git a/mypy/test/update.py b/mypy/test/update.py new file mode 100644 index 0000000000000..798c84fa122df --- /dev/null +++ b/mypy/test/update.py @@ -0,0 +1,8 @@ +import pytest + + +def pytest_addoption(parser): + group = parser.getgroup('mypy') + group.addoption('--update-data', action='store_true', default=False, + help='Update test data to reflect actual output' + ' (supported only for certain tests)') diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 3dba6e9b818cb..e24fb548d8537 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -19,6 +19,7 @@ class B: def bar(self, x: 'B', y: A) -> None: pass [out] main:5: error: Argument 1 to "foo" of "A" has incompatible type "B"; expected "A" +main:5: error: more! main:6: error: "A" has no attribute "bar" [case testMethodCallWithSubtype] From 4b63be694ce53a90224c26dd90376693660616a7 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 15:42:01 -0700 Subject: [PATCH 08/15] wip refactor --- conftest.py | 2 +- mypy/test/collect.py | 58 ------------------------------ mypy/test/data.py | 86 +++++++++++++++++++++++++++++++++++++++----- mypy/test/update.py | 8 ----- 4 files changed, 78 insertions(+), 76 deletions(-) diff --git a/conftest.py b/conftest.py index 987135e4d0020..9673db23c2fcb 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,3 @@ pytest_plugins = [ - 'mypy.test.collect', + 'mypy.test.data', ] diff --git a/mypy/test/collect.py b/mypy/test/collect.py index c863caedcef79..e69de29bb2d1d 100644 --- a/mypy/test/collect.py +++ b/mypy/test/collect.py @@ -1,58 +0,0 @@ -import os - -import pytest - -from mypy.test.data import DataSuite, DataDrivenTestCase - - -def pytest_pycollect_makeitem(collector, name, obj): - if not isinstance(obj, type) or not issubclass(obj, DataSuite): - return None - #os.write(3, ('collecting: %r %r %r\n' % (collector, name, obj)).encode('utf-8')) - - return MypyDataSuite(name, parent=collector) - - -class MypyDataSuite(pytest.Class): - def collect(self): - for case in self.obj.cases(): - yield MypyDataCase(case.name, self, case) - - -class MypyDataCase(pytest.Item): - def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase): - self.skip = False - if name.endswith('-skip'): - self.skip = True - name = name[:-len('-skip')] - - super().__init__(name, parent) - self.obj = obj - - def runtest(self): - if self.skip: - pytest.skip() - update_data = self.config.getoption('--update-data', False) - self.parent.obj(update_data=update_data).run_case(self.obj) - - def setup(self): - self.obj.set_up() - - def teardown(self): - self.obj.tear_down() - - def reportinfo(self): - return self.obj.file, self.obj.line, self.obj.name - - def repr_failure(self, excinfo): - if excinfo.errisinstance(SystemExit): - # We assume that before doing exit() (which raises SystemExit) we've printed - # enough context about what happened so that a stack trace is not useful. - # In particular, uncaught exceptions during semantic analysis or type checking - # call exit() and they already print out a stack trace. - excrepr = excinfo.exconly() - else: - self.parent._prunetraceback(excinfo) - excrepr = excinfo.getrepr(style='short') - - return "data: {}:{}\n{}".format(self.obj.file, self.obj.line, excrepr) diff --git a/mypy/test/data.py b/mypy/test/data.py index f13a4df19ad90..d0c97303505f4 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -6,6 +6,7 @@ from os import remove, rmdir import shutil +import pytest from typing import Callable, List, Tuple, Set, Optional from mypy.myunit import TestCase, SkipTestCaseException @@ -218,15 +219,6 @@ def __init__(self, id: str, arg: str, data: List[str], file: str, self.line = line -class DataSuite: - @classmethod - def cases(cls) -> List[DataDrivenTestCase]: - return [] - - def run_case(self, testcase: DataDrivenTestCase) -> None: - raise NotImplementedError - - def parse_test_data(l: List[str], fnam: str) -> List[TestItem]: """Parse a list of lines that represent a sequence of test items.""" @@ -345,3 +337,79 @@ def fix_win_path(line: str) -> str: filename, lineno, message = m.groups() return '{}:{}{}'.format(filename.replace('/', '\\'), lineno or '', message) + + +## +# +# pytest setup +# +## + + +def pytest_addoption(parser): + group = parser.getgroup('mypy') + group.addoption('--update-data', action='store_true', default=False, + help='Update test data to reflect actual output' + ' (supported only for certain tests)') + + +def pytest_pycollect_makeitem(collector, name, obj): + if not isinstance(obj, type) or not issubclass(obj, DataSuite): + return None + #os.write(3, ('collecting: %r %r %r\n' % (collector, name, obj)).encode('utf-8')) + + return MypyDataSuite(name, parent=collector) + + +class MypyDataSuite(pytest.Class): + def collect(self): + for case in self.obj.cases(): + yield MypyDataCase(case.name, self, case) + + +class MypyDataCase(pytest.Item): + def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase): + self.skip = False + if name.endswith('-skip'): + self.skip = True + name = name[:-len('-skip')] + + super().__init__(name, parent) + self.obj = obj + + def runtest(self): + if self.skip: + pytest.skip() + update_data = self.config.getoption('--update-data', False) + self.parent.obj(update_data=update_data).run_case(self.obj) + + def setup(self): + self.obj.set_up() + + def teardown(self): + self.obj.tear_down() + + def reportinfo(self): + return self.obj.file, self.obj.line, self.obj.name + + def repr_failure(self, excinfo): + if excinfo.errisinstance(SystemExit): + # We assume that before doing exit() (which raises SystemExit) we've printed + # enough context about what happened so that a stack trace is not useful. + # In particular, uncaught exceptions during semantic analysis or type checking + # call exit() and they already print out a stack trace. + excrepr = excinfo.exconly() + else: + self.parent._prunetraceback(excinfo) + excrepr = excinfo.getrepr(style='short') + + return "data: {}:{}\n{}".format(self.obj.file, self.obj.line, excrepr) + + +class DataSuite: + @classmethod + def cases(cls) -> List[DataDrivenTestCase]: + return [] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + raise NotImplementedError diff --git a/mypy/test/update.py b/mypy/test/update.py index 798c84fa122df..e69de29bb2d1d 100644 --- a/mypy/test/update.py +++ b/mypy/test/update.py @@ -1,8 +0,0 @@ -import pytest - - -def pytest_addoption(parser): - group = parser.getgroup('mypy') - group.addoption('--update-data', action='store_true', default=False, - help='Update test data to reflect actual output' - ' (supported only for certain tests)') From 390812b9244ca014337d90e643f6188afed9bedb Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 15:44:43 -0700 Subject: [PATCH 09/15] revert accidentally-committed test change to test data --- test-data/unit/check-classes.test | 1 - 1 file changed, 1 deletion(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index e24fb548d8537..3dba6e9b818cb 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -19,7 +19,6 @@ class B: def bar(self, x: 'B', y: A) -> None: pass [out] main:5: error: Argument 1 to "foo" of "A" has incompatible type "B"; expected "A" -main:5: error: more! main:6: error: "A" has no attribute "bar" [case testMethodCallWithSubtype] From 8c051dce167b2a653f618fc32ada0dcaccc9012e Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 15:55:11 -0700 Subject: [PATCH 10/15] tweaks for polish --- mypy/test/data.py | 6 ++++-- pytest.ini | 8 +++++++- test-requirements.txt | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index d0c97303505f4..e0e2b1f4a0317 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -22,6 +22,10 @@ def parse_test_cases( """Parse a file with test case descriptions. Return an array of test cases. + + NB this function and DataDrivenTestCase are shared between the + myunit and pytest codepaths -- if something looks redundant, + that's likely the reason. """ if not include_path: @@ -356,8 +360,6 @@ def pytest_addoption(parser): def pytest_pycollect_makeitem(collector, name, obj): if not isinstance(obj, type) or not issubclass(obj, DataSuite): return None - #os.write(3, ('collecting: %r %r %r\n' % (collector, name, obj)).encode('utf-8')) - return MypyDataSuite(name, parent=collector) diff --git a/pytest.ini b/pytest.ini index 4071a7cd5b55e..4a9d2894ef961 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,11 @@ [pytest] +# testpaths is new in 2.8 minversion = 2.8 + testpaths = mypy/test + python_files = test*.py -python_functions = + +# empty patterns for default python collector, to stick to our plugin's collector +python_classes = +python_functions = diff --git a/test-requirements.txt b/test-requirements.txt index a0f8e13a9e8d1..2993972a5b324 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ flake8 typed-ast -pytest>=2.9 +pytest>=2.8 From 6b0860b7e628b0893409bd687a3f000619ff57a3 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 16:56:40 -0700 Subject: [PATCH 11/15] fix self-check --- mypy/test/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index e0e2b1f4a0317..9f07e5e3bcf9f 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -6,7 +6,7 @@ from os import remove, rmdir import shutil -import pytest +import pytest # type: ignore from typing import Callable, List, Tuple, Set, Optional from mypy.myunit import TestCase, SkipTestCaseException @@ -370,7 +370,7 @@ def collect(self): class MypyDataCase(pytest.Item): - def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase): + def __init__(self, name: str, parent: MypyDataSuite, obj: DataDrivenTestCase) -> None: self.skip = False if name.endswith('-skip'): self.skip = True From f7b15bdfa41d16c9e63e5fe854d1d4d941574c66 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 16:56:49 -0700 Subject: [PATCH 12/15] wire up into runtests --- runtests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/runtests.py b/runtests.py index d6d67bf5294ba..ea411f719d745 100755 --- a/runtests.py +++ b/runtests.py @@ -93,6 +93,13 @@ def add_mypy_package(self, name: str, packagename: str) -> None: def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: self.add_mypy_cmd(name, ['-c'] + list(args), cwd=cwd) + def add_pytest(self, name: str, pytest_args: List[str]) -> None: + full_name = 'pytest %s' % name + if not self.allow(full_name): + return + args = [sys.executable, '-m', 'pytest'] + pytest_args + self.waiter.add(LazySubprocess(full_name, args, env=self.env)) + def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: name = 'run %s' % name if not self.allow(name): @@ -187,6 +194,16 @@ def add_imports(driver: Driver) -> None: driver.add_flake8('module %s' % mod, f) +PYTEST_FILES = ['mypy/test/{}.py'.format(name) for name in [ + 'testcheck', +]] + + +def add_pytest(driver: Driver) -> None: + for f in PYTEST_FILES: + driver.add_pytest(f, [f]) + + def add_myunit(driver: Driver) -> None: for f in find_files('mypy', prefix='test', suffix='.py'): mod = file_to_module(f) @@ -199,6 +216,9 @@ def add_myunit(driver: Driver) -> None: # parsing tests separately since they are much slower than # proper unit tests. pass + elif f in PYTEST_FILES: + # This module has been converted to pytest; don't try to use myunit. + pass else: driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) @@ -362,6 +382,7 @@ def main() -> None: add_cmdline(driver) add_basic(driver) add_selftypecheck(driver) + add_pytest(driver) add_myunit(driver) add_imports(driver) add_stubs(driver) From 53035e3fdd776b4d9587339c1480821ba3892ea9 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 26 Jul 2016 17:07:48 -0700 Subject: [PATCH 13/15] and update README --- README.md | 17 ++++++++++++----- runtests.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 052afb31f417a..65bf4d4d61f66 100644 --- a/README.md +++ b/README.md @@ -184,19 +184,26 @@ To run all tests, run the script `runtests.py` in the mypy repository: Note that some tests will be disabled for older python versions. This will run all tests, including integration and regression tests, -and will type check mypy and verify that all stubs are valid. You can also -run unit tests only, which run pretty quickly: - - $ ./runtests.py unit-test +and will type check mypy and verify that all stubs are valid. You can run a subset of test suites by passing positive or negative filters: $ ./runtests.py lex parse -x lint -x stub -If you want to run individual unit tests, you can run `myunit` directly, or +For example, to run unit tests only, which run pretty quickly: + + $ ./runtests.py unit-test pytest + +The unit test suites are driven by a mixture of test frameworks: +mypy's own `myunit` framework, and `pytest`, which we're in the +process of migrating to. For finer control over which unit tests are +run and how, you can run `py.test` or `scripts/myunit` directly, or pass inferior arguments via `-a`: + $ py.test mypy/test/testcheck.py -v -k MethodCall + $ ./runtests.py -v 'pytest mypy/test/testcheck' -a -v -a -k -a MethodCall + $ PYTHONPATH=$PWD scripts/myunit -m mypy.test.testlex -v '*backslash*' $ ./runtests.py mypy.test.testlex -a -v -a '*backslash*' diff --git a/runtests.py b/runtests.py index ea411f719d745..f89644d38f9a5 100755 --- a/runtests.py +++ b/runtests.py @@ -201,7 +201,7 @@ def add_imports(driver: Driver) -> None: def add_pytest(driver: Driver) -> None: for f in PYTEST_FILES: - driver.add_pytest(f, [f]) + driver.add_pytest(f, [f] + driver.arglist) def add_myunit(driver: Driver) -> None: From 3ffd2b08c77e6ce87809498faa0e4ffc9c608f5e Mon Sep 17 00:00:00 2001 From: Greg Price Date: Wed, 27 Jul 2016 15:11:36 -0700 Subject: [PATCH 14/15] explain type: ignore --- mypy/test/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index 9f07e5e3bcf9f..8b21db3317960 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -6,7 +6,7 @@ from os import remove, rmdir import shutil -import pytest # type: ignore +import pytest # type: ignore # no pytest in typeshed from typing import Callable, List, Tuple, Set, Optional from mypy.myunit import TestCase, SkipTestCaseException From 0cc0f1f562a4cdb7723fcc6e3dbcfb75d2a35451 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Wed, 27 Jul 2016 15:42:25 -0700 Subject: [PATCH 15/15] parse pytest test counts in runtests --- mypy/waiter.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypy/waiter.py b/mypy/waiter.py index 47a493bb4a03c..10fa4028cbacd 100644 --- a/mypy/waiter.py +++ b/mypy/waiter.py @@ -281,6 +281,18 @@ def parse_test_stats_from_output(output: str, fail_type: Optional[str]) -> Tuple Return tuple (number of tests, number of test failures). Default to the entire task representing a single test as a fallback. """ + + # pytest + m = re.search('^=+ (.*) in [0-9.]+ seconds =+\n\Z', output, re.MULTILINE) + if m: + counts = {} + for part in m.group(1).split(', '): # e.g., '3 failed, 32 passed, 345 deselected' + count, key = part.split() + counts[key] = int(count) + return (sum(c for k, c in counts.items() if k != 'deselected'), + counts.get('failed', 0)) + + # myunit m = re.search('^([0-9]+)/([0-9]+) test cases failed(, ([0-9]+) skipped)?.$', output, re.MULTILINE) if m: @@ -289,6 +301,7 @@ def parse_test_stats_from_output(output: str, fail_type: Optional[str]) -> Tuple re.MULTILINE) if m: return int(m.group(1)), 0 + # Couldn't find test counts, so fall back to single test per tasks. if fail_type is not None: return 1, 1