From 4b380299b40cf22b424593100c2d6b86d74f224d Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Fri, 15 Dec 2017 14:43:11 -0800 Subject: [PATCH 01/15] Flush error messages incrementally after processing a file In order to avoid duplicate error messages for errors produced in both load_graph() and process_graph() and to prevent misordered error messages in a number of places, lists of error messages are now tracked per-file. These lists are collected and printed out when a file is complete. To maintain consistency with clients that use .messages() (namely, tests), messages are generated file-at-a-time even when not printing them out incrementally. Fixes #1294 --- mypy/build.py | 26 +++++- mypy/errors.py | 108 +++++++++++++++++----- mypy/main.py | 32 ++++--- mypy/messages.py | 5 +- mypy/test/testerrorstream.py | 76 +++++++++++++++ runtests.py | 3 +- test-data/unit/cmdline.test | 2 +- test-data/unit/errorstream.test | 46 +++++++++ test-data/unit/fine-grained-blockers.test | 3 - test-data/unit/fine-grained.test | 2 +- 10 files changed, 255 insertions(+), 48 deletions(-) create mode 100644 mypy/test/testerrorstream.py create mode 100644 test-data/unit/errorstream.test diff --git a/mypy/build.py b/mypy/build.py index 50140c5550c5..3ee76bdedbe9 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -133,6 +133,8 @@ def build(sources: List[BuildSource], alt_lib_path: Optional[str] = None, bin_dir: Optional[str] = None, saved_cache: Optional[SavedCache] = None, + flush_errors: Optional[Callable[[List[str]], None]] = None, + plugin: Optional[Plugin] = None, ) -> BuildResult: """Analyze a program. @@ -150,6 +152,8 @@ def build(sources: List[BuildSource], bin_dir: directory containing the mypy script, used for finding data directories; if omitted, use '.' as the data directory saved_cache: optional dict with saved cache state for dmypy (read-write!) + flush_errors: optional function to flush errors after a file is processed + plugin: optional plugin that overrides the configured one """ # This seems the most reasonable place to tune garbage collection. gc.set_threshold(50000) @@ -199,7 +203,7 @@ def build(sources: List[BuildSource], reports = Reports(data_dir, options.report_dirs) source_set = BuildSourceSet(sources) errors = Errors(options.show_error_context, options.show_column_numbers) - plugin = load_plugins(options, errors) + plugin = plugin or load_plugins(options, errors) # Construct a build manager object to hold state during the build. # @@ -212,10 +216,12 @@ def build(sources: List[BuildSource], version_id=__version__, plugin=plugin, errors=errors, - saved_cache=saved_cache) + saved_cache=saved_cache, + flush_errors=flush_errors) try: graph = dispatch(sources, manager) + manager.error_flush(manager.errors.new_messages()) return BuildResult(manager, graph) finally: manager.log("Build finished in %.3f seconds with %d modules, and %d errors" % @@ -518,6 +524,7 @@ class BuildManager: version_id: The current mypy version (based on commit id when possible) plugin: Active mypy plugin(s) errors: Used for reporting all errors + flush_errors: A function for optionally processing errors after each SCC saved_cache: Dict with saved cache state for dmypy and fine-grained incremental mode (read-write!) stats: Dict with various instrumentation numbers @@ -532,6 +539,7 @@ def __init__(self, data_dir: str, version_id: str, plugin: Plugin, errors: Errors, + flush_errors: Optional[Callable[[List[str]], None]] = None, saved_cache: Optional[SavedCache] = None, ) -> None: self.start_time = time.time() @@ -555,6 +563,7 @@ def __init__(self, data_dir: str, self.stale_modules = set() # type: Set[str] self.rechecked_modules = set() # type: Set[str] self.plugin = plugin + self.flush_errors = flush_errors self.saved_cache = saved_cache if saved_cache is not None else {} # type: SavedCache self.stats = {} # type: Dict[str, Any] # Values are ints or floats @@ -703,6 +712,10 @@ def add_stats(self, **kwds: Any) -> None: def stats_summary(self) -> Mapping[str, object]: return self.stats + def error_flush(self, msgs: List[str]) -> None: + if self.flush_errors: + self.flush_errors(msgs) + def remove_cwd_prefix_from_path(p: str) -> str: """Remove current working directory prefix from p, if present. @@ -1973,6 +1986,10 @@ def write_cache(self) -> None: def dependency_priorities(self) -> List[int]: return [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies] + def generate_unused_ignore_notes(self) -> None: + if self.options.warn_unused_ignores: + self.manager.errors.generate_unused_ignore_notes(self.xpath) + def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: set_orig = set(manager.saved_cache) @@ -1999,9 +2016,6 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: dump_graph(graph) return graph process_graph(graph, manager) - if manager.options.warn_unused_ignores: - # TODO: This could also be a per-module option. - manager.errors.generate_unused_ignore_notes() updated = preserve_cache(graph) set_updated = set(updated) manager.saved_cache.clear() @@ -2490,6 +2504,8 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No graph[id].transitive_error = True for id in stale: graph[id].finish_passes() + graph[id].generate_unused_ignore_notes() + manager.error_flush(manager.errors.new_file_messages(graph[id].xpath)) graph[id].write_cache() graph[id].mark_as_rechecked() diff --git a/mypy/errors.py b/mypy/errors.py index 923c5924422f..0b7a16fc22f1 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -90,8 +90,16 @@ class Errors: current error context (nested imports). """ - # List of generated error messages. - error_info = None # type: List[ErrorInfo] + # Map from files to generated error messages. Is an OrderedDict so + # that it can be used to order messages based on the order the + # files were processed. + error_info_map = None # type: Dict[str, List[ErrorInfo]] + + # The size of error_info the last time that error messages were flushed + new_errors_start_map = None # type: Dict[str, int] + + # A cache of the formatted messages + formatted_messages = None # type: List[str] # Current error context: nested import context/stack, as a list of (path, line) pairs. import_ctx = None # type: List[Tuple[str, int]] @@ -141,8 +149,10 @@ def __init__(self, show_error_context: bool = False, self.initialize() def initialize(self) -> None: - self.error_info = [] + self.error_info_map = OrderedDict() + self.new_errors_start_map = defaultdict(int) self.import_ctx = [] + self.formatted_messages = [] self.error_files = set() self.type_name = [None] self.function_or_member = [None] @@ -289,6 +299,11 @@ def report(self, target=self.current_target()) self.add_error_info(info) + def _add_error_info(self, info: ErrorInfo) -> None: + if info.file not in self.error_info_map: + self.error_info_map[info.file] = [] + self.error_info_map[info.file].append(info) + def add_error_info(self, info: ErrorInfo) -> None: (file, line) = cast(Tuple[str, int], info.origin) # see issue 1855 if not info.blocker: # Blockers cannot be ignored @@ -302,18 +317,18 @@ def add_error_info(self, info: ErrorInfo) -> None: if info.message in self.only_once_messages: return self.only_once_messages.add(info.message) - self.error_info.append(info) + self._add_error_info(info) self.error_files.add(file) - def generate_unused_ignore_notes(self) -> None: - for file, ignored_lines in self.ignored_lines.items(): - if not self.is_typeshed_file(file): - for line in ignored_lines - self.used_ignored_lines[file]: - # Don't use report since add_error_info will ignore the error! - info = ErrorInfo(self.import_context(), file, self.current_module(), None, - None, line, -1, 'note', "unused 'type: ignore' comment", - False, False) - self.error_info.append(info) + def generate_unused_ignore_notes(self, file: str) -> None: + ignored_lines = self.ignored_lines[file] + if not self.is_typeshed_file(file): + for line in ignored_lines - self.used_ignored_lines[file]: + # Don't use report since add_error_info will ignore the error! + info = ErrorInfo(self.import_context(), file, self.current_module(), None, + None, line, -1, 'note', "unused 'type: ignore' comment", + False, False) + self._add_error_info(info) def is_typeshed_file(self, file: str) -> bool: # gross, but no other clear way to tell @@ -321,21 +336,22 @@ def is_typeshed_file(self, file: str) -> bool: def num_messages(self) -> int: """Return the number of generated messages.""" - return len(self.error_info) + return sum(len(x) for x in self.error_info_map.values()) def is_errors(self) -> bool: """Are there any generated errors?""" - return bool(self.error_info) + return bool(self.error_info_map) def is_blockers(self) -> bool: """Are the any errors that are blockers?""" - return any(err for err in self.error_info if err.blocker) + return any(err for errs in self.error_info_map.values() for err in errs if err.blocker) def blocker_module(self) -> Optional[str]: """Return the module with a blocking error, or None if not possible.""" - for err in self.error_info: - if err.blocker: - return err.module + for errs in self.error_info_map.values(): + for err in errs: + if err.blocker: + return err.module return None def is_errors_for_file(self, file: str) -> bool: @@ -347,17 +363,22 @@ def raise_error(self) -> None: Render the messages suitable for displaying. """ + # self.new_messages() will format all messages that haven't already + # been returned from a new_module_messages() call. Count how many + # we've seen before that. + already_seen = len(self.formatted_messages) raise CompileError(self.messages(), use_stdout=True, - module_with_blocker=self.blocker_module()) + module_with_blocker=self.blocker_module(), + num_already_seen=already_seen) - def messages(self) -> List[str]: + def format_messages(self, error_info: List[ErrorInfo]) -> List[str]: """Return a string list that represents the error messages. Use a form suitable for displaying to the user. """ a = [] # type: List[str] - errors = self.render_messages(self.sort_messages(self.error_info)) + errors = self.render_messages(self.sort_messages(error_info)) errors = self.remove_duplicates(errors) for file, line, column, severity, message in errors: s = '' @@ -375,12 +396,48 @@ def messages(self) -> List[str]: a.append(s) return a + def new_file_messages(self, path: str) -> List[str]: + """Return a string list of new error messages from a given file. + + Use a form suitable for displaying to the user. + Formatted messages are cached in the order they are generated + by new_file_messages() in order to have consistency in output + between incrementally generated messages and .messages() calls. + """ + if path not in self.error_info_map: + return [] + msgs = self.format_messages(self.error_info_map[path][self.new_errors_start_map[path]:]) + self.new_errors_start_map[path] = len(self.error_info_map[path]) + self.formatted_messages += msgs + return msgs + + def new_messages(self) -> List[str]: + """Return a string list of new error messages. + + Use a form suitable for displaying to the user. + Errors from different files are ordered based on the order in which + they first generated an error. + """ + msgs = [] + for key in self.error_info_map.keys(): + msgs.extend(self.new_file_messages(key)) + return msgs + + def messages(self) -> List[str]: + """Return a string list that represents the error messages. + + Use a form suitable for displaying to the user. + """ + self.new_messages() + return self.formatted_messages + def targets(self) -> Set[str]: """Return a set of all targets that contain errors.""" # TODO: Make sure that either target is always defined or that not being defined # is okay for fine-grained incremental checking. return set(info.target - for info in self.error_info + for errs in self.error_info_map.values() + for info in errs if info.target) def render_messages(self, errors: List[ErrorInfo]) -> List[Tuple[Optional[str], int, int, @@ -517,15 +574,18 @@ class CompileError(Exception): use_stdout = False # Can be set in case there was a module with a blocking error module_with_blocker = None # type: Optional[str] + num_already_seen = 0 def __init__(self, messages: List[str], use_stdout: bool = False, - module_with_blocker: Optional[str] = None) -> None: + module_with_blocker: Optional[str] = None, + num_already_seen: int = 0) -> None: super().__init__('\n'.join(messages)) self.messages = messages self.use_stdout = use_stdout self.module_with_blocker = module_with_blocker + self.num_already_seen = num_already_seen class DecodeError(Exception): diff --git a/mypy/main.py b/mypy/main.py index 2b3deae49f62..733410d968fa 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -8,7 +8,7 @@ import sys import time -from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple +from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Callable from mypy import build from mypy import defaults @@ -62,13 +62,27 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] sources, options = process_options(args) serious = False + errors = False + + def flush_errors(a: List[str]) -> None: + nonlocal errors + if a: + errors = True + f = sys.stderr if serious else sys.stdout + try: + for m in a: + f.write(m + '\n') + except BrokenPipeError: + pass + try: - res = type_check_only(sources, bin_dir, options) + res = type_check_only(sources, bin_dir, options, flush_errors) a = res.errors except CompileError as e: a = e.messages if not e.use_stdout: serious = True + flush_errors(a[e.num_already_seen:]) if options.warn_unused_configs and options.unused_configs: print("Warning: unused section(s) in %s: %s" % (options.config_file, @@ -77,13 +91,7 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: if options.junit_xml: t1 = time.time() util.write_junit_xml(t1 - t0, serious, a, options.junit_xml) - if a: - f = sys.stderr if serious else sys.stdout - try: - for m in a: - f.write(m + '\n') - except BrokenPipeError: - pass + if errors: sys.exit(1) @@ -112,11 +120,13 @@ def readlinkabs(link: str) -> str: def type_check_only(sources: List[BuildSource], bin_dir: Optional[str], - options: Options) -> BuildResult: + options: Options, + flush_errors: Optional[Callable[[List[str]], None]]) -> BuildResult: # Type-check the program and dependencies. return build.build(sources=sources, bin_dir=bin_dir, - options=options) + options=options, + flush_errors=flush_errors) FOOTER = """environment variables: diff --git a/mypy/messages.py b/mypy/messages.py index 7c0df42894f6..3c13a2c41db1 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -153,8 +153,9 @@ def copy(self) -> 'MessageBuilder': def add_errors(self, messages: 'MessageBuilder') -> None: """Add errors in messages to this builder.""" if self.disable_count <= 0: - for info in messages.errors.error_info: - self.errors.add_error_info(info) + for errs in messages.errors.error_info_map.values(): + for info in errs: + self.errors.add_error_info(info) def disable_errors(self) -> None: self.disable_count += 1 diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py new file mode 100644 index 000000000000..e8bad150fb31 --- /dev/null +++ b/mypy/test/testerrorstream.py @@ -0,0 +1,76 @@ +"""Tests for mypy incremental error output.""" +from typing import List, Callable, Optional + +import os + +from mypy import defaults, build +from mypy.test.config import test_temp_dir +from mypy.myunit import AssertionFailure +from mypy.test.helpers import assert_string_arrays_equal +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.build import BuildSource +from mypy.errors import CompileError +from mypy.options import Options +from mypy.plugin import Plugin, ChainedPlugin, DefaultPlugin, FunctionContext +from mypy.nodes import CallExpr, StrExpr +from mypy.types import Type + + +class ErrorStreamSuite(DataSuite): + files = ['errorstream.test'] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_error_stream(testcase) + + +def test_error_stream(testcase: DataDrivenTestCase) -> None: + """Perform a single error streaming test case. + + The argument contains the description of the test case. + """ + options = Options() + options.show_traceback = True + + a = [] + + def flush_errors(msgs: List[str]) -> None: + nonlocal a + if msgs: + a.append('==== Errors flushed ====') + a += msgs + plugin = ChainedPlugin(options, [LoggingPlugin(options, flush_errors), DefaultPlugin(options)]) + + sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))] + try: + build.build(sources=sources, + options=options, + alt_lib_path=test_temp_dir, + flush_errors=flush_errors, + plugin=plugin) + except CompileError as e: + a.append('==== Blocking error ====') + a += e.messages[e.num_already_seen:] + + assert_string_arrays_equal(testcase.output, a, + 'Invalid output ({}, line {})'.format( + testcase.file, testcase.line)) + + +# Use a typechecking plugin to allow test cases to emit messages +# during typechecking. This allows us to verify that error messages +# from one SCC are printed before later ones are typechecked. +class LoggingPlugin(Plugin): + def __init__(self, options: Options, log: Callable[[List[str]], None]) -> None: + super().__init__(options) + self.log = log + + def get_function_hook(self, fullname: str) -> Optional[Callable[[FunctionContext], Type]]: + if fullname == 'log.log_checking': + return self.hook + return None + + def hook(self, ctx: FunctionContext) -> Type: + assert(isinstance(ctx.context, CallExpr) and len(ctx.context.args) > 0 and + isinstance(ctx.context.args[0], StrExpr)) + self.log([ctx.context.args[0].value]) + return ctx.default_return_type diff --git a/runtests.py b/runtests.py index d4712bbfbabb..e9d9a000c695 100755 --- a/runtests.py +++ b/runtests.py @@ -213,7 +213,8 @@ def test_path(*names: str): 'testtransform', 'testtypegen', 'testparse', - 'testsemanal' + 'testsemanal', + 'testerrorstream', ) SLOW_FILES = test_path( diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 9ed8e602e278..27d57cd7f449 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -400,9 +400,9 @@ bla bla [file error.py] bla bla [out] +normal.py:2: error: Unsupported operand types for + ("int" and "str") main.py:4: note: Import of 'error' ignored main.py:4: note: (Using --follow-imports=error, module not passed on command line) -normal.py:2: error: Unsupported operand types for + ("int" and "str") main.py:5: error: Revealed type is 'builtins.int' main.py:6: error: Revealed type is 'builtins.int' main.py:7: error: Revealed type is 'Any' diff --git a/test-data/unit/errorstream.test b/test-data/unit/errorstream.test new file mode 100644 index 000000000000..b98eae2ab697 --- /dev/null +++ b/test-data/unit/errorstream.test @@ -0,0 +1,46 @@ +-- Test cases for incremental error streaming. Each test case consists of two +-- sections. +-- The first section contains [case NAME] followed by the input code, while +-- the second section contains [out] followed by the output from the checker. +-- Each time errors are reported, '==== Errors flushed ====' is printed. +-- The log.log_checking() function will immediately emit a message from +-- a plugin when a call to it is checked, which can be used to verify that +-- error messages are printed before doing later typechecking work. +-- +-- The input file name in errors is "file". +-- +-- Comments starting with "--" in this file will be ignored, except for lines +-- starting with "----" that are not ignored. The first two dashes of these +-- lines are interpreted as escapes and removed. + +[case testErrorStream] +import b +[file log.py] +def log_checking(msg: str) -> None: ... +[file a.py] +1 + '' +[file b.py] +import a +import log +log.log_checking('Checking b') # Make sure that a has been flushed before this is checked +'' / 2 +[out] +==== Errors flushed ==== +a.py:1: error: Unsupported operand types for + ("int" and "str") +==== Errors flushed ==== +Checking b +==== Errors flushed ==== +b.py:4: error: Unsupported operand types for / ("str" and "int") + +[case testBlockers] +import b +[file a.py] +1 + '' +[file b.py] +import a +break +[out] +==== Errors flushed ==== +a.py:1: error: Unsupported operand types for + ("int" and "str") +==== Blocking error ==== +b.py:2: error: 'break' outside loop diff --git a/test-data/unit/fine-grained-blockers.test b/test-data/unit/fine-grained-blockers.test index f4af0626a185..9eaf25eeea05 100644 --- a/test-data/unit/fine-grained-blockers.test +++ b/test-data/unit/fine-grained-blockers.test @@ -255,9 +255,6 @@ a.py:1: error: invalid syntax main:1: error: Cannot find module named 'a' main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) b.py:1: error: Cannot find module named 'a' --- TODO: Remove redundant errors -main:1: error: Cannot find module named 'a' -b.py:1: error: Cannot find module named 'a' [case testModifyFileWhileBlockingErrorElsewhere] import a diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 3f7eecdbb06b..6022004d2260 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -1032,8 +1032,8 @@ main:2: error: Revealed type is 'contextlib.GeneratorContextManager[builtins.Non == a.py:1: error: Cannot find module named 'b' a.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) -main:2: error: Revealed type is 'contextlib.GeneratorContextManager[builtins.None]' a.py:3: error: Cannot find module named 'b' +main:2: error: Revealed type is 'contextlib.GeneratorContextManager[builtins.None]' == main:2: error: Revealed type is 'contextlib.GeneratorContextManager[builtins.None]' From 76f945fc47193da18240172e41cd3d9fa883a600 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Thu, 21 Dec 2017 12:34:20 -0800 Subject: [PATCH 02/15] Add more tests and stream blocking errors as well --- mypy/build.py | 30 ++++++++++++++++++++++----- mypy/errors.py | 2 +- mypy/main.py | 13 ++++-------- mypy/test/testerrorstream.py | 36 +++++++++++++++++++-------------- test-data/unit/errorstream.test | 28 ++++++++++++++++++++++++- 5 files changed, 78 insertions(+), 31 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 3ee76bdedbe9..ef3eaa4c9b2c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -25,9 +25,10 @@ import time from os.path import dirname, basename import errno +from functools import wraps from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List, - Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable) + Mapping, NamedTuple, Optional, Set, Tuple, TypeVar, Union, Callable) # Can't use TYPE_CHECKING because it's not in the Python 3.5.1 stdlib MYPY = False if MYPY: @@ -127,13 +128,32 @@ def is_source(self, file: MypyFile) -> bool: # be updated in place with newly computed cache data. See dmypy.py. SavedCache = Dict[str, Tuple['CacheMeta', MypyFile, Dict[Expression, Type]]] +F = TypeVar('F', bound=Callable[..., Any]) + +def flush_compile_errors(f: F) -> F: + """Catch and flush out any messages from a CompileError thrown in build.""" + @wraps(f) + def func(*args, **kwargs): + # type: (*Any, **Any) -> Any + try: + return f(*args, **kwargs) + except CompileError as e: + serious = not e.use_stdout + error_flush = kwargs.get('flush_errors', None) + if error_flush: + error_flush(e.messages[e.num_already_seen:], serious) + raise + return cast(F, func) + + +@flush_compile_errors def build(sources: List[BuildSource], options: Options, alt_lib_path: Optional[str] = None, bin_dir: Optional[str] = None, saved_cache: Optional[SavedCache] = None, - flush_errors: Optional[Callable[[List[str]], None]] = None, + flush_errors: Optional[Callable[[List[str], bool], None]] = None, plugin: Optional[Plugin] = None, ) -> BuildResult: """Analyze a program. @@ -539,7 +559,7 @@ def __init__(self, data_dir: str, version_id: str, plugin: Plugin, errors: Errors, - flush_errors: Optional[Callable[[List[str]], None]] = None, + flush_errors: Optional[Callable[[List[str], bool], None]] = None, saved_cache: Optional[SavedCache] = None, ) -> None: self.start_time = time.time() @@ -712,9 +732,9 @@ def add_stats(self, **kwds: Any) -> None: def stats_summary(self) -> Mapping[str, object]: return self.stats - def error_flush(self, msgs: List[str]) -> None: + def error_flush(self, msgs: List[str], serious: bool=False) -> None: if self.flush_errors: - self.flush_errors(msgs) + self.flush_errors(msgs, serious) def remove_cwd_prefix_from_path(p: str) -> str: diff --git a/mypy/errors.py b/mypy/errors.py index 0b7a16fc22f1..334cdad99853 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -428,7 +428,7 @@ def messages(self) -> List[str]: Use a form suitable for displaying to the user. """ - self.new_messages() + self.new_messages() # Updates formatted_messages as a side effect return self.formatted_messages def targets(self) -> Set[str]: diff --git a/mypy/main.py b/mypy/main.py index 733410d968fa..4916ec5e0151 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -61,13 +61,8 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: if args is None: args = sys.argv[1:] sources, options = process_options(args) - serious = False - errors = False - def flush_errors(a: List[str]) -> None: - nonlocal errors - if a: - errors = True + def flush_errors(a: List[str], serious: bool) -> None: f = sys.stderr if serious else sys.stdout try: for m in a: @@ -75,6 +70,7 @@ def flush_errors(a: List[str]) -> None: except BrokenPipeError: pass + serious = False try: res = type_check_only(sources, bin_dir, options, flush_errors) a = res.errors @@ -82,7 +78,6 @@ def flush_errors(a: List[str]) -> None: a = e.messages if not e.use_stdout: serious = True - flush_errors(a[e.num_already_seen:]) if options.warn_unused_configs and options.unused_configs: print("Warning: unused section(s) in %s: %s" % (options.config_file, @@ -91,7 +86,7 @@ def flush_errors(a: List[str]) -> None: if options.junit_xml: t1 = time.time() util.write_junit_xml(t1 - t0, serious, a, options.junit_xml) - if errors: + if a: sys.exit(1) @@ -121,7 +116,7 @@ def readlinkabs(link: str) -> str: def type_check_only(sources: List[BuildSource], bin_dir: Optional[str], options: Options, - flush_errors: Optional[Callable[[List[str]], None]]) -> BuildResult: + flush_errors: Optional[Callable[[List[str], bool], None]]) -> BuildResult: # Type-check the program and dependencies. return build.build(sources=sources, bin_dir=bin_dir, diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index e8bad150fb31..c44504eeda79 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -31,36 +31,42 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: options = Options() options.show_traceback = True - a = [] + logged_messages: List[str] = [] + real_messages: List[str] = [] - def flush_errors(msgs: List[str]) -> None: - nonlocal a + def flush_errors(msgs: List[str], serious: bool, is_real: bool=True) -> None: if msgs: - a.append('==== Errors flushed ====') - a += msgs + logged_messages.append('==== Errors flushed ====') + logged_messages.extend(msgs) + if is_real: + real_messages.extend(msgs) + plugin = ChainedPlugin(options, [LoggingPlugin(options, flush_errors), DefaultPlugin(options)]) sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))] try: - build.build(sources=sources, - options=options, - alt_lib_path=test_temp_dir, - flush_errors=flush_errors, - plugin=plugin) + res = build.build(sources=sources, + options=options, + alt_lib_path=test_temp_dir, + flush_errors=flush_errors, + plugin=plugin) + reported_messages = res.errors except CompileError as e: - a.append('==== Blocking error ====') - a += e.messages[e.num_already_seen:] + reported_messages = e.messages - assert_string_arrays_equal(testcase.output, a, + assert_string_arrays_equal(testcase.output, logged_messages, 'Invalid output ({}, line {})'.format( testcase.file, testcase.line)) + assert_string_arrays_equal(reported_messages, real_messages, + 'Streamed/reported mismatch ({}, line {})'.format( + testcase.file, testcase.line)) # Use a typechecking plugin to allow test cases to emit messages # during typechecking. This allows us to verify that error messages # from one SCC are printed before later ones are typechecked. class LoggingPlugin(Plugin): - def __init__(self, options: Options, log: Callable[[List[str]], None]) -> None: + def __init__(self, options: Options, log: Callable[[List[str], bool, bool], None]) -> None: super().__init__(options) self.log = log @@ -72,5 +78,5 @@ def get_function_hook(self, fullname: str) -> Optional[Callable[[FunctionContext def hook(self, ctx: FunctionContext) -> Type: assert(isinstance(ctx.context, CallExpr) and len(ctx.context.args) > 0 and isinstance(ctx.context.args[0], StrExpr)) - self.log([ctx.context.args[0].value]) + self.log([ctx.context.args[0].value], False, False) return ctx.default_return_type diff --git a/test-data/unit/errorstream.test b/test-data/unit/errorstream.test index b98eae2ab697..e32dd5f89cf3 100644 --- a/test-data/unit/errorstream.test +++ b/test-data/unit/errorstream.test @@ -39,8 +39,34 @@ import b [file b.py] import a break +1 / '' # won't get reported, after a blocker [out] ==== Errors flushed ==== a.py:1: error: Unsupported operand types for + ("int" and "str") -==== Blocking error ==== +==== Errors flushed ==== b.py:2: error: 'break' outside loop + +[case testCycles] +import a +[file a.py] +import b +1 + '' +def f() -> int: + reveal_type(b.x) + return b.x +y = 0 + 0 +[file b.py] +import a +def g() -> int: + reveal_type(a.y) + return a.y +1 / '' +x = 1 + 1 + +[out] +==== Errors flushed ==== +b.py:3: error: Revealed type is 'builtins.int' +b.py:5: error: Unsupported operand types for / ("int" and "str") +==== Errors flushed ==== +a.py:2: error: Unsupported operand types for + ("int" and "str") +a.py:4: error: Revealed type is 'builtins.int' From 310611d746930bb3279ef680278605b3a8722975 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Thu, 21 Dec 2017 12:51:05 -0800 Subject: [PATCH 03/15] Check for streaming errors matching in testcheck --- mypy/test/testcheck.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 65c3d89eae21..f51459404f46 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -169,15 +169,22 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) # Always set to none so we're forced to reread the module in incremental mode sources.append(BuildSource(program_path, module_name, None if incremental_step else program_text)) + streamed_messages = [] + + def flush_errors(msgs: List[str], serious: bool) -> None: + streamed_messages.extend(msgs) + res = None try: res = build.build(sources=sources, options=options, - alt_lib_path=test_temp_dir) + alt_lib_path=test_temp_dir, + flush_errors=flush_errors) a = res.errors except CompileError as e: a = e.messages a = normalize_error_messages(a) + streamed_messages = normalize_error_messages(streamed_messages) # Make sure error messages match if incremental_step == 0: @@ -197,6 +204,9 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) if output != a and self.update_data: update_testcase_output(testcase, a) assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line)) + assert_string_arrays_equal(a, streamed_messages, + "Streamed/reported error mismatch: " + + msg.format(testcase.file, testcase.line)) if incremental_step and res: if options.follow_imports == 'normal' and testcase.output is None: From 059de143442c8b11763dc4a95bac345a294c9df1 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 2 Jan 2018 11:58:06 -0800 Subject: [PATCH 04/15] Don't use variable type annotations --- mypy/test/testerrorstream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index c44504eeda79..a1558091afb8 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -31,8 +31,8 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: options = Options() options.show_traceback = True - logged_messages: List[str] = [] - real_messages: List[str] = [] + logged_messages = [] # type: List[str] + real_messages = [] # type: List[str] def flush_errors(msgs: List[str], serious: bool, is_real: bool=True) -> None: if msgs: From c9a5a969792552d2ef277d428e26284df006bd51 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 2 Jan 2018 15:20:58 -0800 Subject: [PATCH 05/15] Flush the file buffers after writing for better behavior when wrapped --- mypy/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/main.py b/mypy/main.py index 4916ec5e0151..6fa83815cd0b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -67,6 +67,7 @@ def flush_errors(a: List[str], serious: bool) -> None: try: for m in a: f.write(m + '\n') + f.flush() except BrokenPipeError: pass From 76bb7f867b52da4dc7a30a17e1442e4c9dabd712 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 2 Jan 2018 17:21:03 -0800 Subject: [PATCH 06/15] ditch the decorator --- mypy/build.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ef3eaa4c9b2c..dbe75e9277e8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -28,7 +28,7 @@ from functools import wraps from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List, - Mapping, NamedTuple, Optional, Set, Tuple, TypeVar, Union, Callable) + Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable) # Can't use TYPE_CHECKING because it's not in the Python 3.5.1 stdlib MYPY = False if MYPY: @@ -128,26 +128,7 @@ def is_source(self, file: MypyFile) -> bool: # be updated in place with newly computed cache data. See dmypy.py. SavedCache = Dict[str, Tuple['CacheMeta', MypyFile, Dict[Expression, Type]]] -F = TypeVar('F', bound=Callable[..., Any]) - -def flush_compile_errors(f: F) -> F: - """Catch and flush out any messages from a CompileError thrown in build.""" - @wraps(f) - def func(*args, **kwargs): - # type: (*Any, **Any) -> Any - try: - return f(*args, **kwargs) - except CompileError as e: - serious = not e.use_stdout - error_flush = kwargs.get('flush_errors', None) - if error_flush: - error_flush(e.messages[e.num_already_seen:], serious) - raise - return cast(F, func) - - -@flush_compile_errors def build(sources: List[BuildSource], options: Options, alt_lib_path: Optional[str] = None, @@ -175,6 +156,23 @@ def build(sources: List[BuildSource], flush_errors: optional function to flush errors after a file is processed plugin: optional plugin that overrides the configured one """ + try: + return _build(sources, options, alt_lib_path, bin_dir, saved_cache, flush_errors, plugin) + except CompileError as e: + serious = not e.use_stdout + if flush_errors: + flush_errors(e.messages[e.num_already_seen:], serious) + raise + + +def _build(sources: List[BuildSource], + options: Options, + alt_lib_path: Optional[str] = None, + bin_dir: Optional[str] = None, + saved_cache: Optional[SavedCache] = None, + flush_errors: Optional[Callable[[List[str], bool], None]] = None, + plugin: Optional[Plugin] = None, + ) -> BuildResult: # This seems the most reasonable place to tune garbage collection. gc.set_threshold(50000) From 5c9abec810acce51a7ee688b8cbb52532d130a6a Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 3 Jan 2018 09:38:16 -0800 Subject: [PATCH 07/15] Eliminate the indexing --- mypy/build.py | 2 +- mypy/errors.py | 27 +++++++++++++++------------ mypy/test/testerrorstream.py | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index dbe75e9277e8..cdf09df4c85f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -161,7 +161,7 @@ def build(sources: List[BuildSource], except CompileError as e: serious = not e.use_stdout if flush_errors: - flush_errors(e.messages[e.num_already_seen:], serious) + flush_errors(e.fresh_messages, serious) raise diff --git a/mypy/errors.py b/mypy/errors.py index 334cdad99853..3eb8c21e83a3 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -95,8 +95,8 @@ class Errors: # files were processed. error_info_map = None # type: Dict[str, List[ErrorInfo]] - # The size of error_info the last time that error messages were flushed - new_errors_start_map = None # type: Dict[str, int] + # Errors that haven't been flushed out yet + fresh_error_info_map = None # type: Dict[str, List[ErrorInfo]] # A cache of the formatted messages formatted_messages = None # type: List[str] @@ -150,7 +150,7 @@ def __init__(self, show_error_context: bool = False, def initialize(self) -> None: self.error_info_map = OrderedDict() - self.new_errors_start_map = defaultdict(int) + self.fresh_error_info_map = OrderedDict() self.import_ctx = [] self.formatted_messages = [] self.error_files = set() @@ -302,7 +302,9 @@ def report(self, def _add_error_info(self, info: ErrorInfo) -> None: if info.file not in self.error_info_map: self.error_info_map[info.file] = [] + self.fresh_error_info_map[info.file] = [] self.error_info_map[info.file].append(info) + self.fresh_error_info_map[info.file].append(info) def add_error_info(self, info: ErrorInfo) -> None: (file, line) = cast(Tuple[str, int], info.origin) # see issue 1855 @@ -365,12 +367,13 @@ def raise_error(self) -> None: """ # self.new_messages() will format all messages that haven't already # been returned from a new_module_messages() call. Count how many - # we've seen before that. + # we've seen before that so we can determine which are fresh. already_seen = len(self.formatted_messages) - raise CompileError(self.messages(), + messages = self.messages() + raise CompileError(messages, use_stdout=True, module_with_blocker=self.blocker_module(), - num_already_seen=already_seen) + fresh_messages=messages[already_seen:]) def format_messages(self, error_info: List[ErrorInfo]) -> List[str]: """Return a string list that represents the error messages. @@ -406,8 +409,8 @@ def new_file_messages(self, path: str) -> List[str]: """ if path not in self.error_info_map: return [] - msgs = self.format_messages(self.error_info_map[path][self.new_errors_start_map[path]:]) - self.new_errors_start_map[path] = len(self.error_info_map[path]) + msgs = self.format_messages(self.fresh_error_info_map[path]) + self.fresh_error_info_map[path] = [] self.formatted_messages += msgs return msgs @@ -518,7 +521,7 @@ def render_messages(self, errors: List[ErrorInfo]) -> List[Tuple[Optional[str], def sort_messages(self, errors: List[ErrorInfo]) -> List[ErrorInfo]: """Sort an array of error messages locally by line number. - I.e., sort a run of consecutive messages with the same file + I.e., sort a run of consecutive messages with the same context by line number, but otherwise retain the general ordering of the messages. """ @@ -574,18 +577,18 @@ class CompileError(Exception): use_stdout = False # Can be set in case there was a module with a blocking error module_with_blocker = None # type: Optional[str] - num_already_seen = 0 + fresh_messages = None # type: List[str] def __init__(self, messages: List[str], use_stdout: bool = False, module_with_blocker: Optional[str] = None, - num_already_seen: int = 0) -> None: + fresh_messages: Optional[List[str]] = None) -> None: super().__init__('\n'.join(messages)) self.messages = messages self.use_stdout = use_stdout self.module_with_blocker = module_with_blocker - self.num_already_seen = num_already_seen + self.fresh_messages = fresh_messages if fresh_messages is not None else messages class DecodeError(Exception): diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index a1558091afb8..c500eba79970 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -34,7 +34,7 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: logged_messages = [] # type: List[str] real_messages = [] # type: List[str] - def flush_errors(msgs: List[str], serious: bool, is_real: bool=True) -> None: + def flush_errors(msgs: List[str], serious: bool, is_real: bool = True) -> None: if msgs: logged_messages.append('==== Errors flushed ====') logged_messages.extend(msgs) From 844d5caab5b54f686e9807c9a826b3f367f7af45 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 3 Jan 2018 15:17:59 -0800 Subject: [PATCH 08/15] Drop plugin part of the test, do test cleanup --- mypy/build.py | 7 ++----- mypy/test/testerrorstream.py | 31 +++---------------------------- test-data/unit/errorstream.test | 22 ++-------------------- 3 files changed, 7 insertions(+), 53 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index cdf09df4c85f..3a5ab338257c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -135,7 +135,6 @@ def build(sources: List[BuildSource], bin_dir: Optional[str] = None, saved_cache: Optional[SavedCache] = None, flush_errors: Optional[Callable[[List[str], bool], None]] = None, - plugin: Optional[Plugin] = None, ) -> BuildResult: """Analyze a program. @@ -154,10 +153,9 @@ def build(sources: List[BuildSource], directories; if omitted, use '.' as the data directory saved_cache: optional dict with saved cache state for dmypy (read-write!) flush_errors: optional function to flush errors after a file is processed - plugin: optional plugin that overrides the configured one """ try: - return _build(sources, options, alt_lib_path, bin_dir, saved_cache, flush_errors, plugin) + return _build(sources, options, alt_lib_path, bin_dir, saved_cache, flush_errors) except CompileError as e: serious = not e.use_stdout if flush_errors: @@ -171,7 +169,6 @@ def _build(sources: List[BuildSource], bin_dir: Optional[str] = None, saved_cache: Optional[SavedCache] = None, flush_errors: Optional[Callable[[List[str], bool], None]] = None, - plugin: Optional[Plugin] = None, ) -> BuildResult: # This seems the most reasonable place to tune garbage collection. gc.set_threshold(50000) @@ -221,7 +218,7 @@ def _build(sources: List[BuildSource], reports = Reports(data_dir, options.report_dirs) source_set = BuildSourceSet(sources) errors = Errors(options.show_error_context, options.show_column_numbers) - plugin = plugin or load_plugins(options, errors) + plugin = load_plugins(options, errors) # Construct a build manager object to hold state during the build. # diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index c500eba79970..a27e9a4f5556 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -11,7 +11,6 @@ from mypy.build import BuildSource from mypy.errors import CompileError from mypy.options import Options -from mypy.plugin import Plugin, ChainedPlugin, DefaultPlugin, FunctionContext from mypy.nodes import CallExpr, StrExpr from mypy.types import Type @@ -34,22 +33,18 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: logged_messages = [] # type: List[str] real_messages = [] # type: List[str] - def flush_errors(msgs: List[str], serious: bool, is_real: bool = True) -> None: + def flush_errors(msgs: List[str], serious: bool) -> None: if msgs: logged_messages.append('==== Errors flushed ====') logged_messages.extend(msgs) - if is_real: - real_messages.extend(msgs) - - plugin = ChainedPlugin(options, [LoggingPlugin(options, flush_errors), DefaultPlugin(options)]) + real_messages.extend(msgs) sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))] try: res = build.build(sources=sources, options=options, alt_lib_path=test_temp_dir, - flush_errors=flush_errors, - plugin=plugin) + flush_errors=flush_errors) reported_messages = res.errors except CompileError as e: reported_messages = e.messages @@ -60,23 +55,3 @@ def flush_errors(msgs: List[str], serious: bool, is_real: bool = True) -> None: assert_string_arrays_equal(reported_messages, real_messages, 'Streamed/reported mismatch ({}, line {})'.format( testcase.file, testcase.line)) - - -# Use a typechecking plugin to allow test cases to emit messages -# during typechecking. This allows us to verify that error messages -# from one SCC are printed before later ones are typechecked. -class LoggingPlugin(Plugin): - def __init__(self, options: Options, log: Callable[[List[str], bool, bool], None]) -> None: - super().__init__(options) - self.log = log - - def get_function_hook(self, fullname: str) -> Optional[Callable[[FunctionContext], Type]]: - if fullname == 'log.log_checking': - return self.hook - return None - - def hook(self, ctx: FunctionContext) -> Type: - assert(isinstance(ctx.context, CallExpr) and len(ctx.context.args) > 0 and - isinstance(ctx.context.args[0], StrExpr)) - self.log([ctx.context.args[0].value], False, False) - return ctx.default_return_type diff --git a/test-data/unit/errorstream.test b/test-data/unit/errorstream.test index e32dd5f89cf3..6877a2098f88 100644 --- a/test-data/unit/errorstream.test +++ b/test-data/unit/errorstream.test @@ -1,36 +1,18 @@ --- Test cases for incremental error streaming. Each test case consists of two --- sections. --- The first section contains [case NAME] followed by the input code, while --- the second section contains [out] followed by the output from the checker. +-- Test cases for incremental error streaming. -- Each time errors are reported, '==== Errors flushed ====' is printed. --- The log.log_checking() function will immediately emit a message from --- a plugin when a call to it is checked, which can be used to verify that --- error messages are printed before doing later typechecking work. --- --- The input file name in errors is "file". --- --- Comments starting with "--" in this file will be ignored, except for lines --- starting with "----" that are not ignored. The first two dashes of these --- lines are interpreted as escapes and removed. [case testErrorStream] import b -[file log.py] -def log_checking(msg: str) -> None: ... [file a.py] 1 + '' [file b.py] import a -import log -log.log_checking('Checking b') # Make sure that a has been flushed before this is checked '' / 2 [out] ==== Errors flushed ==== a.py:1: error: Unsupported operand types for + ("int" and "str") ==== Errors flushed ==== -Checking b -==== Errors flushed ==== -b.py:4: error: Unsupported operand types for / ("str" and "int") +b.py:2: error: Unsupported operand types for / ("str" and "int") [case testBlockers] import b From c9d23551f76ad6da93c36bd090d73b7f7bdbf8f6 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 3 Jan 2018 15:23:17 -0800 Subject: [PATCH 09/15] Remove error_files --- mypy/errors.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index 3eb8c21e83a3..ce5647ec6c1d 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -104,9 +104,6 @@ class Errors: # Current error context: nested import context/stack, as a list of (path, line) pairs. import_ctx = None # type: List[Tuple[str, int]] - # Set of files with errors. - error_files = None # type: Set[str] - # Path name prefix that is removed from all paths, if set. ignore_prefix = None # type: str @@ -153,7 +150,6 @@ def initialize(self) -> None: self.fresh_error_info_map = OrderedDict() self.import_ctx = [] self.formatted_messages = [] - self.error_files = set() self.type_name = [None] self.function_or_member = [None] self.ignored_lines = OrderedDict() @@ -320,7 +316,6 @@ def add_error_info(self, info: ErrorInfo) -> None: return self.only_once_messages.add(info.message) self._add_error_info(info) - self.error_files.add(file) def generate_unused_ignore_notes(self, file: str) -> None: ignored_lines = self.ignored_lines[file] @@ -358,7 +353,7 @@ def blocker_module(self) -> Optional[str]: def is_errors_for_file(self, file: str) -> bool: """Are there any errors for the given file?""" - return file in self.error_files + return file in self.error_info_map def raise_error(self) -> None: """Raise a CompileError with the generated messages. From 699ba0b3d268cd676552965630e103a8c38b8fff Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 3 Jan 2018 15:27:14 -0800 Subject: [PATCH 10/15] fix a variable name --- mypy/errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index ce5647ec6c1d..1f184a6a75b2 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -417,8 +417,8 @@ def new_messages(self) -> List[str]: they first generated an error. """ msgs = [] - for key in self.error_info_map.keys(): - msgs.extend(self.new_file_messages(key)) + for path in self.error_info_map.keys(): + msgs.extend(self.new_file_messages(path)) return msgs def messages(self) -> List[str]: From 376982dfc69a500ef0d38b2d18521763fd1796d5 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Thu, 4 Jan 2018 14:59:12 -0800 Subject: [PATCH 11/15] Ditch the caching system, run all error aggregation through the callback --- mypy/build.py | 42 +++++++++++++++++++--------- mypy/errors.py | 53 +++++++++++++++--------------------- mypy/main.py | 13 +++++---- mypy/test/testcheck.py | 11 +------- mypy/test/testerrorstream.py | 16 ++++------- mypy/test/testgraph.py | 1 + 6 files changed, 65 insertions(+), 71 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 3a5ab338257c..5aa4cf931f06 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -81,7 +81,7 @@ def __init__(self, manager: 'BuildManager', graph: Graph) -> None: self.graph = graph self.files = manager.modules self.types = manager.all_types # Non-empty for tests only or if dumping deps - self.errors = manager.errors.messages() + self.errors = [] # type: List[str] # Filled in by build if desired class BuildSource: @@ -144,6 +144,11 @@ def build(sources: List[BuildSource], Return BuildResult if successful or only non-blocking errors were found; otherwise raise CompileError. + If a flush_errors callback is provided, all error messages will be + passed to it and the errors and messages fields of BuildResult and + CompileError (respectively) will be empty. Otherwise those fields will + report any error messages. + Args: sources: list of sources to build options: build options @@ -153,22 +158,34 @@ def build(sources: List[BuildSource], directories; if omitted, use '.' as the data directory saved_cache: optional dict with saved cache state for dmypy (read-write!) flush_errors: optional function to flush errors after a file is processed + """ + # If we were not given a flush_errors, we use one that will populate those + # fields for callers that want the traditional API. + messages = [] + + def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: + messages.extend(new_messages) + + flush_errors = flush_errors or default_flush_errors + try: - return _build(sources, options, alt_lib_path, bin_dir, saved_cache, flush_errors) + result = _build(sources, options, alt_lib_path, bin_dir, saved_cache, flush_errors) + result.errors = messages + return result except CompileError as e: serious = not e.use_stdout - if flush_errors: - flush_errors(e.fresh_messages, serious) + flush_errors(e.messages, serious) + e.messages = messages raise def _build(sources: List[BuildSource], options: Options, - alt_lib_path: Optional[str] = None, - bin_dir: Optional[str] = None, - saved_cache: Optional[SavedCache] = None, - flush_errors: Optional[Callable[[List[str], bool], None]] = None, + alt_lib_path: Optional[str], + bin_dir: Optional[str], + saved_cache: Optional[SavedCache], + flush_errors: Callable[[List[str], bool], None], ) -> BuildResult: # This seems the most reasonable place to tune garbage collection. gc.set_threshold(50000) @@ -539,7 +556,7 @@ class BuildManager: version_id: The current mypy version (based on commit id when possible) plugin: Active mypy plugin(s) errors: Used for reporting all errors - flush_errors: A function for optionally processing errors after each SCC + flush_errors: A function for processing errors after each SCC saved_cache: Dict with saved cache state for dmypy and fine-grained incremental mode (read-write!) stats: Dict with various instrumentation numbers @@ -554,7 +571,7 @@ def __init__(self, data_dir: str, version_id: str, plugin: Plugin, errors: Errors, - flush_errors: Optional[Callable[[List[str], bool], None]] = None, + flush_errors: Callable[[List[str], bool], None], saved_cache: Optional[SavedCache] = None, ) -> None: self.start_time = time.time() @@ -728,8 +745,7 @@ def stats_summary(self) -> Mapping[str, object]: return self.stats def error_flush(self, msgs: List[str], serious: bool=False) -> None: - if self.flush_errors: - self.flush_errors(msgs, serious) + self.flush_errors(msgs, serious) def remove_cwd_prefix_from_path(p: str) -> str: @@ -2520,7 +2536,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No for id in stale: graph[id].finish_passes() graph[id].generate_unused_ignore_notes() - manager.error_flush(manager.errors.new_file_messages(graph[id].xpath)) + manager.error_flush(manager.errors.file_messages(graph[id].xpath)) graph[id].write_cache() graph[id].mark_as_rechecked() diff --git a/mypy/errors.py b/mypy/errors.py index 1f184a6a75b2..afbf39e84044 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -95,11 +95,8 @@ class Errors: # files were processed. error_info_map = None # type: Dict[str, List[ErrorInfo]] - # Errors that haven't been flushed out yet - fresh_error_info_map = None # type: Dict[str, List[ErrorInfo]] - - # A cache of the formatted messages - formatted_messages = None # type: List[str] + # Files that we have reported the errors for + flushed_files = None # type: Set[str] # Current error context: nested import context/stack, as a list of (path, line) pairs. import_ctx = None # type: List[Tuple[str, int]] @@ -147,9 +144,8 @@ def __init__(self, show_error_context: bool = False, def initialize(self) -> None: self.error_info_map = OrderedDict() - self.fresh_error_info_map = OrderedDict() + self.flushed_files = set() self.import_ctx = [] - self.formatted_messages = [] self.type_name = [None] self.function_or_member = [None] self.ignored_lines = OrderedDict() @@ -298,9 +294,7 @@ def report(self, def _add_error_info(self, info: ErrorInfo) -> None: if info.file not in self.error_info_map: self.error_info_map[info.file] = [] - self.fresh_error_info_map[info.file] = [] self.error_info_map[info.file].append(info) - self.fresh_error_info_map[info.file].append(info) def add_error_info(self, info: ErrorInfo) -> None: (file, line) = cast(Tuple[str, int], info.origin) # see issue 1855 @@ -361,14 +355,10 @@ def raise_error(self) -> None: Render the messages suitable for displaying. """ # self.new_messages() will format all messages that haven't already - # been returned from a new_module_messages() call. Count how many - # we've seen before that so we can determine which are fresh. - already_seen = len(self.formatted_messages) - messages = self.messages() - raise CompileError(messages, + # been returned from a new_module_messages() call. + raise CompileError(self.new_messages(), use_stdout=True, - module_with_blocker=self.blocker_module(), - fresh_messages=messages[already_seen:]) + module_with_blocker=self.blocker_module()) def format_messages(self, error_info: List[ErrorInfo]) -> List[str]: """Return a string list that represents the error messages. @@ -394,20 +384,15 @@ def format_messages(self, error_info: List[ErrorInfo]) -> List[str]: a.append(s) return a - def new_file_messages(self, path: str) -> List[str]: + def file_messages(self, path: str) -> List[str]: """Return a string list of new error messages from a given file. Use a form suitable for displaying to the user. - Formatted messages are cached in the order they are generated - by new_file_messages() in order to have consistency in output - between incrementally generated messages and .messages() calls. """ if path not in self.error_info_map: return [] - msgs = self.format_messages(self.fresh_error_info_map[path]) - self.fresh_error_info_map[path] = [] - self.formatted_messages += msgs - return msgs + self.flushed_files.add(path) + return self.format_messages(self.error_info_map[path]) def new_messages(self) -> List[str]: """Return a string list of new error messages. @@ -418,7 +403,8 @@ def new_messages(self) -> List[str]: """ msgs = [] for path in self.error_info_map.keys(): - msgs.extend(self.new_file_messages(path)) + if path not in self.flushed_files: + msgs.extend(self.file_messages(path)) return msgs def messages(self) -> List[str]: @@ -426,8 +412,10 @@ def messages(self) -> List[str]: Use a form suitable for displaying to the user. """ - self.new_messages() # Updates formatted_messages as a side effect - return self.formatted_messages + msgs = [] + for path in self.error_info_map.keys(): + msgs.extend(self.file_messages(path)) + return msgs def targets(self) -> Set[str]: """Return a set of all targets that contain errors.""" @@ -566,24 +554,27 @@ class CompileError(Exception): It can be a parse, semantic analysis, type check or other compilation-related error. + + CompileErrors raised from an errors object carry all of the + messages that have not been reported out by error streaming. + This is patched up by build.build to contain either all error + messages (if errors were streamed) or none (if they were not). + """ messages = None # type: List[str] use_stdout = False # Can be set in case there was a module with a blocking error module_with_blocker = None # type: Optional[str] - fresh_messages = None # type: List[str] def __init__(self, messages: List[str], use_stdout: bool = False, - module_with_blocker: Optional[str] = None, - fresh_messages: Optional[List[str]] = None) -> None: + module_with_blocker: Optional[str] = None) -> None: super().__init__('\n'.join(messages)) self.messages = messages self.use_stdout = use_stdout self.module_with_blocker = module_with_blocker - self.fresh_messages = fresh_messages if fresh_messages is not None else messages class DecodeError(Exception): diff --git a/mypy/main.py b/mypy/main.py index 6fa83815cd0b..35279c820253 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -62,21 +62,22 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] sources, options = process_options(args) + messages = [] + def flush_errors(a: List[str], serious: bool) -> None: + messages.extend(a) f = sys.stderr if serious else sys.stdout try: for m in a: f.write(m + '\n') f.flush() except BrokenPipeError: - pass + sys.exit(1) serious = False try: - res = type_check_only(sources, bin_dir, options, flush_errors) - a = res.errors + type_check_only(sources, bin_dir, options, flush_errors) except CompileError as e: - a = e.messages if not e.use_stdout: serious = True if options.warn_unused_configs and options.unused_configs: @@ -86,8 +87,8 @@ def flush_errors(a: List[str], serious: bool) -> None: file=sys.stderr) if options.junit_xml: t1 = time.time() - util.write_junit_xml(t1 - t0, serious, a, options.junit_xml) - if a: + util.write_junit_xml(t1 - t0, serious, messages, options.junit_xml) + if messages: sys.exit(1) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index f51459404f46..0bdf645e25c7 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -169,22 +169,16 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) # Always set to none so we're forced to reread the module in incremental mode sources.append(BuildSource(program_path, module_name, None if incremental_step else program_text)) - streamed_messages = [] - - def flush_errors(msgs: List[str], serious: bool) -> None: - streamed_messages.extend(msgs) res = None try: res = build.build(sources=sources, options=options, - alt_lib_path=test_temp_dir, - flush_errors=flush_errors) + alt_lib_path=test_temp_dir) a = res.errors except CompileError as e: a = e.messages a = normalize_error_messages(a) - streamed_messages = normalize_error_messages(streamed_messages) # Make sure error messages match if incremental_step == 0: @@ -204,9 +198,6 @@ def flush_errors(msgs: List[str], serious: bool) -> None: if output != a and self.update_data: update_testcase_output(testcase, a) assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line)) - assert_string_arrays_equal(a, streamed_messages, - "Streamed/reported error mismatch: " + - msg.format(testcase.file, testcase.line)) if incremental_step and res: if options.follow_imports == 'normal' and testcase.output is None: diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index a27e9a4f5556..0c2a96d0e423 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -31,27 +31,21 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: options.show_traceback = True logged_messages = [] # type: List[str] - real_messages = [] # type: List[str] def flush_errors(msgs: List[str], serious: bool) -> None: if msgs: logged_messages.append('==== Errors flushed ====') logged_messages.extend(msgs) - real_messages.extend(msgs) sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))] try: - res = build.build(sources=sources, - options=options, - alt_lib_path=test_temp_dir, - flush_errors=flush_errors) - reported_messages = res.errors + build.build(sources=sources, + options=options, + alt_lib_path=test_temp_dir, + flush_errors=flush_errors) except CompileError as e: - reported_messages = e.messages + pass assert_string_arrays_equal(testcase.output, logged_messages, 'Invalid output ({}, line {})'.format( testcase.file, testcase.line)) - assert_string_arrays_equal(reported_messages, real_messages, - 'Streamed/reported mismatch ({}, line {})'.format( - testcase.file, testcase.line)) diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index dbbe4872aa75..e47234925b0b 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -49,6 +49,7 @@ def _make_manager(self) -> BuildManager: version_id=__version__, plugin=Plugin(options), errors=errors, + flush_errors=lambda msgs, serious: None, ) return manager From caa84775620d08c89484ace74cc96af4dc9b3100 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Thu, 4 Jan 2018 15:58:04 -0800 Subject: [PATCH 12/15] Get rid of the .messages() method --- mypy/errors.py | 13 ++----------- mypy/server/update.py | 13 +++++++------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index afbf39e84044..c8071e3545bc 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -407,16 +407,6 @@ def new_messages(self) -> List[str]: msgs.extend(self.file_messages(path)) return msgs - def messages(self) -> List[str]: - """Return a string list that represents the error messages. - - Use a form suitable for displaying to the user. - """ - msgs = [] - for path in self.error_info_map.keys(): - msgs.extend(self.file_messages(path)) - return msgs - def targets(self) -> Set[str]: """Return a set of all targets that contain errors.""" # TODO: Make sure that either target is always defined or that not being defined @@ -603,7 +593,8 @@ def report_internal_error(err: Exception, file: Optional[str], line: int, # Dump out errors so far, they often provide a clue. # But catch unexpected errors rendering them. try: - for msg in errors.messages(): + errors.flushed_files = set() # Print out already flushed messages too + for msg in errors.new_messages(): print(msg) except Exception as e: print("Failed to dump errors:", repr(e), file=sys.stderr) diff --git a/mypy/server/update.py b/mypy/server/update.py index 73b224f43379..d6cb0707b846 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -195,9 +195,9 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], result = update_single_isolated(module, path, manager, previous_modules) if isinstance(result, BlockedUpdate): # Blocking error -- just give up - module, path, remaining = result + module, path, remaining, errors = result self.previous_modules = get_module_to_path_map(manager) - return manager.errors.messages(), remaining, (module, path), True + return errors, remaining, (module, path), True assert isinstance(result, NormalUpdate) # Work around #4124 module, path, remaining, tree, graph = result @@ -230,7 +230,7 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], self.previous_modules = get_module_to_path_map(manager) self.type_maps = extract_type_maps(graph) - return manager.errors.messages(), remaining, (module, path), False + return manager.errors.new_messages(), remaining, (module, path), False def mark_all_meta_as_memory_only(graph: Dict[str, State], @@ -271,7 +271,8 @@ def get_all_dependencies(manager: BuildManager, graph: Dict[str, State], # are similar to NormalUpdate (but there are fewer). BlockedUpdate = NamedTuple('BlockedUpdate', [('module', str), ('path', str), - ('remaining', List[Tuple[str, str]])]) + ('remaining', List[Tuple[str, str]]), + ('messages', List[str])]) UpdateResult = Union[NormalUpdate, BlockedUpdate] @@ -318,7 +319,7 @@ def update_single_isolated(module: str, remaining_modules = [(module, path)] else: remaining_modules = [] - return BlockedUpdate(err.module_with_blocker, path, remaining_modules) + return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages) if not os.path.isfile(path): graph = delete_module(module, graph, manager) @@ -353,7 +354,7 @@ def update_single_isolated(module: str, manager.modules.clear() manager.modules.update(old_modules) del graph[module] - return BlockedUpdate(module, path, remaining_modules) + return BlockedUpdate(module, path, remaining_modules, err.messages) state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() From 92dfc2daaa426ef51240768f149be4d660b82ef4 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Thu, 4 Jan 2018 17:00:14 -0800 Subject: [PATCH 13/15] Key errors based on origin file --- mypy/build.py | 1 - mypy/errors.py | 17 ++++++++++------- test-data/unit/check-kwargs.test | 10 ++++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5aa4cf931f06..41257083b6e0 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -253,7 +253,6 @@ def _build(sources: List[BuildSource], try: graph = dispatch(sources, manager) - manager.error_flush(manager.errors.new_messages()) return BuildResult(manager, graph) finally: manager.log("Build finished in %.3f seconds with %d modules, and %d errors" % diff --git a/mypy/errors.py b/mypy/errors.py index c8071e3545bc..00a16f0c34af 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -51,6 +51,9 @@ class ErrorInfo: # Only report this particular messages once per program. only_once = False + # Actual origin of the error message + origin = None # type: Tuple[str, int] + # Fine-grained incremental target where this was reported target = None # type: Optional[str] @@ -291,13 +294,13 @@ def report(self, target=self.current_target()) self.add_error_info(info) - def _add_error_info(self, info: ErrorInfo) -> None: - if info.file not in self.error_info_map: - self.error_info_map[info.file] = [] - self.error_info_map[info.file].append(info) + def _add_error_info(self, file: str, info: ErrorInfo) -> None: + if file not in self.error_info_map: + self.error_info_map[file] = [] + self.error_info_map[file].append(info) def add_error_info(self, info: ErrorInfo) -> None: - (file, line) = cast(Tuple[str, int], info.origin) # see issue 1855 + file, line = info.origin if not info.blocker: # Blockers cannot be ignored if file in self.ignored_lines and line in self.ignored_lines[file]: # Annotation requests us to ignore all errors on this line. @@ -309,7 +312,7 @@ def add_error_info(self, info: ErrorInfo) -> None: if info.message in self.only_once_messages: return self.only_once_messages.add(info.message) - self._add_error_info(info) + self._add_error_info(file, info) def generate_unused_ignore_notes(self, file: str) -> None: ignored_lines = self.ignored_lines[file] @@ -319,7 +322,7 @@ def generate_unused_ignore_notes(self, file: str) -> None: info = ErrorInfo(self.import_context(), file, self.current_module(), None, None, line, -1, 'note', "unused 'type: ignore' comment", False, False) - self._add_error_info(info) + self._add_error_info(file, info) def is_typeshed_file(self, file: str) -> bool: # gross, but no other clear way to tell diff --git a/test-data/unit/check-kwargs.test b/test-data/unit/check-kwargs.test index f65b626ace8f..14a398b54f1d 100644 --- a/test-data/unit/check-kwargs.test +++ b/test-data/unit/check-kwargs.test @@ -390,8 +390,14 @@ A.B(x=1) # E: Unexpected keyword argument "x" for "B" [case testUnexpectedMethodKwargFromOtherModule] import m -m.A(x=1) # E: Unexpected keyword argument "x" for "A" +m.A(x=1) [file m.py] +1+'asdf' class A: - def __init__(self) -> None: # N: "A" defined here + def __init__(self) -> None: pass + +[out] +tmp/m.py:1: error: Unsupported operand types for + ("int" and "str") +main:2: error: Unexpected keyword argument "x" for "A" +tmp/m.py:3: note: "A" defined here From cc64198041dcd6698535522c33f08a79b360d9da Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Mon, 8 Jan 2018 16:26:40 -0800 Subject: [PATCH 14/15] Perform various cleanups --- mypy/build.py | 10 +++++----- mypy/errors.py | 4 ++-- mypy/main.py | 8 ++++---- mypy/options.py | 1 + mypy/test/testerrorstream.py | 2 +- test-data/unit/check-kwargs.test | 4 +++- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 41257083b6e0..f3f369189086 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -25,7 +25,6 @@ import time from os.path import dirname, basename import errno -from functools import wraps from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List, Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable) @@ -174,6 +173,10 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: result.errors = messages return result except CompileError as e: + # CompileErrors raised from an errors object carry all of the + # messages that have not been reported out by error streaming. + # Patch it up to contain either none or all none of the messages, + # depending on whether we are flushing errors. serious = not e.use_stdout flush_errors(e.messages, serious) e.messages = messages @@ -743,9 +746,6 @@ def add_stats(self, **kwds: Any) -> None: def stats_summary(self) -> Mapping[str, object]: return self.stats - def error_flush(self, msgs: List[str], serious: bool=False) -> None: - self.flush_errors(msgs, serious) - def remove_cwd_prefix_from_path(p: str) -> str: """Remove current working directory prefix from p, if present. @@ -2535,7 +2535,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No for id in stale: graph[id].finish_passes() graph[id].generate_unused_ignore_notes() - manager.error_flush(manager.errors.file_messages(graph[id].xpath)) + manager.flush_errors(manager.errors.file_messages(graph[id].xpath), False) graph[id].write_cache() graph[id].mark_as_rechecked() diff --git a/mypy/errors.py b/mypy/errors.py index 00a16f0c34af..189dfb2c3af3 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -295,6 +295,7 @@ def report(self, self.add_error_info(info) def _add_error_info(self, file: str, info: ErrorInfo) -> None: + assert file not in self.flushed_files if file not in self.error_info_map: self.error_info_map[file] = [] self.error_info_map[file].append(info) @@ -358,7 +359,7 @@ def raise_error(self) -> None: Render the messages suitable for displaying. """ # self.new_messages() will format all messages that haven't already - # been returned from a new_module_messages() call. + # been returned from a file_messages() call. raise CompileError(self.new_messages(), use_stdout=True, module_with_blocker=self.blocker_module()) @@ -596,7 +597,6 @@ def report_internal_error(err: Exception, file: Optional[str], line: int, # Dump out errors so far, they often provide a clue. # But catch unexpected errors rendering them. try: - errors.flushed_files = set() # Print out already flushed messages too for msg in errors.new_messages(): print(msg) except Exception as e: diff --git a/mypy/main.py b/mypy/main.py index 35279c820253..5037224f0d5c 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -64,12 +64,12 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: messages = [] - def flush_errors(a: List[str], serious: bool) -> None: - messages.extend(a) + def flush_errors(new_messages: List[str], serious: bool) -> None: + messages.extend(new_messages) f = sys.stderr if serious else sys.stdout try: - for m in a: - f.write(m + '\n') + for msg in new_messages: + f.write(msg + '\n') f.flush() except BrokenPipeError: sys.exit(1) diff --git a/mypy/options.py b/mypy/options.py index dd9bfe08c095..0a826f16a405 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -34,6 +34,7 @@ class Options: "show_none_errors", "warn_no_return", "warn_return_any", + "warn_unused_ignores", "ignore_errors", "strict_boolean", "no_implicit_optional", diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index 0c2a96d0e423..55e4e3d9228e 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -44,7 +44,7 @@ def flush_errors(msgs: List[str], serious: bool) -> None: alt_lib_path=test_temp_dir, flush_errors=flush_errors) except CompileError as e: - pass + assert e.messages == [] assert_string_arrays_equal(testcase.output, logged_messages, 'Invalid output ({}, line {})'.format( diff --git a/test-data/unit/check-kwargs.test b/test-data/unit/check-kwargs.test index 14a398b54f1d..d7be6685ef88 100644 --- a/test-data/unit/check-kwargs.test +++ b/test-data/unit/check-kwargs.test @@ -396,8 +396,10 @@ m.A(x=1) class A: def __init__(self) -> None: pass - [out] +-- Note that the messages appear "out of order" because the m.py:3 +-- message is really an attachment to the main:2 error and should be +-- reported with it. tmp/m.py:1: error: Unsupported operand types for + ("int" and "str") main:2: error: Unexpected keyword argument "x" for "A" tmp/m.py:3: note: "A" defined here From 548078c26de89fc432c9f0f70c8a50350179f223 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 9 Jan 2018 15:16:16 -0800 Subject: [PATCH 15/15] update warn_unused_ignores documentation --- docs/source/config_file.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index a6b64578ea29..9c5b935bf25c 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -74,9 +74,6 @@ The following global flags may only be set in the global section - ``warn_redundant_casts`` (Boolean, default False) warns about casting an expression to its inferred type. -- ``warn_unused_ignores`` (Boolean, default False) warns about - unneeded ``# type: ignore`` comments. - - ``warn_unused_configs`` (Boolean, default False) warns about per-module sections in the config file that didn't match any files processed in the current run. @@ -203,6 +200,9 @@ overridden by the pattern sections matching the module name. returning a value with type ``Any`` from a function declared with a non- ``Any`` return type. +- ``warn_unused_ignores`` (Boolean, default False) warns about + unneeded ``# type: ignore`` comments. + - ``strict_boolean`` (Boolean, default False) makes using non-boolean expressions in conditions an error.