Fix staleness on changed follow imports options (#20773)

ilevkivskyi · web-flow · commit fe6fe1d047a8 · 2026-02-12T16:02:29.000Z
This fixes the coarse grained part of #7777. I do not change semantics for daemon for two reasons: * Performance penalty from this fix will be minor for regular incremental, but may be well visible for the daemon. * And (more importantly) and just don't want to open this another can of worms (yet). Some notes: * The general logic is straightforward, invalidate dependent modules, if import options for their dependencies changed. * It looks like the problem only affects suppressed dependencies, in all other situations we either already invalidate because something gets added to/removed from dependencies (e.g. when `follow_imports = skip <-> normal`), or there is no effect on the caller/importer (e.g. when `follow_imports = normal <-> silent`). * This requires another entry for `CacheMeta` but good new is I have an idea how to get rid of another entry I added in one of recent PRs. Essentially I will need to record early (import) errors anyway to send to parallel workers after we switch to parallel parsing, and if we do this we won't need the `imports_ignored` entry.
diff --git a/mypy/build.py b/mypy/build.py
@@ -895,6 +895,9 @@ def __init__(
         # raw parsed trees not analyzed with mypy. We use these to find absolute
         # location of a symbol used as a location for an error message.
         self.extra_trees: dict[str, MypyFile] = {}
+        # Snapshot of import-related options per module. We record these even for
+        # suppressed imports, since they can affect errors in the callers.
+        self.import_options: dict[str, dict[str, object]] = {}
         # Cache for transitive dependency check (expensive).
         self.transitive_deps_cache: dict[tuple[int, int], bool] = {}
 
@@ -1871,6 +1874,7 @@ def write_cache(
     tree: MypyFile,
     dependencies: list[str],
     suppressed: list[str],
+    suppressed_deps_opts: bytes,
     imports_ignored: dict[int, list[str]],
     dep_prios: list[int],
     dep_lines: list[int],
@@ -1989,6 +1993,7 @@ def write_cache(
         suppressed=suppressed,
         imports_ignored=imports_ignored,
         options=options_snapshot(id, manager),
+        suppressed_deps_opts=suppressed_deps_opts,
         dep_prios=dep_prios,
         dep_lines=dep_lines,
         interface_hash=interface_hash,
@@ -2274,6 +2279,7 @@ def new_state(
             import_context = []
         id = id or "__main__"
         options = manager.options.clone_for_module(id)
+        manager.import_options[id] = options.dep_import_options()
 
         ignore_all = False
         if not path and source is None:
@@ -2571,7 +2577,16 @@ def is_fresh(self) -> bool:
         # self.meta.dependencies when a dependency is dropped due to
         # suppression by silent mode.  However, when a suppressed
         # dependency is added back we find out later in the process.
-        return self.meta is not None and self.dependencies == self.meta.dependencies
+        # Additionally, we need to verify that import following options are
+        # same for suppressed dependencies, even if the first check is OK.
+        return (
+            self.meta is not None
+            and self.dependencies == self.meta.dependencies
+            and (
+                self.options.fine_grained_incremental
+                or self.meta.suppressed_deps_opts == self.suppressed_deps_opts()
+            )
+        )
 
     def mark_as_rechecked(self) -> None:
         """Marks this module as having been fully re-analyzed by the type-checker."""
@@ -3020,6 +3035,15 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None:
             merge_dependencies(self.compute_fine_grained_deps(), deps)
             type_state.update_protocol_deps(deps)
 
+    def suppressed_deps_opts(self) -> bytes:
+        return json_dumps(
+            {
+                dep: self.manager.import_options[dep]
+                for dep in self.suppressed
+                if self.priorities.get(dep) != PRI_INDIRECT
+            }
+        )
+
     def write_cache(self) -> tuple[CacheMeta, str] | None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
         # We don't support writing cache files in fine-grained incremental mode.
@@ -3051,6 +3075,7 @@ def write_cache(self) -> tuple[CacheMeta, str] | None:
             self.tree,
             list(self.dependencies),
             list(self.suppressed),
+            self.suppressed_deps_opts(),
             self.imports_ignored,
             dep_prios,
             dep_lines,
@@ -3126,10 +3151,8 @@ def generate_unused_ignore_notes(self) -> None:
             self.options.warn_unused_ignores
             or codes.UNUSED_IGNORE in self.options.enabled_error_codes
         ) and codes.UNUSED_IGNORE not in self.options.disabled_error_codes:
-            # If this file was initially loaded from the cache, it may have suppressed
-            # dependencies due to imports with ignores on them. We need to generate
-            # those errors to avoid spuriously flagging them as unused ignores.
-            if self.meta:
+            # We only need this for the daemon, regular incremental does this unconditionally.
+            if self.meta and self.options.fine_grained_incremental:
                 self.verify_dependencies(suppressed_only=True)
             self.manager.errors.generate_unused_ignore_errors(self.xpath)
 
@@ -3710,20 +3733,22 @@ def load_graph(
         #   but A's cached *indirect* dependency on C is wrong.
         dependencies = [dep for dep in st.dependencies if st.priorities.get(dep) != PRI_INDIRECT]
         if not manager.use_fine_grained_cache():
-            # TODO: Ideally we could skip here modules that appeared in st.suppressed
-            # because they are not in build with `follow-imports=skip`.
-            # This way we could avoid overhead of cloning options in `State.__init__()`
-            # below to get the option value. This is quite minor performance loss however.
             added = [dep for dep in st.suppressed if find_module_simple(dep, manager)]
         else:
             # During initial loading we don't care about newly added modules,
             # they will be taken care of during fine-grained update. See also
-            # comment about this in `State.__init__()`.
+            # comment about this in `State.new_state()`.
             added = []
         for dep in st.ancestors + dependencies + st.suppressed:
             ignored = dep in st.suppressed_set and dep not in entry_points
             if ignored and dep not in added:
                 manager.missing_modules.add(dep)
+                # TODO: for now we skip this in the daemon as a performance optimization.
+                # This however creates a correctness issue, see #7777 and State.is_fresh().
+                if not manager.use_fine_grained_cache():
+                    manager.import_options[dep] = manager.options.clone_for_module(
+                        dep
+                    ).dep_import_options()
             elif dep not in graph:
                 try:
                     if dep in st.ancestors:
@@ -4129,6 +4154,9 @@ def process_stale_scc(
     t2 = time.time()
     stale = scc
     for id in stale:
+        # Re-generate import errors in case this module was loaded from the cache.
+        if graph[id].meta:
+            graph[id].verify_dependencies(suppressed_only=True)
         # We may already have parsed the module, or not.
         # If the former, parse_file() is a no-op.
         graph[id].parse_file()
diff --git a/mypy/cache.py b/mypy/cache.py
@@ -69,7 +69,7 @@
 from mypy_extensions import u8
 
 # High-level cache layout format
-CACHE_VERSION: Final = 3
+CACHE_VERSION: Final = 4
 
 SerializedError: _TypeAlias = tuple[str | None, int | str, int, int, int, str, str, str | None]
 
@@ -91,6 +91,7 @@ def __init__(
         suppressed: list[str],
         imports_ignored: dict[int, list[str]],
         options: dict[str, object],
+        suppressed_deps_opts: bytes,
         dep_prios: list[int],
         dep_lines: list[int],
         dep_hashes: list[bytes],
@@ -112,6 +113,7 @@ def __init__(
         self.suppressed = suppressed  # dependencies that weren't imported
         self.imports_ignored = imports_ignored  # type ignore codes by line
         self.options = options  # build options snapshot
+        self.suppressed_deps_opts = suppressed_deps_opts  # hash of import-related options
         # dep_prios and dep_lines are both aligned with dependencies + suppressed
         self.dep_prios = dep_prios
         self.dep_lines = dep_lines
@@ -136,6 +138,7 @@ def serialize(self) -> dict[str, Any]:
             "suppressed": self.suppressed,
             "imports_ignored": {str(line): codes for line, codes in self.imports_ignored.items()},
             "options": self.options,
+            "suppressed_deps_opts": self.suppressed_deps_opts.hex(),
             "dep_prios": self.dep_prios,
             "dep_lines": self.dep_lines,
             "dep_hashes": [dep.hex() for dep in self.dep_hashes],
@@ -164,6 +167,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None:
                     int(line): codes for line, codes in meta["imports_ignored"].items()
                 },
                 options=meta["options"],
+                suppressed_deps_opts=bytes.fromhex(meta["suppressed_deps_opts"]),
                 dep_prios=meta["dep_prios"],
                 dep_lines=meta["dep_lines"],
                 dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]],
@@ -191,6 +195,7 @@ def write(self, data: WriteBuffer) -> None:
             write_int(data, line)
             write_str_list(data, codes)
         write_json(data, self.options)
+        write_bytes(data, self.suppressed_deps_opts)
         write_int_list(data, self.dep_prios)
         write_int_list(data, self.dep_lines)
         write_bytes_list(data, self.dep_hashes)
@@ -220,6 +225,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None:
                     read_int(data): read_str_list(data) for _ in range(read_int_bare(data))
                 },
                 options=read_json(data),
+                suppressed_deps_opts=read_bytes(data),
                 dep_prios=read_int_list(data),
                 dep_lines=read_int_list(data),
                 dep_hashes=read_bytes_list(data),
diff --git a/mypy/options.py b/mypy/options.py
@@ -616,3 +616,11 @@ def select_options_affecting_cache(self) -> dict[str, object]:
                 val = sorted([code.code for code in val])
             result[opt] = val
         return result
+
+    def dep_import_options(self) -> dict[str, object]:
+        # These are options that can affect dependent modules as well.
+        return {
+            "ignore_missing_imports": self.ignore_missing_imports,
+            "follow_imports": self.follow_imports,
+            "follow_imports_for_stubs": self.follow_imports_for_stubs,
+        }
diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test
@@ -7797,6 +7797,104 @@ tmp/b.py:5: note: "lol" of "BB" defined here
 tmp/a.py:2: error: Unexpected keyword argument "uhhhh" for "lol" of "BB"
 tmp/b.py:5: note: "lol" of "BB" defined here
 
+[case testMissingImportUnIgnoredInConfig]
+# flags: --config-file tmp/mypy.ini
+from foo import bar
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = True
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = False
+[out]
+[out2]
+main:2: error: Cannot find implementation or library stub for module named "foo"
+main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testMissingImportUnIgnoredInConfig2]
+# flags: --config-file tmp/mypy.ini
+from foo import bar
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = False
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = True
+[out]
+main:2: error: Cannot find implementation or library stub for module named "foo"
+main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+[out2]
+
+[case testMissingImportUnIgnoredInConfig3]
+# flags: --config-file tmp/mypy.ini
+from foo import bar
+[file foo.py]
+bar = 1
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+follow_imports = skip
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+follow_imports = error
+[out]
+[out2]
+main:2: error: Import of "foo" ignored
+main:2: note: (Using --follow-imports=error, module not passed on command line)
+
+[case testMissingImportUnIgnoredInConfig4]
+# flags: --config-file tmp/mypy.ini
+from foo import bar
+[file foo.py]
+bar = 1
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+follow_imports = error
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+follow_imports = skip
+[out]
+main:2: error: Import of "foo" ignored
+main:2: note: (Using --follow-imports=error, module not passed on command line)
+[out2]
+
+[case testMissingImportUnIgnoredInConfig5]
+# flags: --config-file tmp/mypy.ini --warn-unused-ignores
+from foo import bar  # type: ignore[import-not-found]
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = True
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = False
+[out]
+main:2: error: Unused "type: ignore" comment
+[out2]
+
+[case testMissingImportUnIgnoredInConfig6]
+# flags: --config-file tmp/mypy.ini  --warn-unused-ignores
+from foo import bar  # type: ignore[import-not-found]
+[file mypy.ini]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = False
+[file mypy.ini.2]
+\[mypy]
+\[mypy-foo]
+ignore_missing_imports = True
+[out]
+[out2]
+main:2: error: Unused "type: ignore" comment
+
 [case testIndirectDependencyReorderModulesNoCrash]
 import a
 [file a.py]
diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test
@@ -3262,3 +3262,30 @@ import b  # type: ignore[import-not-found]
 [out]
 [out2]
 tmp/a.py:1: error: Unused "type: ignore" comment
+
+[case testTypeIgnoredImportsWorkWithCacheIncremental3]
+# flags: --warn-unused-ignores
+import a
+[file a.py]
+import b  # type: ignore[import-not-found]
+[file b.py]
+[delete b.py.2]
+[out]
+tmp/a.py:1: error: Unused "type: ignore" comment
+[out2]
+
+[case testImportErrorStaleLoadedFromCacheIncremental]
+import a
+[file a.py]
+import b
+import c
+[file c.py]
+x: int
+[file c.py.2]
+x: str
+[out]
+tmp/a.py:1: error: Cannot find implementation or library stub for module named "b"
+tmp/a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+[out2]
+tmp/a.py:1: error: Cannot find implementation or library stub for module named "b"
+tmp/a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
