Add a plugin hook for specifying per-module configuration data (python#7871)

msullivan · web-flow · commit 47b928dac8a5 · 2019-11-04T16:24:04.000-08:00
This is useful in general for plugins that might have per-module
configuration and we will be able to use it to manage the interaction
between mypyc caches and mypy caches without needing to add custom
mypyc hooks.
diff --git a/docs/source/extending_mypy.rst b/docs/source/extending_mypy.rst
@@ -236,6 +236,14 @@ module. It is called before semantic analysis. For example, this can
 be used if a library has dependencies that are dynamically loaded
 based on configuration information.
 
+**report_config_data()** can be used if the plugin has some sort of
+per-module configuration that can affect typechecking. In that case,
+when the configuration for a module changes, we want to invalidate
+mypy's cache for that module so that it can be rechecked. This hook
+should be used to report to mypy any relevant configuration data,
+so that mypy knows to recheck the module if the configuration changes.
+The hooks hould return data encodable as JSON.
+
 Notes about the semantic analyzer
 *********************************
 
diff --git a/mypy/build.py b/mypy/build.py
@@ -269,6 +269,7 @@ def default_data_dir() -> str:
                         ('interface_hash', str),  # hash representing the public interface
                         ('version_id', str),  # mypy version for cache invalidation
                         ('ignore_all', bool),  # if errors were ignored
+                        ('plugin_data', Any),  # config data from plugins
                         ])
 # NOTE: dependencies + suppressed == all reachable imports;
 # suppressed contains those reachable imports that were prevented by
@@ -303,6 +304,7 @@ def cache_meta_from_dict(meta: Dict[str, Any], data_json: str) -> CacheMeta:
         meta.get('interface_hash', ''),
         meta.get('version_id', sentinel),
         meta.get('ignore_all', True),
+        meta.get('plugin_data', None),
     )
 
 
@@ -1162,6 +1164,13 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
         if manager.plugins_snapshot != manager.old_plugins_snapshot:
             manager.log('Metadata abandoned for {}: plugins differ'.format(id))
             return None
+    # So that plugins can return data with tuples in it without
+    # things silently always invalidating modules, we round-trip
+    # the config data. This isn't beautiful.
+    plugin_data = json.loads(json.dumps(manager.plugin.report_config_data(id, path)))
+    if m.plugin_data != plugin_data:
+        manager.log('Metadata abandoned for {}: plugin configuration differs'.format(id))
+        return None
 
     manager.add_stats(fresh_metas=1)
     return m
@@ -1284,6 +1293,7 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str],
                 'interface_hash': meta.interface_hash,
                 'version_id': manager.version_id,
                 'ignore_all': meta.ignore_all,
+                'plugin_data': meta.plugin_data,
             }
             if manager.options.debug_cache:
                 meta_str = json.dumps(meta_dict, indent=2, sort_keys=True)
@@ -1366,6 +1376,8 @@ def write_cache(id: str, path: str, tree: MypyFile,
     data_str = json_dumps(data, manager.options.debug_cache)
     interface_hash = compute_hash(data_str)
 
+    plugin_data = manager.plugin.report_config_data(id, path)
+
     # Obtain and set up metadata
     try:
         st = manager.get_stat(path)
@@ -1429,6 +1441,7 @@ def write_cache(id: str, path: str, tree: MypyFile,
             'interface_hash': interface_hash,
             'version_id': manager.version_id,
             'ignore_all': ignore_all,
+            'plugin_data': plugin_data,
             }
 
     # Write meta cache file
diff --git a/mypy/interpreted_plugin.py b/mypy/interpreted_plugin.py
@@ -41,6 +41,9 @@ def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]:
         assert self._modules is not None
         return lookup_fully_qualified(fullname, self._modules)
 
+    def report_config_data(self, id: str, path: str) -> Any:
+        return None
+
     def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
         return []
 
diff --git a/mypy/plugin.py b/mypy/plugin.py
@@ -459,6 +459,19 @@ def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]:
         assert self._modules is not None
         return lookup_fully_qualified(fullname, self._modules)
 
+    def report_config_data(self, id: str, path: str) -> Any:
+        """Get representation of configuration data for a module.
+
+        The data must be encodable as JSON and will be stored in the
+        cache metadata for the module. A mismatch between the cached
+        values and the returned will result in that module's cache
+        being invalidated and the module being rechecked.
+
+        This can be used to incorporate external configuration information
+        that might require changes to typechecking.
+        """
+        return None
+
     def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
         """Customize dependencies for a module.
 
@@ -666,6 +679,9 @@ def set_modules(self, modules: Dict[str, MypyFile]) -> None:
     def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]:
         return self.plugin.lookup_fully_qualified(fullname)
 
+    def report_config_data(self, id: str, path: str) -> Any:
+        return self.plugin.report_config_data(id, path)
+
     def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
         return self.plugin.get_additional_deps(file)
 
@@ -734,6 +750,10 @@ def set_modules(self, modules: Dict[str, MypyFile]) -> None:
         for plugin in self._plugins:
             plugin.set_modules(modules)
 
+    def report_config_data(self, id: str, path: str) -> Any:
+        config_data = [plugin.report_config_data(id, path) for plugin in self._plugins]
+        return config_data if any(x is not None for x in config_data) else None
+
     def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
         deps = []
         for plugin in self._plugins:
diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test
@@ -4936,6 +4936,24 @@ python_version=3.6
 [out2]
 tmp/a.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "str")
 
+[case testPluginConfigData]
+# flags: --config-file tmp/mypy.ini
+import a
+import b
+[file a.py]
+[file b.py]
+[file test.json]
+{"a": false, "b": false}
+[file test.json.2]
+{"a": true, "b": false}
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/config_data.py
+
+# The config change will force a to be rechecked but not b.
+[rechecked a]
+
 [case testLiteralIncrementalTurningIntoLiteral]
 import mod
 reveal_type(mod.a)
diff --git a/test-data/unit/plugins/config_data.py b/test-data/unit/plugins/config_data.py
@@ -0,0 +1,18 @@
+import os
+import json
+
+from typing import Any
+
+from mypy.plugin import Plugin
+
+
+class ConfigDataPlugin(Plugin):
+    def report_config_data(self, id: str, path: str) -> Any:
+        path = os.path.join('tmp/test.json')
+        with open(path) as f:
+            data = json.load(f)
+        return data.get(id)
+
+
+def plugin(version):
+    return ConfigDataPlugin
