"""Facilities to analyze entire programs, including imported modules.

Parse and analyze the source files of a program in the correct order

(based on file dependencies), and collect the results.

This module only directs a build, which is performed in multiple passes per

file. The individual passes are implemented in separate modules.

The function build() is the main interface to this module.

"""

# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file

from __future__ import annotations

import collections

import contextlib

import gc

import json

import os

import pickle

import platform

import re

import stat

import subprocess

import sys

import time

import types

from collections.abc import Callable, Iterator, Mapping, Sequence, Set as AbstractSet

from concurrent.futures import ThreadPoolExecutor, wait

from heapq import heappop, heappush

from textwrap import dedent

from threading import Lock, Thread

from typing import (

TYPE_CHECKING,

Any,

ClassVar,

Final,

NoReturn,

TextIO,

TypeAlias as _TypeAlias,

TypedDict,

cast,

final,

)

from librt.base64 import b64encode

from librt.internal import (

cache_version,

read_bool,

read_int as read_int_bare,

read_str as read_str_bare,

read_tag,

write_bool,

write_bytes as write_bytes_bare,

write_int as write_int_bare,

write_str as write_str_bare,

write_tag,

)

import mypy.semanal_main

from mypy.cache import (

CACHE_VERSION,

DICT_STR_GEN,

LIST_GEN,

LITERAL_NONE,

CacheMeta,

CacheMetaEx,

ErrorTuple,

JsonValue,

ReadBuffer,

Tag,

WriteBuffer,

read_bytes,

read_int,

read_int_list,

read_int_opt,

read_str,

read_str_list,

read_str_opt,

write_bytes,

write_int,

write_int_list,

write_int_opt,

write_json_value,

write_str,

write_str_list,

write_str_opt,

)

from mypy.checker import DeferredNode, TypeChecker

from mypy.defaults import (

WORKER_CONNECTION_TIMEOUT,

WORKER_DONE_TIMEOUT,

WORKER_SHUTDOWN_TIMEOUT,

WORKER_START_INTERVAL,

WORKER_START_TIMEOUT,

)

from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter

from mypy.errorcodes import ErrorCode

from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error

from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort

from mypy.indirection import TypeIndirectionVisitor

from mypy.ipc import (

BadStatus,

IPCClient,

IPCException,

IPCMessage,

read_status,

ready_to_read,

receive,

send,

)

from mypy.messages import MessageBuilder

from mypy.nodes import (

Decorator,

FileRawData,

FuncDef,

Import,

ImportAll,

ImportBase,

ImportFrom,

MypyFile,

OverloadedFuncDef,

SymbolTable,

)

from mypy.options import OPTIONS_AFFECTING_CACHE_NO_PLATFORM

from mypy.partially_defined import PossiblyUndefinedVariableVisitor

from mypy.semanal import SemanticAnalyzer

from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis

from mypy.util import (

DecodeError,

decode_python_encoding,

get_available_threads,

get_mypy_comments,

hash_digest,

hash_digest_bytes,

is_stub_package_file,

is_sub_path_normabs,

is_typeshed_file,

module_prefix,

os_path_join,

read_py_file,

time_ref,

time_spent_us,

)

if TYPE_CHECKING:

from mypy.report import Reports # Avoid unconditional slow import

from mypy import errorcodes as codes

from mypy.config_parser import get_config_module_names, parse_mypy_comments

from mypy.fixup import NodeFixer

from mypy.freetree import free_tree

from mypy.fscache import FileSystemCache

from mypy.known_modules import get_known_modules, reset_known_modules_cache

from mypy.messages import best_matches, pretty_seq

from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore

from mypy.modulefinder import (

BuildSource as BuildSource,

BuildSourceSet as BuildSourceSet,

FindModuleCache,

ModuleNotFoundReason,

ModuleSearchResult,

SearchPaths,

compute_search_paths,

)

from mypy.modules_state import modules_state

from mypy.nodes import Expression

from mypy.options import Options

from mypy.parse import load_from_raw, parse

from mypy.plugin import ChainedPlugin, Plugin, ReportConfigContext

from mypy.plugins.default import DefaultPlugin

from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor

from mypy.stats import dump_type_stats

from mypy.stubinfo import stub_distribution_name

from mypy.types import Type, instance_cache

from mypy.typestate import reset_global_state, type_state

from mypy.util import json_dumps, json_loads

from mypy.version import __version__

# Switch to True to produce debug output related to fine-grained incremental

# mode only that is useful during development. This produces only a subset of

# output compared to --verbose output. We use a global flag to enable this so

# that it's easy to enable this when running tests.

DEBUG_FINE_GRAINED: Final = False

# These modules are special and should always come from typeshed.

CORE_BUILTIN_MODULES: Final = {

"builtins",

"typing",

"types",

"typing_extensions",

"mypy_extensions",

"_typeshed",

"_collections_abc",

"collections",

"collections.abc",

"sys",

"abc",

}

# We are careful now, we can increase this in future if safe/useful.

MAX_GC_FREEZE_CYCLES: Final = 1

# We store status of initial GC freeze as a global variable to avoid memory

# leaks in tests, where we keep creating new BuildManagers in the same process.

initial_gc_freeze_done = False

Graph: _TypeAlias = dict[str, "State"]

MODULE_RESOLUTION_URL: Final = (

"https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules"

)

class SCC:

"""A simple class that represents a strongly connected component (import cycle)."""

id_counter: ClassVar[int] = 0

def __init__(

self, ids: set[str], scc_id: int | None = None, deps: list[int] | None = None

) -> None:

if scc_id is None:

self.id = SCC.id_counter

SCC.id_counter += 1

else:

self.id = scc_id

# Ids of modules in this cycle.

self.mod_ids = ids

# Direct dependencies, should be populated by the caller.

self.deps: set[int] = set(deps) if deps is not None else set()

# Direct dependencies that have not been processed yet.

# Should be populated by the caller. This set may change during graph

# processing, while the above stays constant.

self.not_ready_deps: set[int] = set()

# SCCs that (directly) depend on this SCC. Note this is a list to

# make processing order more predictable. Dependents will be notified

# that they may be ready in the order in this list.

self.direct_dependents: list[int] = []

# Rough estimate of how much time processing this SCC will take, this

# is used for more efficient scheduling across multiple build workers.

self.size_hint: int = 0

# TODO: Get rid of BuildResult. We might as well return a BuildManager.

class BuildResult:

"""The result of a successful build.

Attributes:

manager: The build manager.

files: Dictionary from module name to related AST node.

types: Dictionary from parse tree node to its inferred type.

used_cache: Whether the build took advantage of a pre-existing cache

errors: List of error messages.

"""

def __init__(self, manager: BuildManager, graph: Graph) -> None:

self.manager = manager

self.graph = graph

self.files = manager.modules

self.types = manager.all_types # Non-empty if export_types True in options

self.used_cache = manager.cache_enabled

self.errors: list[str] = [] # Filled in by build if desired

class WorkerClient:

"""A simple class that represents a mypy build worker."""

conn: IPCClient

def __init__(self, status_file: str, options_data: str, env: Mapping[str, str]) -> None:

self.status_file = status_file

if os.path.isfile(status_file):

os.unlink(status_file)

command = [

sys.executable,

"-m",

"mypy.build_worker",

f"--status-file={status_file}",

f'--options-data="{options_data}"',

]

# Return early without waiting, caller must call connect() before using the client.

self.proc = subprocess.Popen(command, env=env)

self.connected = False

def connect(self) -> None:

end_time = time.time() + WORKER_START_TIMEOUT

last_exception: Exception | None = None

while time.time() < end_time:

try:

data = read_status(self.status_file)

except BadStatus as exc:

last_exception = exc

time.sleep(WORKER_START_INTERVAL)

continue

try:

pid, connection_name = data["pid"], data["connection_name"]

assert isinstance(pid, int), f"Bad PID: {pid}"

assert isinstance(connection_name, str), f"Bad connection name: {connection_name}"

if sys.platform != "win32":

# Windows uses "wrapper processes" to run Python, so we cannot

# verify PIDs reliably.

assert pid == self.proc.pid, f"PID mismatch: {pid} vs {self.proc.pid}"

self.conn = IPCClient(connection_name, WORKER_CONNECTION_TIMEOUT)

self.connected = True

return

except Exception as exc:

last_exception = exc

break

print(f"Failed to establish connection with worker: {last_exception}")

def close(self) -> None:

if self.connected:

self.conn.close()

# Technically we don't need to wait, but otherwise we will get ResourceWarnings.

try:

self.proc.wait(timeout=WORKER_SHUTDOWN_TIMEOUT)

except subprocess.TimeoutExpired:

pass

if os.path.isfile(self.status_file):

os.unlink(self.status_file)

def build_error(msg: str) -> NoReturn:

raise CompileError([f"mypy: error: {msg}"])

def build(

sources: list[BuildSource],

options: Options,

alt_lib_path: str | None = None,

flush_errors: Callable[[str | None, list[str], bool], None] | None = None,

fscache: FileSystemCache | None = None,

stdout: TextIO | None = None,

stderr: TextIO | None = None,

extra_plugins: Sequence[Plugin] | None = None,

worker_env: Mapping[str, str] | None = None,

) -> BuildResult:

"""Analyze a program.

A single call to build performs parsing, semantic analysis and optionally

type checking for the program *and* all imported modules, recursively.

Return BuildResult if successful or only non-blocking errors were found;

otherwise raise CompileError.

If a flush_errors callback is provided, all error messages will be

passed to it and the errors and messages fields of BuildResult and

CompileError (respectively) will be empty. Otherwise, those fields will

report any error messages.

Args:

sources: list of sources to build

options: build options

alt_lib_path: an additional directory for looking up library modules

(takes precedence over other directories)

flush_errors: optional function to flush errors after a file is processed

fscache: optionally a file-system cacher

stdout: Output stream to use instead of `sys.stdout`

stderr: Error stream to use instead of `sys.stderr`

extra_plugins: Plugins to use in addition to those loaded from config

worker_env: An environment to start parallel build workers (used for tests)

"""

# If we were not given a flush_errors, we use one that will populate those

# fields for callers that want the traditional API.

messages = []

# This is mostly for the benefit of tests that use builtins fixtures.

instance_cache.reset()

reset_known_modules_cache()

def default_flush_errors(

filename: str | None, new_messages: list[str], is_serious: bool

) -> None:

messages.extend(new_messages)

flush_errors = flush_errors or default_flush_errors

stdout = stdout or sys.stdout

stderr = stderr or sys.stderr

extra_plugins = extra_plugins or []

# Create metastore before workers to avoid race conditions.

metastore = create_metastore(options, parallel_worker=False)

workers = []

connect_threads = []

# A quasi-unique ID for this specific mypy invocation.

build_id = os.urandom(4).hex()

if options.num_workers > 0:

# TODO: switch to something more efficient than pickle (also in the daemon).

pickled_options = pickle.dumps(options.snapshot())

options_data = b64encode(pickled_options).decode()

workers = [

WorkerClient(

f".mypy_worker.{build_id}.{idx}.json", options_data, worker_env or os.environ

)

for idx in range(options.num_workers)

]

sources_message = SourcesDataMessage(sources=sources)

buf = WriteBuffer()

sources_message.write(buf)

sources_data = buf.getvalue()

def connect(wc: WorkerClient, data: bytes) -> None:

# Start loading sources in each worker as soon as it is up.

wc.connect()

if not wc.connected:

# Caller should detect this and fail gracefully.

return

wc.conn.write_bytes(data)

# We don't wait for workers to be ready until they are actually needed.

for worker in workers:

thread = Thread(target=connect, args=(worker, sources_data))

thread.start()

connect_threads.append(thread)

try:

result = build_inner(

sources,

options,

alt_lib_path,

flush_errors,

fscache,

stdout,

stderr,

extra_plugins,

workers,

connect_threads,

metastore,

)

result.errors = messages

return result

except CompileError as e:

# CompileErrors raised from an errors object carry all the

# messages that have not been reported out by error streaming.

# Patch it up to contain either none or all none of the messages,

# depending on whether we are flushing errors.

serious = not e.use_stdout

flush_errors(None, e.messages, serious)

e.messages = messages

raise

finally:

# In case of an early crash it is better to wait for workers to become ready, and

# shut them down cleanly. Otherwise, they will linger until connection timeout.

for thread in connect_threads:

thread.join()

for worker in workers:

if not worker.connected:

continue

try:

send(worker.conn, SccRequestMessage(scc_id=None, import_errors={}, mod_data={}))

except (OSError, IPCException):

pass

for worker in workers:

worker.close()

def build_inner(

sources: list[BuildSource],

options: Options,

alt_lib_path: str | None,

flush_errors: Callable[[str | None, list[str], bool], None],

fscache: FileSystemCache | None,

stdout: TextIO,

stderr: TextIO,

extra_plugins: Sequence[Plugin],

workers: list[WorkerClient],

connect_threads: list[Thread],

metastore: MetadataStore,

) -> BuildResult:

if platform.python_implementation() == "CPython":

# Run gc less frequently, as otherwise we can spend a large fraction of

# cpu in gc. This seems the most reasonable place to tune garbage collection.

gc.set_threshold(200 * 1000, 30, 30)

data_dir = default_data_dir()

fscache = fscache or FileSystemCache()

search_paths = compute_search_paths(sources, options, data_dir, alt_lib_path)

reports = None

if options.report_dirs:

# Import lazily to avoid slowing down startup.

from mypy.report import Reports

reports = Reports(data_dir, options.report_dirs)

source_set = BuildSourceSet(sources)

cached_read = fscache.read

errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read))

# Record import errors so that they can be replayed by the workers.

if workers:

errors.global_watcher = True

plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)

# Validate error codes after plugins are loaded.

options.process_error_codes(error_callback=build_error)

# Construct a build manager object to hold state during the build.

#

# Ignore current directory prefix in error messages.

manager = BuildManager(

data_dir,

search_paths,

ignore_prefix=os.getcwd(),

source_set=source_set,

reports=reports,

options=options,

version_id=__version__,

plugin=plugin,

plugins_snapshot=snapshot,

errors=errors,

error_formatter=None if options.output is None else OUTPUT_CHOICES.get(options.output),

flush_errors=flush_errors,

fscache=fscache,

stdout=stdout,

stderr=stderr,

metastore=metastore,

)

manager.workers = workers

if manager.verbosity() >= 2:

manager.trace(repr(options))

reset_global_state()

try:

graph = dispatch(sources, manager, stdout, connect_threads)

if not options.fine_grained_incremental:

type_state.reset_all_subtype_caches()

if options.timing_stats is not None:

dump_timing_stats(options.timing_stats, graph)

if options.line_checking_stats is not None:

dump_line_checking_stats(options.line_checking_stats, graph)

warn_unused_configs(options, flush_errors)

return BuildResult(manager, graph)

finally:

manager.commit()

manager.log(

"Build finished in %.3f seconds with %d modules, and %d errors"

% (

time.time() - manager.start_time,

len(manager.modules),

manager.errors.num_messages(),

)

)

manager.dump_stats()

if reports is not None:

# Finish the HTML or XML reports even if CompileError was raised.

reports.finish()

if os.path.isdir(options.cache_dir):

add_catch_all_gitignore(options.cache_dir)

exclude_from_backups(options.cache_dir)

if os.path.isdir(options.cache_dir):

record_missing_stub_packages(options.cache_dir, manager.missing_stub_packages)

def warn_unused_configs(

options: Options, flush_errors: Callable[[str | None, list[str], bool], None]

) -> None:

if options.warn_unused_configs and options.unused_configs and not options.non_interactive:

unused = get_config_module_names(

options.config_file,

[glob for glob in options.per_module_options.keys() if glob in options.unused_configs],

)

flush_errors(

None, ["{}: note: unused section(s): {}".format(options.config_file, unused)], False

)

def default_data_dir() -> str:

"""Returns directory containing typeshed directory."""

return os.path.dirname(__file__)

def normpath(path: str, options: Options) -> str:

"""Convert path to absolute; but to relative in bazel mode.

(Bazel's distributed cache doesn't like filesystem metadata to

end up in output files.)

"""

# TODO: Could we always use relpath? (A worry in non-bazel

# mode would be that a moved file may change its full module

# name without changing its size, mtime or hash.)

if options.bazel:

return os.path.relpath(path)

else:

return os.path.abspath(path)

# NOTE: dependencies + suppressed == all reachable imports;

# suppressed contains those reachable imports that were prevented by

# silent mode or simply not found.

# Metadata for the fine-grained dependencies file associated with a module.

class FgDepMeta(TypedDict):

path: str

mtime: int

# Priorities used for imports. (Here, top-level includes inside a class.)

# These are used to determine a more predictable order in which the

# nodes in an import cycle are processed.

PRI_HIGH: Final = 5 # top-level "from X import blah"

PRI_MED: Final = 10 # top-level "import X"

PRI_LOW: Final = 20 # either form inside a function

PRI_MYPY: Final = 25 # inside "if MYPY" or "if TYPE_CHECKING"

PRI_INDIRECT: Final = 30 # an indirect dependency

PRI_ALL: Final = 99 # include all priorities

def import_priority(imp: ImportBase, toplevel_priority: int) -> int:

"""Compute import priority from an import node."""

if not imp.is_top_level:

# Inside a function

return PRI_LOW

if imp.is_mypy_only:

# Inside "if MYPY" or "if typing.TYPE_CHECKING"

return max(PRI_MYPY, toplevel_priority)

# A regular import; priority determined by argument.

return toplevel_priority

def load_plugins_from_config(

options: Options, errors: Errors, stdout: TextIO

) -> tuple[list[Plugin], dict[str, str]]:

"""Load all configured plugins.

Return a list of all the loaded plugins from the config file.

The second return value is a snapshot of versions/hashes of loaded user

plugins (for cache validation).

"""

import importlib

snapshot: dict[str, str] = {}

if not options.config_file:

return [], snapshot

line = find_config_file_line_number(options.config_file, "mypy", "plugins")

if line == -1:

line = 1 # We need to pick some line number that doesn't look too confusing

def plugin_error(message: str) -> NoReturn:

errors.report(line, 0, message)

errors.raise_error(use_stdout=False)

custom_plugins: list[Plugin] = []

errors.set_file(options.config_file, None, options)

for plugin_path in options.plugins:

func_name = "plugin"

plugin_dir: str | None = None

if ":" in os.path.basename(plugin_path):

plugin_path, func_name = plugin_path.rsplit(":", 1)

if plugin_path.endswith(".py"):

# Plugin paths can be relative to the config file location.

plugin_path = os_path_join(os.path.dirname(options.config_file), plugin_path)

if not os.path.isfile(plugin_path):

plugin_error(f'Can\'t find plugin "{plugin_path}"')

# Use an absolute path to avoid populating the cache entry

# for 'tmp' during tests, since it will be different in

# different tests.

plugin_dir = os.path.abspath(os.path.dirname(plugin_path))

fnam = os.path.basename(plugin_path)

module_name = fnam[:-3]

sys.path.insert(0, plugin_dir)

elif re.search(r"[\\/]", plugin_path):

fnam = os.path.basename(plugin_path)

plugin_error(f'Plugin "{fnam}" does not have a .py extension')

else:

module_name = plugin_path

try:

module = importlib.import_module(module_name)

except Exception as exc:

plugin_error(f'Error importing plugin "{plugin_path}": {exc}')

finally:

if plugin_dir is not None:

assert sys.path[0] == plugin_dir

del sys.path[0]

if not hasattr(module, func_name):

plugin_error(

'Plugin "{}" does not define entry point function "{}"'.format(

plugin_path, func_name

)

)

try:

plugin_type = getattr(module, func_name)(__version__)

except Exception:

print(f"Error calling the plugin(version) entry point of {plugin_path}\n", file=stdout)

raise # Propagate to display traceback

if not isinstance(plugin_type, type):

plugin_error(

'Type object expected as the return value of "plugin"; got {!r} (in {})'.format(

plugin_type, plugin_path

)

)

if not issubclass(plugin_type, Plugin):

plugin_error(

'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" '

"(in {})".format(plugin_path)

)

try:

custom_plugins.append(plugin_type(options))

snapshot[module_name] = take_module_snapshot(module)

except Exception:

print(f"Error constructing plugin instance of {plugin_type.__name__}\n", file=stdout)

raise # Propagate to display traceback

return custom_plugins, snapshot

def load_plugins(

options: Options, errors: Errors, stdout: TextIO, extra_plugins: Sequence[Plugin]

) -> tuple[Plugin, dict[str, str]]:

"""Load all configured plugins.

Return a plugin that encapsulates all plugins chained together. Always

at least include the default plugin (it's last in the chain).

The second return value is a snapshot of versions/hashes of loaded user

plugins (for cache validation).

"""

custom_plugins, snapshot = load_plugins_from_config(options, errors, stdout)

custom_plugins += extra_plugins

default_plugin: Plugin = DefaultPlugin(options)

if not custom_plugins:

return default_plugin, snapshot

# Custom plugins take precedence over the default plugin.

return ChainedPlugin(options, custom_plugins + [default_plugin]), snapshot

def take_module_snapshot(module: types.ModuleType) -> str:

"""Take plugin module snapshot by recording its version and hash.

We record _both_ hash and the version to detect more possible changes

(e.g. if there is a change in modules imported by a plugin).

"""

if hasattr(module, "__file__"):

assert module.__file__ is not None

with open(module.__file__, "rb") as f:

digest = hash_digest(f.read())

else:

digest = "unknown"

ver = getattr(module, "__version__", "none")

return f"{ver}:{digest}"

def find_config_file_line_number(path: str, section: str, setting_name: str) -> int:

"""Return the approximate location of setting_name within mypy config file.

Return -1 if can't determine the line unambiguously.

"""

in_desired_section = False

try:

results = []

with open(path, encoding="UTF-8") as f:

for i, line in enumerate(f):

line = line.strip()

if line.startswith("[") and line.endswith("]"):

current_section = line[1:-1].strip()

in_desired_section = current_section == section

elif in_desired_section and re.match(rf"{setting_name}\s*=", line):

results.append(i + 1)

if len(results) == 1:

return results[0]

except OSError:

pass

return -1

class BuildManager:

"""This class holds shared state for building a mypy program.

It is used to coordinate parsing, import processing, semantic

analysis and type checking. The actual build steps are carried

out by dispatch().

Attributes:

data_dir: Mypy data directory (contains stubs)

search_paths: SearchPaths instance indicating where to look for modules

modules: Mapping of module ID to MypyFile (shared by the passes)

semantic_analyzer:

Semantic analyzer, pass 2

all_types: Map {Expression: Type} from all modules (enabled by export_types)

options: Build options

missing_modules: Modules that could not be imported (or intentionally skipped)

stale_modules: Set of modules that needed to be rechecked (only used by tests)

fg_deps_meta: Metadata for fine-grained dependencies caches associated with modules

fg_deps: A fine-grained dependency map

version_id: The current mypy version (based on commit id when possible)

plugin: Active mypy plugin(s)

plugins_snapshot:

Snapshot of currently active user plugins (versions and hashes)

old_plugins_snapshot:

Plugins snapshot from previous incremental run (or None in

non-incremental mode and if cache was not found)

errors: Used for reporting all errors

flush_errors: A function for processing errors after each SCC

cache_enabled: Whether cache is being read. This is set based on options,

but is disabled if fine-grained cache loading fails

and after an initial fine-grained load. This doesn't

determine whether we write cache files or not.

quickstart_state:

A cache of filename -> mtime/size/hash info used to avoid

needing to hash source files when using a cache with mismatching mtimes

stats: Dict with various instrumentation numbers, it is used

not only for debugging, but also required for correctness,

in particular to check consistency of the fine-grained dependency cache.

fscache: A file system cacher

ast_cache: AST cache to speed up mypy daemon

"""

def __init__(

self,

data_dir: str,

search_paths: SearchPaths,

ignore_prefix: str,

source_set: BuildSourceSet,

reports: Reports | None,

options: Options,

version_id: str,

plugin: Plugin,

plugins_snapshot: dict[str, str],

errors: Errors,

flush_errors: Callable[[str | None, list[str], bool], None],

fscache: FileSystemCache,

stdout: TextIO,

stderr: TextIO,

error_formatter: ErrorFormatter | None = None,

parallel_worker: bool = False,

metastore: MetadataStore | None = None,

) -> None:

self.stats: dict[str, Any] = {} # Values are ints or floats

# Use in cases where we need to prevent race conditions in stats reporting.

self.stats_lock = Lock()

self.stdout = stdout

self.stderr = stderr

self.start_time = time.time()

self.data_dir = data_dir

self.errors = errors

self.errors.set_ignore_prefix(ignore_prefix)

self.error_formatter = error_formatter

self.search_paths = search_paths

self.source_set = source_set

self.reports = reports

self.options = options

self.version_id = version_id

self.modules: dict[str, MypyFile] = {}

# Share same modules dictionary with the global fixer state.

# We need to set allow_missing when doing a fine-grained cache

# load because we need to gracefully handle missing modules.

modules_state.modules = self.modules

modules_state.node_fixer = NodeFixer(self.modules, self.options.use_fine_grained_cache)

self.import_map: dict[str, set[str]] = {}

self.missing_modules: dict[str, int] = {}

self.fg_deps_meta: dict[str, FgDepMeta] = {}

# fg_deps holds the dependencies of every module that has been

# processed. We store this in BuildManager so that we can compute

# dependencies as we go, which allows us to free ASTs and type information,

# saving a ton of memory on net.

self.fg_deps: dict[str, set[str]] = {}

# Always convert the plugin to a ChainedPlugin so that it can be manipulated if needed

if not isinstance(plugin, ChainedPlugin):

plugin = ChainedPlugin(options, [plugin])

self.plugin = plugin

# These allow quickly skipping logging and stats collection calls. Note

# that some stats impact mypy behavior, so be careful when skipping stats

# collection calls.

self.stats_enabled = self.options.dump_build_stats

self.logging_enabled = self.options.verbosity >= 1

self.tracing_enabled = self.options.verbosity >= 2

# Set of namespaces (module or class) that are being populated during semantic

# analysis and may have missing definitions.

self.incomplete_namespaces: set[str] = set()

self.semantic_analyzer = SemanticAnalyzer(

self.modules,

self.missing_modules,

self.incomplete_namespaces,

self.errors,

self.plugin,

self.import_map,

)

self.all_types: dict[Expression, Type] = {} # Enabled by export_types

self.indirection_detector = TypeIndirectionVisitor()

self.stale_modules: set[str] = set()

self.rechecked_modules: set[str] = set()

self.flush_errors = flush_errors

has_reporters = reports is not None and reports.reporters

self.cache_enabled = (

options.incremental

and (not options.fine_grained_incremental or options.use_fine_grained_cache)

and not has_reporters

)

self.fscache = fscache

self.cwd = os.getcwd()

self.find_module_cache = FindModuleCache(

self.search_paths, self.fscache, self.options, source_set=self.source_set

)

for module in CORE_BUILTIN_MODULES:

if options.use_builtins_fixtures:

continue

path = self.find_module_cache.find_module(module, fast_path=True)

if not isinstance(path, str):

raise CompileError(

[f"Failed to find builtin module {module}, perhaps typeshed is broken?"]

)

if is_typeshed_file(options.abs_custom_typeshed_dir, path) or is_stub_package_file(

path

):

continue

raise CompileError(

[

f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"',

f'note: A user-defined top-level module with name "{module}" is not supported',

]

)

if metastore is None:

metastore = create_metastore(options, parallel_worker=parallel_worker)

self.metastore = metastore

# a mapping from source files to their corresponding shadow files

# for efficient lookup

self.shadow_map: dict[str, str] = {}

if self.options.shadow_file is not None:

self.shadow_map = dict(self.options.shadow_file)

# a mapping from each file being typechecked to its possible shadow file

self.shadow_equivalence_map: dict[str, str | None] = {}

self.plugin = plugin

self.plugins_snapshot = plugins_snapshot

self.old_plugins_snapshot = read_plugins_snapshot(self)

if self.verbosity() >= 2:

self.trace(f"Plugins snapshot (fresh) {json.dumps(self.plugins_snapshot)}")

self.quickstart_state = read_quickstart_file(options, self.stdout)

# Fine grained targets (module top levels and top level functions) processed by

# the semantic analyzer, used only for testing. Currently used only by the new

# semantic analyzer. Tuple of module and target name.

self.processed_targets: list[tuple[str, str]] = []

# Missing stub packages encountered.

self.missing_stub_packages: set[str] = set()

# Cache for mypy ASTs that have completed semantic analysis

# pass 1. When multiple files are added to the build in a

# single daemon increment, only one of the files gets added

# per step and the others are discarded. This gets repeated

# until all the files have been added. This means that a

# new file can be processed O(n**2) times. This cache

# avoids most of this redundant work.

self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {}

# Number of times we used GC optimization hack for fresh SCCs.

self.gc_freeze_cycles = 0

# Mapping from SCC id to corresponding SCC instance. This is populated

# in process_graph().

self.scc_by_id: dict[int, SCC] = {}

# Mapping from module id to the SCC it belongs to. This is populated

# in process_graph().

self.scc_by_mod_id: dict[str, SCC] = {}

# Global topological order for SCCs. This exists to make order of processing

# SCCs more predictable.

self.top_order: list[int] = []

# Stale SCCs that are queued for processing. Each tuple contains SCC size hint,

# SCC adding order (tie-breaker), and the SCC itself.

self.scc_queue: list[tuple[int, int, SCC]] = []

# SCCs that have been fully processed.

self.done_sccs: set[int] = set()

# Parallel build workers, list is empty for in-process type-checking.

self.workers: list[WorkerClient] = []

# We track which workers are currently free in the coordinator process.

# This is a tiny bit faster and conceptually simpler than check which ones

# are writeable each time we want to submit an SCC for processing.

self.free_workers: set[int] = set()

# A global adding order for SCC queue, see comment above.

self.queue_order: int = 0

# Is this an instance used by a parallel worker?

self.parallel_worker = parallel_worker

# Snapshot of import-related options per module. We record these even for

# suppressed imports, since they can affect errors in the callers. Bytes

# value is opaque but can be compared to detect changes in options.

self.import_options: dict[str, bytes] = {}

# Cache for transitive dependency check (expensive).

self.transitive_deps_cache: dict[tuple[int, int], bool] = {}

# Packages for which we know presence or absence of __getattr__().

self.known_partial_packages: dict[str, bool] = {}

def dump_stats(self) -> None:

if self.stats_enabled:

lines = ["Stats:"]

for key, value in sorted(self.stats_summary().items()):

fmt = ".3f" if isinstance(value, float) else "d"

lines.append(f"{key + ':':24}{value:{fmt}}")

# Call print once so that we don't get a mess in parallel mode.

print("\n".join(lines) + "\n\n", end="")