From 18a6dfd62c8897085fc2b136d7b85a2ad42ddbfc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 28 Oct 2025 15:53:11 +0000 Subject: [PATCH] Use self-descriptive cache with type tags --- mypy/cache.py | 290 +++++++++++++++++++++++++++------- mypy/nodes.py | 200 ++++++++++++++--------- mypy/types.py | 245 ++++++++++++++++++---------- mypyc/lib-rt/librt_internal.c | 4 + 4 files changed, 529 insertions(+), 210 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index aeb0e8810fd65..0d2db67fac948 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,27 +1,70 @@ +""" +This module contains high-level logic for fixed format serialization. + +Lower-level parts are implemented in C in mypyc/lib-rt/librt_internal.c +Short summary of low-level functionality: +* integers are automatically serialized as 1, 2, or 4 bytes, or arbitrary length. +* str/bytes are serialized as size (1, 2, or 4 bytes) followed by bytes buffer. +* floats are serialized as C doubles. + +At high-level we add type tags as needed so that our format is self-descriptive. +More precisely: +* False, True, and None are stored as just a tag: 0, 1, 2 correspondingly. +* builtin primitives like int/str/bytes/float are stored as their type tag followed + by bare (low-level) representation of the value. Reserved tag range for primitives is + 3 ... 19. +* generic (heterogeneous) list are stored as tag, followed by bare size, followed by + sequence of tagged values. +* homogeneous lists of primitives are stored as tag, followed by bare size, followed + by sequence of bare values. +* reserved tag range for sequence-like builtins is 20 ... 29 +* currently we have only one mapping-like format: string-keyed dictionary with heterogeneous + values. It is stored as tag, followed by bare size, followed by sequence of pairs: bare + string key followed by tagged value. +* reserved tag range for mapping-like builtins is 30 ... 39 +* there is an additional reserved tag range 40 ... 49 for any other builtin collections. +* custom classes (like types, symbols etc.) are stored as tag, followed by a sequence of + tagged field values, followed by a special end tag 255. Names of class fields are + *not* stored, the caller should know the field names and order for the given class tag. +* reserved tag range for symbols (TypeInfo, Var, etc) is 50 ... 79. +* class Instance is the only exception from the above format (since it is the most common one). + It has two extra formats: few most common instances like "builtins.object" are stored as + instance tag followed by a secondary tag, other plain non-generic instances are stored as + instance tag followed by secondary tag followed by fullname as bare string. All generic + readers must handle these. +* reserved tag range for Instance type formats is 80 ... 99, for other types it is 100 ... 149. +* tag 254 is reserved for if we would ever need to extend the tag range to indicated second tag + page. Tags 150 ... 253 are free for everything else (e.g. AST nodes etc). + +General convention is that custom classes implement write() and read() methods for FF +serialization. The write method should write both class tag and end tag. The read method +conventionally *does not* read the start tag (to simplify logic for unions). Known exceptions +are MypyFile.read() and SymbolTableNode.read(), since those two never appear in a union. +""" + from __future__ import annotations from collections.abc import Sequence -from typing import Any, Final +from typing import Any, Final, Union +from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( Buffer as Buffer, read_bool as read_bool, - read_bytes as read_bytes, - read_float as read_float, - read_int as read_int, - read_str as read_str, + read_bytes as read_bytes_bare, + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, read_tag as read_tag, write_bool as write_bool, - write_bytes as write_bytes, - write_float as write_float, - write_int as write_int, - write_str as write_str, + write_bytes as write_bytes_bare, + write_float as write_float_bare, + write_int as write_int_bare, + write_str as write_str_bare, write_tag as write_tag, ) from mypy_extensions import u8 -from mypy.util import json_dumps, json_loads - class CacheMeta: """Class representing cache metadata for a module.""" @@ -125,7 +168,7 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.dependencies) write_int(data, self.data_mtime) write_str_list(data, self.suppressed) - write_bytes(data, json_dumps(self.options)) + write_json(data, self.options) write_int_list(data, self.dep_prios) write_int_list(data, self.dep_lines) write_bytes_list(data, self.dep_hashes) @@ -133,7 +176,9 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.error_lines) write_str(data, self.version_id) write_bool(data, self.ignore_all) - write_bytes(data, json_dumps(self.plugin_data)) + # Plugin data may be not a dictionary, so we use + # a more generic write_json_value() here. + write_json_value(data, self.plugin_data) @classmethod def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: @@ -148,7 +193,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: data_mtime=read_int(data), data_file=data_file, suppressed=read_str_list(data), - options=json_loads(read_bytes(data)), + options=read_json(data), dep_prios=read_int_list(data), dep_lines=read_int_list(data), dep_hashes=read_bytes_list(data), @@ -156,7 +201,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: error_lines=read_str_list(data), version_id=read_str(data), ignore_all=read_bool(data), - plugin_data=json_loads(read_bytes(data)), + plugin_data=read_json_value(data), ) except ValueError: return None @@ -165,114 +210,243 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: # Always use this type alias to refer to type tags. Tag = u8 -LITERAL_INT: Final[Tag] = 1 -LITERAL_STR: Final[Tag] = 2 -LITERAL_BOOL: Final[Tag] = 3 -LITERAL_FLOAT: Final[Tag] = 4 -LITERAL_COMPLEX: Final[Tag] = 5 -LITERAL_NONE: Final[Tag] = 6 +# Primitives. +LITERAL_FALSE: Final[Tag] = 0 +LITERAL_TRUE: Final[Tag] = 1 +LITERAL_NONE: Final[Tag] = 2 +LITERAL_INT: Final[Tag] = 3 +LITERAL_STR: Final[Tag] = 4 +LITERAL_BYTES: Final[Tag] = 5 +LITERAL_FLOAT: Final[Tag] = 6 +LITERAL_COMPLEX: Final[Tag] = 7 + +# Collections. +LIST_GEN: Final[Tag] = 20 +LIST_INT: Final[Tag] = 21 +LIST_STR: Final[Tag] = 22 +LIST_BYTES: Final[Tag] = 23 +DICT_STR_GEN: Final[Tag] = 30 + +# Misc classes. +EXTRA_ATTRS: Final[Tag] = 150 +DT_SPEC: Final[Tag] = 151 + +END_TAG: Final[Tag] = 255 def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: - return read_int(data) + return read_int_bare(data) elif tag == LITERAL_STR: - return read_str(data) - elif tag == LITERAL_BOOL: - return read_bool(data) + return read_str_bare(data) + elif tag == LITERAL_FALSE: + return False + elif tag == LITERAL_TRUE: + return True elif tag == LITERAL_FLOAT: - return read_float(data) + return read_float_bare(data) assert False, f"Unknown literal tag {tag}" +# There is an intentional asymmetry between read and write for literals because +# None and/or complex values are only allowed in some contexts but not in others. def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): - write_tag(data, LITERAL_BOOL) write_bool(data, value) elif isinstance(value, int): write_tag(data, LITERAL_INT) - write_int(data, value) + write_int_bare(data, value) elif isinstance(value, str): write_tag(data, LITERAL_STR) - write_str(data, value) + write_str_bare(data, value) elif isinstance(value, float): write_tag(data, LITERAL_FLOAT) - write_float(data, value) + write_float_bare(data, value) elif isinstance(value, complex): write_tag(data, LITERAL_COMPLEX) - write_float(data, value.real) - write_float(data, value.imag) + write_float_bare(data, value.real) + write_float_bare(data, value.imag) else: write_tag(data, LITERAL_NONE) +def read_int(data: Buffer) -> int: + assert read_tag(data) == LITERAL_INT + return read_int_bare(data) + + +def write_int(data: Buffer, value: int) -> None: + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + + +def read_str(data: Buffer) -> str: + assert read_tag(data) == LITERAL_STR + return read_str_bare(data) + + +def write_str(data: Buffer, value: str) -> None: + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + + +def read_bytes(data: Buffer) -> bytes: + assert read_tag(data) == LITERAL_BYTES + return read_bytes_bare(data) + + +def write_bytes(data: Buffer, value: bytes) -> None: + write_tag(data, LITERAL_BYTES) + write_bytes_bare(data, value) + + def read_int_opt(data: Buffer) -> int | None: - if read_bool(data): - return read_int(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_INT + return read_int_bare(data) def write_int_opt(data: Buffer, value: int | None) -> None: if value is not None: - write_bool(data, True) - write_int(data, value) + write_tag(data, LITERAL_INT) + write_int_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_str_opt(data: Buffer) -> str | None: - if read_bool(data): - return read_str(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_STR + return read_str_bare(data) def write_str_opt(data: Buffer, value: str | None) -> None: if value is not None: - write_bool(data, True) - write_str(data, value) + write_tag(data, LITERAL_STR) + write_str_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_int_list(data: Buffer) -> list[int]: - size = read_int(data) - return [read_int(data) for _ in range(size)] + assert read_tag(data) == LIST_INT + size = read_int_bare(data) + return [read_int_bare(data) for _ in range(size)] def write_int_list(data: Buffer, value: list[int]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_INT) + write_int_bare(data, len(value)) for item in value: - write_int(data, item) + write_int_bare(data, item) def read_str_list(data: Buffer) -> list[str]: - size = read_int(data) - return [read_str(data) for _ in range(size)] + assert read_tag(data) == LIST_STR + size = read_int_bare(data) + return [read_str_bare(data) for _ in range(size)] def write_str_list(data: Buffer, value: Sequence[str]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_STR) + write_int_bare(data, len(value)) for item in value: - write_str(data, item) + write_str_bare(data, item) def read_bytes_list(data: Buffer) -> list[bytes]: - size = read_int(data) - return [read_bytes(data) for _ in range(size)] + assert read_tag(data) == LIST_BYTES + size = read_int_bare(data) + return [read_bytes_bare(data) for _ in range(size)] def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_BYTES) + write_int_bare(data, len(value)) for item in value: - write_bytes(data, item) + write_bytes_bare(data, item) def read_str_opt_list(data: Buffer) -> list[str | None]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_str_opt(data) for _ in range(size)] def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: write_str_opt(data, item) + + +JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] + + +def read_json_value(data: Buffer) -> JsonValue: + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + if tag == LITERAL_FALSE: + return False + if tag == LITERAL_TRUE: + return True + if tag == LITERAL_INT: + return read_int_bare(data) + if tag == LITERAL_STR: + return read_str_bare(data) + if tag == LIST_GEN: + size = read_int_bare(data) + return [read_json_value(data) for _ in range(size)] + if tag == DICT_STR_GEN: + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + assert False, f"Invalid JSON tag: {tag}" + + +# Currently tuples are used by mypyc plugin. They will be normalized to +# JSON lists after a roundtrip. +def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> None: + if value is None: + write_tag(data, LITERAL_NONE) + elif isinstance(value, bool): + write_bool(data, value) + elif isinstance(value, int): + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + elif isinstance(value, str): + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + elif isinstance(value, (list, tuple)): + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) + for val in value: + write_json_value(data, val) + elif isinstance(value, dict): + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) + else: + assert False, f"Invalid JSON value: {value}" + + +# These are functions for JSON *dictionaries* specifically. Unfortunately, we +# must use imprecise types here, because the callers use imprecise types. +def read_json(data: Buffer) -> dict[str, Any]: + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + + +def write_json(data: Buffer, value: dict[str, Any]) -> None: + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) diff --git a/mypy/nodes.py b/mypy/nodes.py index 040f3fc28dce0..6cf984e5a2185 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from abc import abstractmethod from collections import defaultdict @@ -11,19 +10,31 @@ from typing import TYPE_CHECKING, Any, Callable, Final, Optional, TypeVar, Union, cast from typing_extensions import TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) from mypy_extensions import trait import mypy.strconv from mypy.cache import ( + DICT_STR_GEN, + DT_SPEC, + END_TAG, + LIST_GEN, + LIST_STR, LITERAL_COMPLEX, LITERAL_NONE, Buffer, Tag, read_bool, - read_float, read_int, read_int_list, read_int_opt, + read_json, read_literal, read_str, read_str_list, @@ -34,6 +45,7 @@ write_int, write_int_list, write_int_opt, + write_json, write_literal, write_str, write_str_list, @@ -432,6 +444,7 @@ def write(self, data: Buffer) -> None: write_str(data, self.path) write_bool(data, self.is_partial_stub_package) write_str_list(data, sorted(self.future_import_flags)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> MypyFile: @@ -444,6 +457,7 @@ def read(cls, data: Buffer) -> MypyFile: tree.is_partial_stub_package = read_bool(data) tree.future_import_flags = set(read_str_list(data)) tree.is_cache_skeleton = True + assert read_tag(data) == END_TAG return tree @@ -720,30 +734,33 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED_FUNC_DEF) - write_int(data, len(self.items)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.items)) for item in self.items: item.write(data) mypy.types.write_type_opt(data, self.type) write_str(data, self._fullname) if self.impl is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.impl.write(data) write_flags(data, self, FUNCBASE_FLAGS) write_str_opt(data, self.deprecated) write_int_opt(data, self.setter_index) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> OverloadedFuncDef: - res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int(data))]) + assert read_tag(data) == LIST_GEN + res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) typ = mypy.types.read_type_opt(data) if typ is not None: assert isinstance(typ, mypy.types.ProperType) res.type = typ res._fullname = read_str(data) - if read_bool(data): - res.impl = read_overload_part(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + res.impl = read_overload_part(data, tag) # set line for empty overload items, as not set in __init__ if len(res.items) > 0: res.set_line(res.impl.line) @@ -751,6 +768,7 @@ def read(cls, data: Buffer) -> OverloadedFuncDef: res.deprecated = read_str_opt(data) res.setter_index = read_int_opt(data) # NOTE: res.info will be set in the fixup phase. + assert read_tag(data) == END_TAG return res def is_dynamic(self) -> bool: @@ -1039,19 +1057,20 @@ def write(self, data: Buffer) -> None: write_int_list(data, [int(ak.value) for ak in self.arg_kinds]) write_int(data, self.abstract_status) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) write_str_opt(data, self.original_first_arg) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> FuncDef: name = read_str(data) typ: mypy.types.FunctionLike | None = None - if read_bool(data): - typ = mypy.types.read_function_like(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + typ = mypy.types.read_function_like(data, tag) ret = FuncDef(name, [], Block([]), typ) ret._fullname = read_str(data) read_flags(data, ret, FUNCDEF_FLAGS) @@ -1059,7 +1078,9 @@ def read(cls, data: Buffer) -> FuncDef: ret.arg_names = read_str_opt_list(data) ret.arg_kinds = [ARG_KINDS[ak] for ak in read_int_list(data)] ret.abstract_status = read_int(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ret.dataclass_transform_spec = DataclassTransformSpec.read(data) ret.deprecated = read_str_opt(data) ret.original_first_arg = read_str_opt(data) @@ -1067,6 +1088,7 @@ def read(cls, data: Buffer) -> FuncDef: del ret.arguments del ret.max_pos del ret.min_args + assert read_tag(data) == END_TAG return ret @@ -1146,6 +1168,7 @@ def write(self, data: Buffer) -> None: self.func.write(data) self.var.write(data) write_bool(data, self.is_overload) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Decorator: @@ -1155,6 +1178,7 @@ def read(cls, data: Buffer) -> Decorator: var = Var.read(data) dec = Decorator(func, [], var) dec.is_overload = read_bool(data) + assert read_tag(data) == END_TAG return dec def is_dynamic(self) -> bool: @@ -1341,6 +1365,7 @@ def write(self, data: Buffer) -> None: write_str(data, self._fullname) write_flags(data, self, VAR_FLAGS) write_literal(data, self.final_value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Var: @@ -1348,18 +1373,20 @@ def read(cls, data: Buffer) -> Var: typ = mypy.types.read_type_opt(data) v = Var(name, typ) setter_type: mypy.types.CallableType | None = None - if read_bool(data): - assert read_tag(data) == mypy.types.CALLABLE_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == mypy.types.CALLABLE_TYPE setter_type = mypy.types.CallableType.read(data) v.setter_type = setter_type v.is_ready = False # Override True default set in __init__ v._fullname = read_str(data) read_flags(data, v, VAR_FLAGS) - marker = read_tag(data) - if marker == LITERAL_COMPLEX: - v.final_value = complex(read_float(data), read_float(data)) - elif marker != LITERAL_NONE: - v.final_value = read_literal(data, marker) + tag = read_tag(data) + if tag == LITERAL_COMPLEX: + v.final_value = complex(read_float_bare(data), read_float_bare(data)) + elif tag != LITERAL_NONE: + v.final_value = read_literal(data, tag) + assert read_tag(data) == END_TAG return v @@ -1477,15 +1504,13 @@ def write(self, data: Buffer) -> None: write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) write_str(data, self.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ClassDef: - res = ClassDef( - read_str(data), - Block([]), - [mypy.types.read_type_var_like(data) for _ in range(read_int(data))], - ) + res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) res.fullname = read_str(data) + assert read_tag(data) == END_TAG return res @@ -2913,10 +2938,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarExpr: - return TypeVarExpr( + ret = TypeVarExpr( read_str(data), read_str(data), mypy.types.read_type_list(data), @@ -2924,6 +2950,8 @@ def read(cls, data: Buffer) -> TypeVarExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecExpr(TypeVarLikeExpr): @@ -2962,16 +2990,19 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecExpr: - return ParamSpecExpr( + ret = ParamSpecExpr( read_str(data), read_str(data), mypy.types.read_type(data), mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleExpr(TypeVarLikeExpr): @@ -3031,12 +3062,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleExpr: assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) - return TypeVarTupleExpr( + ret = TypeVarTupleExpr( read_str(data), read_str(data), mypy.types.read_type(data), @@ -3044,6 +3076,8 @@ def read(cls, data: Buffer) -> TypeVarTupleExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeAliasExpr(Expression): @@ -3934,20 +3968,19 @@ def write(self, data: Buffer) -> None: mypy.types.write_type_opt(data, self.tuple_type) mypy.types.write_type_opt(data, self.typeddict_type) write_flags(data, self, TypeInfo.FLAGS) - write_str(data, json.dumps(self.metadata)) + write_json(data, self.metadata) if self.slots is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) write_str_list(data, sorted(self.slots)) write_str_list(data, self.deletable_attributes) mypy.types.write_type_opt(data, self.self_type) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeInfo: @@ -3963,7 +3996,8 @@ def read(cls, data: Buffer) -> TypeInfo: ti.type_vars = read_str_list(data) ti.has_param_spec_type = read_bool(data) ti.bases = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == mypy.types.INSTANCE ti.bases.append(mypy.types.Instance.read(data)) # NOTE: ti.mro will be set in the fixup phase based on these @@ -3978,34 +4012,37 @@ def read(cls, data: Buffer) -> TypeInfo: # rechecked, it can tell that the mro has changed. ti._mro_refs = read_str_list(data) ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.alt_promote = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.declared_metaclass = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.metaclass_type = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TUPLE_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TUPLE_TYPE ti.tuple_type = mypy.types.TupleType.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPED_DICT_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPED_DICT_TYPE ti.typeddict_type = mypy.types.TypedDictType.read(data) read_flags(data, ti, TypeInfo.FLAGS) - metadata = read_str(data) - if metadata != "{}": - ti.metadata = json.loads(metadata) - if read_bool(data): - ti.slots = set(read_str_list(data)) + ti.metadata = read_json(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LIST_STR + ti.slots = {read_str_bare(data) for _ in range(read_int_bare(data))} ti.deletable_attributes = read_str_list(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPE_VAR_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPE_VAR_TYPE ti.self_type = mypy.types.TypeVarType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ti.dataclass_transform_spec = DataclassTransformSpec.read(data) ti.deprecated = read_str_opt(data) + assert read_tag(data) == END_TAG return ti @@ -4290,14 +4327,15 @@ def write(self, data: Buffer) -> None: write_bool(data, self.no_args) write_bool(data, self.normalized) write_bool(data, self.python_3_12_type_alias) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAlias: fullname = read_str(data) module = read_str(data) target = mypy.types.read_type(data) - alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] - return TypeAlias( + alias_tvars = mypy.types.read_type_var_likes(data) + ret = TypeAlias( target, fullname, module, @@ -4308,6 +4346,8 @@ def read(cls, data: Buffer) -> TypeAlias: normalized=read_bool(data), python_3_12_type_alias=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret class PlaceholderNode(SymbolNode): @@ -4568,6 +4608,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: return stnode def write(self, data: Buffer, prefix: str, name: str) -> None: + write_tag(data, SYMBOL_TABLE_NODE) write_int(data, self.kind) write_bool(data, self.module_hidden) write_bool(data, self.module_public) @@ -4595,9 +4636,11 @@ def write(self, data: Buffer, prefix: str, name: str) -> None: if cross_ref is None: assert self.node is not None self.node.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> SymbolTableNode: + assert read_tag(data) == SYMBOL_TABLE_NODE sym = SymbolTableNode(read_int(data), None) sym.module_hidden = read_bool(data) sym.module_public = read_bool(data) @@ -4608,6 +4651,7 @@ def read(cls, data: Buffer) -> SymbolTableNode: sym.node = read_symbol(data) else: sym.cross_ref = cross_ref + assert read_tag(data) == END_TAG return sym @@ -4671,18 +4715,23 @@ def write(self, data: Buffer, fullname: str) -> None: if key == "__builtins__" or value.no_serialize: continue size += 1 - write_int(data, size) + # We intentionally tag SymbolTable as a simple dictionary str -> SymbolTableNode. + write_tag(data, DICT_STR_GEN) + write_int_bare(data, size) for key in sorted(self): value = self[key] if key == "__builtins__" or value.no_serialize: continue - write_str(data, key) + write_str_bare(data, key) value.write(data, fullname, key) @classmethod def read(cls, data: Buffer) -> SymbolTable: - size = read_int(data) - return SymbolTable([(read_str(data), SymbolTableNode.read(data)) for _ in range(size)]) + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return SymbolTable( + [(read_str_bare(data), SymbolTableNode.read(data)) for _ in range(size)] + ) class DataclassTransformSpec: @@ -4735,21 +4784,25 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: ) def write(self, data: Buffer) -> None: + write_tag(data, DT_SPEC) write_bool(data, self.eq_default) write_bool(data, self.order_default) write_bool(data, self.kw_only_default) write_bool(data, self.frozen_default) write_str_list(data, self.field_specifiers) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DataclassTransformSpec: - return DataclassTransformSpec( + ret = DataclassTransformSpec( eq_default=read_bool(data), order_default=read_bool(data), kw_only_default=read_bool(data), frozen_default=read_bool(data), field_specifiers=tuple(read_str_list(data)), ) + assert read_tag(data) == END_TAG + return ret def get_flags(node: Node, names: list[str]) -> list[str]: @@ -4889,17 +4942,19 @@ def local_definitions( yield from local_definitions(node.names, fullname, node) -MYPY_FILE: Final[Tag] = 0 -OVERLOADED_FUNC_DEF: Final[Tag] = 1 -FUNC_DEF: Final[Tag] = 2 -DECORATOR: Final[Tag] = 3 -VAR: Final[Tag] = 4 -TYPE_VAR_EXPR: Final[Tag] = 5 -PARAM_SPEC_EXPR: Final[Tag] = 6 -TYPE_VAR_TUPLE_EXPR: Final[Tag] = 7 -TYPE_INFO: Final[Tag] = 8 -TYPE_ALIAS: Final[Tag] = 9 -CLASS_DEF: Final[Tag] = 10 +# See docstring for mypy/cache.py for reserved tag ranges. +MYPY_FILE: Final[Tag] = 50 +OVERLOADED_FUNC_DEF: Final[Tag] = 51 +FUNC_DEF: Final[Tag] = 52 +DECORATOR: Final[Tag] = 53 +VAR: Final[Tag] = 54 +TYPE_VAR_EXPR: Final[Tag] = 55 +PARAM_SPEC_EXPR: Final[Tag] = 56 +TYPE_VAR_TUPLE_EXPR: Final[Tag] = 57 +TYPE_INFO: Final[Tag] = 58 +TYPE_ALIAS: Final[Tag] = 59 +CLASS_DEF: Final[Tag] = 60 +SYMBOL_TABLE_NODE: Final[Tag] = 61 def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: @@ -4926,8 +4981,9 @@ def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: assert False, f"Unknown symbol tag {tag}" -def read_overload_part(data: Buffer) -> OverloadPart: - tag = read_tag(data) +def read_overload_part(data: Buffer, tag: Tag | None = None) -> OverloadPart: + if tag is None: + tag = read_tag(data) if tag == DECORATOR: return Decorator.read(data) if tag == FUNC_DEF: diff --git a/mypy/types.py b/mypy/types.py index 426d560c2bf7e..6013f4c252988 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -8,9 +8,21 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, NewType, TypeVar, Union, cast, overload from typing_extensions import Self, TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) + import mypy.nodes from mypy.bogus_type import Bogus from mypy.cache import ( + DICT_STR_GEN, + END_TAG, + EXTRA_ATTRS, + LIST_GEN, + LITERAL_NONE, Buffer, Tag, read_bool, @@ -440,11 +452,13 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) assert self.alias is not None write_str(data, self.alias.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAliasType: alias = TypeAliasType(None, read_type_list(data)) alias.type_ref = read_str(data) + assert read_tag(data) == END_TAG return alias @@ -724,10 +738,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarType: - return TypeVarType( + ret = TypeVarType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -736,6 +751,8 @@ def read(cls, data: Buffer) -> TypeVarType: read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecFlavor: @@ -876,12 +893,13 @@ def write(self, data: Buffer) -> None: write_int(data, self.flavor) self.upper_bound.write(data) self.default.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecType: assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) - return ParamSpecType( + ret = ParamSpecType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -890,6 +908,8 @@ def read(cls, data: Buffer) -> ParamSpecType: read_type(data), prefix=prefix, ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleType(TypeVarLikeType): @@ -956,12 +976,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.min_len) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypeVarTupleType( + ret = TypeVarTupleType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -970,6 +991,8 @@ def read(cls, data: Buffer) -> TypeVarTupleType: read_type(data), min_len=read_int(data), ) + assert read_tag(data) == END_TAG + return ret def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var_tuple(self) @@ -1108,15 +1131,18 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) write_str_opt(data, self.original_str_expr) write_str_opt(data, self.original_str_fallback) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnboundType: - return UnboundType( + ret = UnboundType( read_str(data), read_type_list(data), original_str_expr=read_str_opt(data), original_str_fallback=read_str_opt(data), ) + assert read_tag(data) == END_TAG + return ret class CallableArgument(ProperType): @@ -1215,10 +1241,13 @@ def serialize(self) -> JsonDict: def write(self, data: Buffer) -> None: write_tag(data, UNPACK_TYPE) self.type.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnpackType: - return UnpackType(read_type(data)) + ret = UnpackType(read_type(data)) + assert read_tag(data) == END_TAG + return ret @classmethod def deserialize(cls, data: JsonDict) -> UnpackType: @@ -1326,15 +1355,19 @@ def write(self, data: Buffer) -> None: write_type_opt(data, self.source_any) write_int(data, self.type_of_any) write_str_opt(data, self.missing_import_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> AnyType: - if read_bool(data): - assert read_tag(data) == ANY_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == ANY_TYPE source_any = AnyType.read(data) else: source_any = None - return AnyType(read_int(data), source_any, read_str_opt(data)) + ret = AnyType(read_int(data), source_any, read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class UninhabitedType(ProperType): @@ -1384,9 +1417,11 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: def write(self, data: Buffer) -> None: write_tag(data, UNINHABITED_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UninhabitedType: + assert read_tag(data) == END_TAG return UninhabitedType() @@ -1423,9 +1458,11 @@ def deserialize(cls, data: JsonDict) -> NoneType: def write(self, data: Buffer) -> None: write_tag(data, NONE_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> NoneType: + assert read_tag(data) == END_TAG return NoneType() def is_singleton_type(self) -> bool: @@ -1478,10 +1515,13 @@ def deserialize(cls, data: JsonDict) -> DeletedType: def write(self, data: Buffer) -> None: write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DeletedType: - return DeletedType(read_str_opt(data)) + ret = DeletedType(read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret # Fake TypeInfo to be used as a placeholder during Instance de-serialization. @@ -1539,13 +1579,17 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: ) def write(self, data: Buffer) -> None: + write_tag(data, EXTRA_ATTRS) write_type_map(data, self.attrs) write_str_list(data, sorted(self.immutable)) write_str_opt(data, self.mod_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ExtraAttrs: - return ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + ret = ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class Instance(ProperType): @@ -1699,17 +1743,17 @@ def write(self, data: Buffer) -> None: write_tag(data, INSTANCE_OBJECT) else: write_tag(data, INSTANCE_SIMPLE) - write_str(data, type_ref) + write_str_bare(data, type_ref) return write_tag(data, INSTANCE_GENERIC) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) if self.extra_attrs is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.extra_attrs.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Instance: @@ -1743,17 +1787,21 @@ def read(cls, data: Buffer) -> Instance: return instance_cache.object_type if tag == INSTANCE_SIMPLE: inst = Instance(NOT_READY, []) - inst.type_ref = read_str(data) + inst.type_ref = read_str_bare(data) return inst assert tag == INSTANCE_GENERIC type_ref = read_str(data) inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref - if read_bool(data): - assert read_tag(data) == LITERAL_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LITERAL_TYPE inst.last_known_value = LiteralType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == EXTRA_ATTRS inst.extra_attrs = ExtraAttrs.read(data) + assert read_tag(data) == END_TAG return inst def copy_modified( @@ -2057,18 +2105,21 @@ def write(self, data: Buffer) -> None: write_str_opt_list(data, self.arg_names) write_type_list(data, self.variables) write_bool(data, self.imprecise_arg_kinds) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Parameters: - return Parameters( + ret = Parameters( read_type_list(data), # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, # we would spend ~20% of types deserialization time in Enum.__call__(). [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), imprecise_arg_kinds=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret def __hash__(self) -> int: return hash( @@ -2593,19 +2644,20 @@ def write(self, data: Buffer) -> None: write_bool(data, self.from_concatenate) write_bool(data, self.imprecise_arg_kinds) write_bool(data, self.unpack_kwargs) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> CallableType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return CallableType( + ret = CallableType( read_type_list(data), [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), read_type(data), fallback, name=read_str_opt(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), is_ellipsis_args=read_bool(data), implicit=read_bool(data), is_bound=read_bool(data), @@ -2615,6 +2667,8 @@ def read(cls, data: Buffer) -> CallableType: imprecise_arg_kinds=read_bool(data), unpack_kwargs=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret # This is a little safety net to prevent reckless special-casing of callables @@ -2694,13 +2748,16 @@ def deserialize(cls, data: JsonDict) -> Overloaded: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED) write_type_list(data, self.items) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Overloaded: items = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == CALLABLE_TYPE items.append(CallableType.read(data)) + assert read_tag(data) == END_TAG return Overloaded(items) @@ -2804,12 +2861,15 @@ def write(self, data: Buffer) -> None: self.partial_fallback.write(data) write_type_list(data, self.items) write_bool(data, self.implicit) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + ret = TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + assert read_tag(data) == END_TAG + return ret def copy_modified( self, *, fallback: Instance | None = None, items: list[Type] | None = None @@ -2987,14 +3047,17 @@ def write(self, data: Buffer) -> None: write_type_map(data, self.items) write_str_list(data, sorted(self.required_keys)) write_str_list(data, sorted(self.readonly_keys)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypedDictType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypedDictType( + ret = TypedDictType( read_type_map(data), set(read_str_list(data)), set(read_str_list(data)), fallback ) + assert read_tag(data) == END_TAG + return ret @property def is_final(self) -> bool: @@ -3248,13 +3311,16 @@ def write(self, data: Buffer) -> None: write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> LiteralType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) tag = read_tag(data) - return LiteralType(read_literal(data, tag), fallback) + ret = LiteralType(read_literal(data, tag), fallback) + assert read_tag(data) == END_TAG + return ret def is_singleton_type(self) -> bool: return self.is_enum_literal() or isinstance(self.value, bool) @@ -3361,10 +3427,13 @@ def write(self, data: Buffer) -> None: write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnionType: - return UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + ret = UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + assert read_tag(data) == END_TAG + return ret class PartialType(ProperType): @@ -3505,10 +3574,13 @@ def deserialize(cls, data: JsonDict) -> Type: def write(self, data: Buffer) -> None: write_tag(data, TYPE_TYPE) self.item.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Type: - return TypeType.make_normalized(read_type(data)) + ret = TypeType.make_normalized(read_type(data)) + assert read_tag(data) == END_TAG + return ret class PlaceholderType(ProperType): @@ -4172,37 +4244,41 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: return tuple(args) -TYPE_ALIAS_TYPE: Final[Tag] = 1 -TYPE_VAR_TYPE: Final[Tag] = 2 -PARAM_SPEC_TYPE: Final[Tag] = 3 -TYPE_VAR_TUPLE_TYPE: Final[Tag] = 4 -UNBOUND_TYPE: Final[Tag] = 5 -UNPACK_TYPE: Final[Tag] = 6 -ANY_TYPE: Final[Tag] = 7 -UNINHABITED_TYPE: Final[Tag] = 8 -NONE_TYPE: Final[Tag] = 9 -DELETED_TYPE: Final[Tag] = 10 -INSTANCE: Final[Tag] = 11 -CALLABLE_TYPE: Final[Tag] = 12 -OVERLOADED: Final[Tag] = 13 -TUPLE_TYPE: Final[Tag] = 14 -TYPED_DICT_TYPE: Final[Tag] = 15 -LITERAL_TYPE: Final[Tag] = 16 -UNION_TYPE: Final[Tag] = 17 -TYPE_TYPE: Final[Tag] = 18 -PARAMETERS: Final[Tag] = 19 - -INSTANCE_STR: Final[Tag] = 101 -INSTANCE_FUNCTION: Final[Tag] = 102 -INSTANCE_INT: Final[Tag] = 103 -INSTANCE_BOOL: Final[Tag] = 104 -INSTANCE_OBJECT: Final[Tag] = 105 -INSTANCE_SIMPLE: Final[Tag] = 106 -INSTANCE_GENERIC: Final[Tag] = 107 - - -def read_type(data: Buffer) -> Type: - tag = read_tag(data) +# See docstring for mypy/cache.py for reserved tag ranges. +# Instance-related tags. +INSTANCE: Final[Tag] = 80 +INSTANCE_SIMPLE: Final[Tag] = 81 +INSTANCE_GENERIC: Final[Tag] = 82 +INSTANCE_STR: Final[Tag] = 83 +INSTANCE_FUNCTION: Final[Tag] = 84 +INSTANCE_INT: Final[Tag] = 85 +INSTANCE_BOOL: Final[Tag] = 86 +INSTANCE_OBJECT: Final[Tag] = 87 + +# Other type tags. +TYPE_ALIAS_TYPE: Final[Tag] = 100 +TYPE_VAR_TYPE: Final[Tag] = 101 +PARAM_SPEC_TYPE: Final[Tag] = 102 +TYPE_VAR_TUPLE_TYPE: Final[Tag] = 103 +UNBOUND_TYPE: Final[Tag] = 104 +UNPACK_TYPE: Final[Tag] = 105 +ANY_TYPE: Final[Tag] = 106 +UNINHABITED_TYPE: Final[Tag] = 107 +NONE_TYPE: Final[Tag] = 108 +DELETED_TYPE: Final[Tag] = 109 +CALLABLE_TYPE: Final[Tag] = 110 +OVERLOADED: Final[Tag] = 111 +TUPLE_TYPE: Final[Tag] = 112 +TYPED_DICT_TYPE: Final[Tag] = 113 +LITERAL_TYPE: Final[Tag] = 114 +UNION_TYPE: Final[Tag] = 115 +TYPE_TYPE: Final[Tag] = 116 +PARAMETERS: Final[Tag] = 117 + + +def read_type(data: Buffer, tag: Tag | None = None) -> Type: + if tag is None: + tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == INSTANCE: return Instance.read(data) @@ -4245,8 +4321,7 @@ def read_type(data: Buffer) -> Type: assert False, f"Unknown type tag {tag}" -def read_function_like(data: Buffer) -> FunctionLike: - tag = read_tag(data) +def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: if tag == CALLABLE_TYPE: return CallableType.read(data) if tag == OVERLOADED: @@ -4254,51 +4329,61 @@ def read_function_like(data: Buffer) -> FunctionLike: assert False, f"Invalid type tag for FunctionLike {tag}" -def read_type_var_like(data: Buffer) -> TypeVarLikeType: - tag = read_tag(data) - if tag == TYPE_VAR_TYPE: - return TypeVarType.read(data) - if tag == PARAM_SPEC_TYPE: - return ParamSpecType.read(data) - if tag == TYPE_VAR_TUPLE_TYPE: - return TypeVarTupleType.read(data) - assert False, f"Invalid type tag for TypeVarLikeType {tag}" +def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: + """Specialized version of read_type_list() for lists of type variables.""" + assert read_tag(data) == LIST_GEN + ret: list[TypeVarLikeType] = [] + for _ in range(read_int_bare(data)): + tag = read_tag(data) + if tag == TYPE_VAR_TYPE: + ret.append(TypeVarType.read(data)) + elif tag == PARAM_SPEC_TYPE: + ret.append(ParamSpecType.read(data)) + elif tag == TYPE_VAR_TUPLE_TYPE: + ret.append(TypeVarTupleType.read(data)) + else: + assert False, f"Invalid type tag for TypeVarLikeType {tag}" + return ret def read_type_opt(data: Buffer) -> Type | None: - if read_bool(data): - return read_type(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + return read_type(data, tag) def write_type_opt(data: Buffer, value: Type | None) -> None: if value is not None: - write_bool(data, True) value.write(data) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_type_list(data: Buffer) -> list[Type]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_type(data) for _ in range(size)] def write_type_list(data: Buffer, value: Sequence[Type]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: item.write(data) def read_type_map(data: Buffer) -> dict[str, Type]: - size = read_int(data) - return {read_str(data): read_type(data) for _ in range(size)} + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_type(data) for _ in range(size)} def write_type_map(data: Buffer, value: dict[str, Type]) -> None: - write_int(data, len(value)) + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) for key in sorted(value): - write_str(data, key) + write_str_bare(data, key) value[key].write(data) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 6f6a110446ade..53e5e2238bd7f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -204,6 +204,10 @@ read_bool_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) char res = _READ(data, char) + if (unlikely((res != 0) & (res != 1))) { + PyErr_SetString(PyExc_ValueError, "invalid bool value"); + return CPY_BOOL_ERROR; + } return res; }