From 9d5e32c76ef28862cb373ee1a33db9268406b83f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 22 Jun 2026 13:39:02 +0100 Subject: [PATCH 01/50] [mypyc] Fix non-deterministic compiler output due to frozensets (#21631) Frozenset literals were emitted in an unpredictable order due to hash randomization. Make the order deterministic. I used some coding agent assist (esp. with tests). --------- Co-authored-by: Piotr Sawicki --- mypyc/codegen/emit.py | 44 ++++++++++++++++++++-------------- mypyc/codegen/literals.py | 24 +++++++++++++++---- mypyc/test/test_emit.py | 26 ++++++++++++++------ mypyc/test/test_literals.py | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 29 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index b89c91343e66c..57cce6a3fe8fe 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -2,14 +2,12 @@ from __future__ import annotations -import pprint import sys -import textwrap from collections.abc import Callable from typing import Final from mypyc.codegen.cstring import c_string_initializer -from mypyc.codegen.literals import Literals +from mypyc.codegen.literals import Literals, literal_sort_key from mypyc.common import ( ATTR_PREFIX, BITMAP_BITS, @@ -237,24 +235,16 @@ def attr(self, name: str) -> str: return ATTR_PREFIX + name def object_annotation(self, obj: object, line: str) -> str: - """Build a C comment with an object's string representation. + """Build a C comment with a literal value's string representation. - If the comment exceeds the line length limit, it's wrapped into a - multiline string (with the extra lines indented to be aligned with - the first line's comment). + This is a debugging aid that makes generated C easier to read. - If it contains illegal characters, an empty string is returned.""" - line_width = self._indent + len(line) - formatted = pprint.pformat(obj, compact=True, indent=1, width=max(90 - line_width, 20)) - if any(x in formatted for x in ("/*", "*/", "\0")): + If it contains illegal characters or is too long, return an empty string. + """ + formatted = stable_literal_repr(obj) + if any(x in formatted for x in ("/*", "*/", "\0")) or len(formatted) >= 256: return "" - - if "\n" in formatted: - first_line, rest = formatted.split("\n", maxsplit=1) - comment_continued = textwrap.indent(rest, (line_width + 3) * " ") - return f" /* {first_line}\n{comment_continued} */" - else: - return f" /* {formatted} */" + return f" /* {formatted} */" def emit_line(self, line: str = "", *, ann: object = None) -> None: if line.startswith("}"): @@ -1486,3 +1476,21 @@ def native_function_doc_initializer(func: FuncIR) -> str: return "NULL" docstring = f"{text_sig}\n--\n\n" return c_string_initializer(docstring.encode("ascii", errors="backslashreplace")) + + +def stable_literal_repr(obj: object) -> str: + """Return a single-line repr of a literal value. + + Behaves like repr() for most values, but renders frozenset members in a + deterministic order (frozenset iteration order is hash-seed dependent). + """ + if isinstance(obj, frozenset): + if not obj: + return "frozenset()" + items = ", ".join(stable_literal_repr(item) for item in sorted(obj, key=literal_sort_key)) + return "frozenset({" + items + "})" + elif isinstance(obj, tuple): + if len(obj) == 1: + return "(" + stable_literal_repr(obj[0]) + ",)" + return "(" + ", ".join(stable_literal_repr(item) for item in obj) + ")" + return repr(obj) diff --git a/mypyc/codegen/literals.py b/mypyc/codegen/literals.py index ed1ff93277167..a8d4650b5c294 100644 --- a/mypyc/codegen/literals.py +++ b/mypyc/codegen/literals.py @@ -65,7 +65,8 @@ def record_literal(self, value: LiteralValue) -> None: elif isinstance(value, frozenset): frozenset_literals = self.frozenset_literals if value not in frozenset_literals: - for item in value: + # Sort members so that we don't depend on frozenset iteration order. + for item in sorted(value, key=literal_sort_key): assert _is_literal_value(item) self.record_literal(item) frozenset_literals[value] = len(frozenset_literals) @@ -140,10 +141,14 @@ def encoded_tuple_values(self) -> list[str]: return self._encode_collection_values(self.tuple_literals) def encoded_frozenset_values(self) -> list[str]: - return self._encode_collection_values(self.frozenset_literals) + # Ensure deterministic frozenset item order by sorting items. + return self._encode_collection_values(self.frozenset_literals, sort_items=True) def _encode_collection_values( - self, values: dict[tuple[object, ...], int] | dict[frozenset[object], int] + self, + values: dict[tuple[object, ...], int] | dict[frozenset[object], int], + *, + sort_items: bool = False, ) -> list[str]: """Encode tuple/frozenset values into a C array. @@ -164,7 +169,8 @@ def _encode_collection_values( for i in range(count): value = value_by_index[i] result.append(str(len(value))) - for item in value: + items = sorted(value, key=literal_sort_key) if sort_items else value + for item in items: assert _is_literal_value(item) index = self.literal_index(item) result.append(str(index)) @@ -299,3 +305,13 @@ def _encode_complex_values(values: dict[complex, int]) -> list[str]: result.append(float_to_c(value.real)) result.append(float_to_c(value.imag)) return result + + +def literal_sort_key(value: object) -> tuple[object, ...]: + """Return a sort key for a literal value.""" + if isinstance(value, frozenset): + # Sort items to avoid depending on the unpredictable iteration order. + return ("frozenset", tuple(sorted(literal_sort_key(item) for item in value))) + elif isinstance(value, tuple): + return ("tuple", tuple(literal_sort_key(item) for item in value)) + return (type(value).__name__, repr(value)) diff --git a/mypyc/test/test_emit.py b/mypyc/test/test_emit.py index 285488e03c9ae..b2199b2dcb3b2 100644 --- a/mypyc/test/test_emit.py +++ b/mypyc/test/test_emit.py @@ -42,9 +42,23 @@ def test_reg(self) -> None: def test_object_annotation(self) -> None: assert self.emitter.object_annotation("hello, world", "line;") == " /* 'hello, world' */" - assert self.emitter.object_annotation(list(range(30)), "line;") == """\ - /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29] */""" + assert self.emitter.object_annotation(42, "line;") == " /* 42 */" + assert self.emitter.object_annotation((1, "x", None), "line;") == " /* (1, 'x', None) */" + # Annotations containing illegal C comment characters are dropped. + assert self.emitter.object_annotation("a /* b */ c", "line;") == "" + + def test_object_annotation_frozenset_is_deterministic(self) -> None: + assert ( + self.emitter.object_annotation(frozenset({"self", "cls"}), "line;") + == self.emitter.object_annotation(frozenset({"cls", "self"}), "line;") + == " /* frozenset({'cls', 'self'}) */" + ) + assert ( + self.emitter.object_annotation((frozenset({"b", "a"}),), "line;") + == self.emitter.object_annotation((frozenset({"a", "b"}),), "line;") + == " /* (frozenset({'a', 'b'}),) */" + ) + assert self.emitter.object_annotation(frozenset(), "line;") == " /* frozenset() */" def test_emit_line(self) -> None: emitter = self.emitter @@ -55,11 +69,9 @@ def test_emit_line(self) -> None: assert emitter.fragments == ["line;\n", "a {\n", " f();\n", "}\n"] emitter = Emitter(self.context, {}) emitter.emit_line("CPyStatics[0];", ann="hello, world") - emitter.emit_line("CPyStatics[1];", ann=list(range(30))) + emitter.emit_line("CPyStatics[1];", ann=42) assert emitter.fragments[0] == "CPyStatics[0]; /* 'hello, world' */\n" - assert emitter.fragments[1] == """\ -CPyStatics[1]; /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29] */\n""" + assert emitter.fragments[1] == "CPyStatics[1]; /* 42 */\n" def test_emit_undefined_value_for_simple_type(self) -> None: emitter = self.emitter diff --git a/mypyc/test/test_literals.py b/mypyc/test/test_literals.py index a8c17d10d30d0..f1dc7f2434414 100644 --- a/mypyc/test/test_literals.py +++ b/mypyc/test/test_literals.py @@ -10,6 +10,7 @@ _encode_int_values, _encode_str_values, format_str_literal, + literal_sort_key, ) @@ -88,3 +89,50 @@ def test_tuple_literal(self) -> None: "7", # Second tuple (length=4) "0", # Third tuple (length=0) ] + + def test_frozenset_literal_index_is_deterministic(self) -> None: + # Index assignment for members must not depend on frozenset iteration + # order (which is hash-seed dependent), so that generated code is + # reproducible. + lit1 = Literals() + lit1.record_literal(frozenset({"self", "cls"})) + lit2 = Literals() + lit2.record_literal(frozenset({"cls", "self"})) + for s in ("self", "cls"): + assert lit1.literal_index(s) == lit2.literal_index(s) + # Members are recorded in sorted order. + assert lit1.literal_index("cls") == 3 + assert lit1.literal_index("self") == 4 + + def test_frozenset_encoding_is_deterministic(self) -> None: + lit1 = Literals() + lit1.record_literal(frozenset({"self", "cls"})) + lit2 = Literals() + lit2.record_literal(frozenset({"cls", "self"})) + assert lit1.encoded_frozenset_values() == lit2.encoded_frozenset_values() + + def test_literal_sort_key_is_total_over_types(self) -> None: + # Heterogeneous, individually unorderable items must still be sorted. + values = ["x", b"y", 1, None, (1, 2), frozenset({1, 2})] + values_reversed = list(reversed(values)) + assert sorted(values, key=literal_sort_key) == sorted( + values_reversed, key=literal_sort_key + ) + + def test_literal_sort_key_with_frozenset(self) -> None: + assert literal_sort_key(frozenset({"a", "b"})) == literal_sort_key(frozenset({"b", "a"})) + assert literal_sort_key((frozenset({"a", "b"}),)) == literal_sort_key( + (frozenset({"b", "a"}),) + ) + assert literal_sort_key(frozenset({"a", frozenset({"b", "c"})})) == literal_sort_key( + frozenset({frozenset({"c", "b"}), "a"}) + ) + + def test_nested_frozenset_literal_index_is_deterministic(self) -> None: + lit1 = Literals() + lit1.record_literal(frozenset({frozenset({"a", "b"}), frozenset({"c", "d"})})) + lit2 = Literals() + lit2.record_literal(frozenset({frozenset({"d", "c"}), frozenset({"b", "a"})})) + for s in ("a", "b", "c", "d"): + assert lit1.literal_index(s) == lit2.literal_index(s) + assert lit1.encoded_frozenset_values() == lit2.encoded_frozenset_values() From 862af99f68160d783b289905f3907dc693a40af2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 22 Jun 2026 14:15:24 +0100 Subject: [PATCH 02/50] [mypyc] Fix non-deterministic ordering of spilled registers (#21632) Sort ops by their order in the basic blocks. --- mypyc/transform/spill.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/mypyc/transform/spill.py b/mypyc/transform/spill.py index d92dd661e7eb4..c2d4d4311375d 100644 --- a/mypyc/transform/spill.py +++ b/mypyc/transform/spill.py @@ -2,6 +2,9 @@ from __future__ import annotations +from collections.abc import Collection +from typing import cast + from mypyc.analysis.dataflow import AnalysisResult, analyze_live_regs, get_cfg from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.class_ir import ClassIR @@ -13,6 +16,7 @@ GetAttr, IncRef, LoadErrorValue, + Op, Register, SetAttr, Value, @@ -31,6 +35,19 @@ def insert_spills(ir: FuncIR, env: ClassIR) -> None: ir.blocks = spill_regs(ir.blocks, env, entry_live, live, ir.arg_regs[0]) +def sort_values(values: Collection[Op], blocks: list[BasicBlock]) -> list[Op]: + if len(values) > 1: + order = {} + i = 0 + for block in blocks: + for op in block.ops: + order[op] = i + i += 1 + return sorted(values, key=lambda v: order[v]) + else: + return list(values) + + def spill_regs( blocks: list[BasicBlock], env: ClassIR, @@ -48,7 +65,9 @@ def spill_regs( env_reg = self_reg spill_locs = {} - for i, val in enumerate(to_spill): + # Sort values to make the order deterministic. All the spilled values are + # known to be Op instances, so the cast is safe. + for i, val in enumerate(sort_values(cast(set[Op], to_spill), blocks)): name = f"{TEMP_ATTR_NAME}2_{i}" env.attributes[name] = val.type if val.type.error_overlap: From 1b206eb2850e0cad975476ec4753e60860c3b96a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 22 Jun 2026 15:36:32 +0100 Subject: [PATCH 03/50] Support .ff files with --cache-map (#21633) Previously only the .json extension was accepted, but we are now generating .ff cache files by default. --- mypy/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 9227e24680ce6..0cf624a3a5d7b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1677,13 +1677,15 @@ def process_cache_map( parser.error(f"Duplicate --cache-map source {source})") if not source.endswith(".py") and not source.endswith(".pyi"): parser.error(f"Invalid --cache-map source {source} (triple[0] must be *.py[i])") - if not meta_file.endswith(".meta.json"): + if not meta_file.endswith((".meta.json", ".meta.ff")): parser.error( - "Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)" % meta_file + "Invalid --cache-map meta_file %s (triple[1] must be *.meta.json or *.meta.ff)" + % meta_file ) - if not data_file.endswith(".data.json"): + if not data_file.endswith((".data.json", ".data.ff")): parser.error( - "Invalid --cache-map data_file %s (triple[2] must be *.data.json)" % data_file + "Invalid --cache-map data_file %s (triple[2] must be *.data.json or *.data.ff)" + % data_file ) options.cache_map[source] = (meta_file, data_file) From 3179030edcee070153b29cf21e06a0966beb088c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 22 Jun 2026 19:06:50 +0100 Subject: [PATCH 04/50] [mypyc] Fix handling of invalid codepoint values in librt.strings (#21634) `isidentifier`, `tolower` and `toupper` would terminate the process if passed a too high Unicode codepoint value. Fix so that all the `is*` functions return false for invalid codepoints and `tolower`/`toupper` functions return them unchanged. I used coding agent assist. --- mypy/typeshed/stubs/librt/librt/strings.pyi | 8 ++++-- mypyc/build.py | 7 ++++- mypyc/lib-rt/strings/librt_strings.h | 31 ++++++++++++++------- mypyc/test-data/run-librt-strings.test | 18 +++++++++--- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/strings.pyi b/mypy/typeshed/stubs/librt/librt/strings.pyi index 94e3b69abf243..af9ae60936d8e 100644 --- a/mypy/typeshed/stubs/librt/librt/strings.pyi +++ b/mypy/typeshed/stubs/librt/librt/strings.pyi @@ -42,7 +42,8 @@ def read_f64_le(b: bytes, index: i64, /) -> float: ... def read_f64_be(b: bytes, index: i64, /) -> float: ... # Codepoint classification helpers operating on i32 codepoints (typically -# obtained via ord(s[i])). Negative inputs return False. +# obtained via ord(s[i])). Out-of-range inputs (negative, or past the maximum +# Unicode code point 0x10FFFF) return False. def isspace(c: i32, /) -> bool: ... def isdigit(c: i32, /) -> bool: ... def isalnum(c: i32, /) -> bool: ... @@ -53,7 +54,8 @@ def isidentifier(c: i32, /) -> bool: ... # uppercase / lowercase expands to multiple codepoints (e.g. U+00DF # uppercases to "SS", U+FB01 to "FI"), returns the input unchanged so # the signature stays i32 -> i32. Use str.upper() / str.lower() for full -# Unicode case conversion when those cases matter. Negative inputs are -# returned unchanged. +# Unicode case conversion when those cases matter. Out-of-range inputs +# (negative, or past the maximum Unicode code point 0x10FFFF) are returned +# unchanged. def toupper(c: i32, /) -> i32: ... def tolower(c: i32, /) -> i32: ... diff --git a/mypyc/build.py b/mypyc/build.py index 13bd50fef3b1a..57438c7d5f52b 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -54,7 +54,12 @@ class ModDesc(NamedTuple): LIBRT_MODULES = [ ModDesc("librt.internal", ["internal/librt_internal.c"], [], ["internal"]), - ModDesc("librt.strings", ["strings/librt_strings.c"], [], ["strings"]), + ModDesc( + "librt.strings", + ["strings/librt_strings.c"], + ["strings/librt_strings.h", "strings/librt_strings_common.h"], + ["strings"], + ), ModDesc( "librt.base64", [ diff --git a/mypyc/lib-rt/strings/librt_strings.h b/mypyc/lib-rt/strings/librt_strings.h index 6c1942667ba44..33af57dc2a2a8 100644 --- a/mypyc/lib-rt/strings/librt_strings.h +++ b/mypyc/lib-rt/strings/librt_strings.h @@ -31,8 +31,9 @@ typedef struct { } StringWriterObject; // Codepoint classification helpers. Inputs are signed i32 for compatibility -// with mypyc's int32_rprimitive; negative values are non-codepoints and -// return false. Defined `static inline` so they compile statically into +// with mypyc's int32_rprimitive; out-of-range values (negative, or past the +// maximum Unicode code point 0x10FFFF) are non-codepoints and return false. +// Defined `static inline` so they compile statically into // both the librt.strings module and any mypyc-compiled extension that // includes this header, avoiding the capsule indirection that would dwarf // the work of a single Py_UNICODE_IS* macro call. @@ -58,12 +59,14 @@ static inline bool LibRTStrings_IsAlpha(int32_t c) { // PyUnicode_IsIdentifier on a 1-character string. Aborts via // CPyError_OutOfMemory on allocation failure to keep this ERR_NEVER. static inline bool LibRTStrings_IsIdentifier(int32_t c) { - if (c < 0) return false; - if (c < 128) { + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; } + // Reject negatives and code points past the Unicode maximum. + if ((uint32_t)c > 0x10FFFF) return false; PyObject *s = PyUnicode_FromOrdinal((int)c); if (s == NULL) { CPyError_OutOfMemory(); @@ -101,9 +104,13 @@ static inline int32_t LibRTStrings_ChangeCase_slow(int32_t c, const char *method // non-ASCII delegates to str.upper on a 1-character string. Returns the // input unchanged when uppercasing expands to multiple codepoints. static inline int32_t LibRTStrings_ToUpper(int32_t c) { - if (c < 0) return c; - if (c >= 'a' && c <= 'z') return c - 32; - if (c < 128) return c; + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { + if (c >= 'a' && c <= 'z') return c - 32; + return c; + } + // Negatives and code points past the Unicode maximum are returned unchanged. + if ((uint32_t)c > 0x10FFFF) return c; return LibRTStrings_ChangeCase_slow(c, "upper"); } @@ -111,9 +118,13 @@ static inline int32_t LibRTStrings_ToUpper(int32_t c) { // non-ASCII delegates to str.lower on a 1-character string. Returns the // input unchanged when lowercasing expands to multiple codepoints. static inline int32_t LibRTStrings_ToLower(int32_t c) { - if (c < 0) return c; - if (c >= 'A' && c <= 'Z') return c + 32; - if (c < 128) return c; + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { + if (c >= 'A' && c <= 'Z') return c + 32; + return c; + } + // Negatives and code points past the Unicode maximum are returned unchanged. + if ((uint32_t)c > 0x10FFFF) return c; return LibRTStrings_ChangeCase_slow(c, "lower"); } diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index 7efff12667d87..333b5962c4c99 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -1449,8 +1449,9 @@ from testutil import assertRaises def test_codepoint_classifiers() -> None: - # Negative values are not codepoints. - for bad in (i32(-1), i32(-113)): + # Out-of-range values are not codepoints: negative, just past the maximum + # valid code point (0x10FFFF), and the largest i32. + for bad in (i32(-1), i32(-113), i32(0x110000), i32(0x7FFFFFFF)): assert not isspace(bad) assert not isdigit(bad) assert not isalnum(bad) @@ -1485,6 +1486,10 @@ def test_codepoint_classifiers_via_any() -> None: assert f(ord(false_input)) is False # Negative values are valid i32, just not codepoints. assert f(-1) is False + # Values within i32 range but past the maximum code point (0x10FFFF) + # are not codepoints either. + assert f(0x110000) is False + assert f(0x7FFFFFFF) is False # Inputs outside i32 range raise OverflowError through the wrapper. with assertRaises(OverflowError, "codepoint out of i32 range"): f(1 << 40) @@ -1509,8 +1514,9 @@ def _expect(c: str, method: str) -> int: def test_codepoint_case_conversion() -> None: - # Negative inputs return unchanged. - for bad in (i32(-1), i32(-113)): + # Out-of-range inputs return unchanged: negative, just past the maximum + # valid code point (0x10FFFF), and the largest i32. + for bad in (i32(-1), i32(-113), i32(0x110000), i32(0x7FFFFFFF)): assert toupper(bad) == bad assert tolower(bad) == bad # Agree with str.upper / str.lower across the full Unicode range @@ -1534,6 +1540,10 @@ def test_codepoint_case_conversion_via_any() -> None: assert f(in_cp) == out_cp # Negative values are valid i32, returned unchanged. assert f(-1) == -1 + # Values within i32 range but past the maximum code point (0x10FFFF) + # are returned unchanged. + assert f(0x110000) == 0x110000 + assert f(0x7FFFFFFF) == 0x7FFFFFFF # Inputs outside i32 range raise OverflowError through the wrapper. with assertRaises(OverflowError, "codepoint out of i32 range"): f(1 << 40) From 24335663f560c3e2b1311464f0edd80e88ea5ff8 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 24 Jun 2026 17:04:30 +0200 Subject: [PATCH 05/50] Fix skipped imports considered stale (#21639) Fixes https://github.com/python/mypy/issues/21102 When the option `follow_imports = skip` is used, dependency states that are initially considered skipped might have their reason changed to not found in `load_graph`. If on the first mypy run, a given suppressed module is written to cache with its suppression reason set to skipped, and it's overridden to not found in a subsequent mypy run, then `suppressed_deps_opts` between the cache and the current run won't match and the module will be considered stale. To fix the issue, change the unconditional overwrite to `setdefault`. This also affects mypyc as the dependency being considered stale means that mypyc will needlessly recompile it instead of reading from cache. Add a unit test to confirm that the dependency output files are not overwritten with the fix. --- mypy/build.py | 2 +- mypyc/build.py | 54 ++++++++++++++------------- mypyc/codegen/emitmodule.py | 19 ++++++---- mypyc/test-data/run-multimodule.test | 33 ++++++++++++++++ mypyc/test/test_run.py | 4 ++ test-data/unit/check-incremental.test | 16 ++++++++ 6 files changed, 93 insertions(+), 35 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 7fc9526ccb2fe..a03a6eb8972cc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -4345,7 +4345,7 @@ def load_graph( for dep in st.ancestors + dependencies + st.suppressed: ignored = dep in st.suppressed_set and dep not in entry_points if ignored and dep not in added: - manager.missing_modules[dep] = SuppressionReason.NOT_FOUND + manager.missing_modules.setdefault(dep, SuppressionReason.NOT_FOUND) # TODO: for now we skip this in the daemon as a performance optimization. # This however creates a correctness issue, see #7777 and State.is_fresh(). if not manager.use_fine_grained_cache() or manager.options.warn_unused_configs: diff --git a/mypyc/build.py b/mypyc/build.py index 57438c7d5f52b..b46365d263e6a 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -328,34 +328,36 @@ def generate_c( emit_messages(options, e.messages, time.time() - t0, serious=(not e.use_stdout)) sys.exit(1) - t1 = time.time() - if result.errors: - emit_messages(options, result.errors, t1 - t0) - sys.exit(1) - - if compiler_options.verbose: - print(f"Parsed and typechecked in {t1 - t0:.3f}s") - - errors = Errors(options) - modules, ctext, mapper = emitmodule.compile_modules_to_c( - result, compiler_options=compiler_options, errors=errors, groups=groups - ) - t2 = time.time() - emit_messages(options, errors.new_messages(), t2 - t1) - if errors.num_errors: - # No need to stop the build if only warnings were emitted. - sys.exit(1) - - if compiler_options.verbose: - print(f"Compiled to C in {t2 - t1:.3f}s") - - if options.mypyc_annotation_file: - generate_annotated_html(options.mypyc_annotation_file, result, modules, mapper) + try: + t1 = time.time() + if result.errors: + emit_messages(options, result.errors, t1 - t0) + sys.exit(1) - # Collect SourceDep dependencies - source_deps = sorted(emitmodule.collect_source_dependencies(modules), key=lambda d: d.path) + if compiler_options.verbose: + print(f"Parsed and typechecked in {t1 - t0:.3f}s") - return ctext, "\n".join(format_modules(modules)), source_deps + errors = Errors(options) + modules, ctext, mapper = emitmodule.compile_modules_to_c( + result, compiler_options=compiler_options, errors=errors, groups=groups + ) + t2 = time.time() + emit_messages(options, errors.new_messages(), t2 - t1) + if errors.num_errors: + # No need to stop the build if only warnings were emitted. + sys.exit(1) + + if compiler_options.verbose: + print(f"Compiled to C in {t2 - t1:.3f}s") + + if options.mypyc_annotation_file: + generate_annotated_html(options.mypyc_annotation_file, result, modules, mapper) + + # Collect SourceDep dependencies + source_deps = sorted(emitmodule.collect_source_dependencies(modules), key=lambda d: d.path) + return ctext, "\n".join(format_modules(modules)), source_deps + finally: + result.manager.metastore.close() def build_using_shared_lib( diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index e2cd0a829dd77..6e3c122aa13f6 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -210,15 +210,18 @@ def parse_and_typecheck( ) -> BuildResult: assert options.strict_optional, "strict_optional must be turned on" mypyc_plugin = MypycPlugin(options, compiler_options, groups) - result = build( - sources=sources, - options=options, - alt_lib_path=alt_lib_path, - fscache=fscache, - extra_plugins=[mypyc_plugin], - ) - mypyc_plugin.metastore.close() + try: + result = build( + sources=sources, + options=options, + alt_lib_path=alt_lib_path, + fscache=fscache, + extra_plugins=[mypyc_plugin], + ) + finally: + mypyc_plugin.metastore.close() if result.errors: + result.manager.metastore.close() raise CompileError(result.errors) return result diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index e586a3cba22e9..4cf391d312dff 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -1246,6 +1246,39 @@ import native [rechecked other_a] +-- Test that importing a skipped module does not force rechecks. +[case testIncrementalCompilationFollowImportsSkip] +from other_dep import f +assert f() == 1 + +[file other_dep.py] +import skipped + +def f() -> int: + return 1 + +def g() -> int: + return 2 + +# Files under "skipped" are not typechecked because of the "--follow_imports = skip" option. +[file skipped/__init__.py] +x = 1 + +[file skipped/other_child.py] +import skipped + +def child() -> int: + return 1 + +[file native.py.2] +from other_dep import g +assert g() == 2 + +[file driver.py] +import native + +[rechecked native] + [case testSeparateCompilationWithUndefinedAttribute] from other_a import A diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index e7be5fcf8425a..9004a28ebf598 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -234,6 +234,10 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. options.per_module_options["unchecked.*"] = {"follow_imports": "error"} + # Avoid checking modules/packages named 'skipped', to provide a way + # to test interacting with code ignored by follow_imports=skip. + options.per_module_options["skipped"] = {"follow_imports": "skip"} + options.per_module_options["skipped.*"] = {"follow_imports": "skip"} source = build.BuildSource("native.py", "native", None) sources = [source] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 7ff9fbef06a53..18931dd9f152f 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -8204,3 +8204,19 @@ reveal_type(x) tmp/a.py:2: note: Revealed type is "builtins.int" [out2] tmp/a.py:2: note: Revealed type is "builtins.str" + +[case testIncrementalFollowImportsSkipStubWithSkippedRuntimeDependency] +# flags: --follow-imports=skip +import pkg.foo +[file pkg/__init__.py] + +[file pkg/foo.pyi] +from pkg import helper + +x = 1 +[file pkg/helper.py] +y = 2 +[rechecked] +[stale] +[out2] +[out3] From 5374fec1e0e676db587bebc36a1192b9819bc559 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 29 Jun 2026 17:31:02 +0200 Subject: [PATCH 06/50] [mypyc] Fix function wrapper memory leak (#21654) The `name` and `code` variables passed to the function wrapper init function are increfed inside it before storing them in the function wrapper object but they are not decrefed after the init function returns which creates a leak. Change the init function to instead steal `name` and `code` and document it. --- mypyc/lib-rt/function_wrapper.c | 21 +++++++++++++------- mypyc/test-data/run-async.test | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/function_wrapper.c b/mypyc/lib-rt/function_wrapper.c index 16b1b59303481..dfea6aed1d1c3 100644 --- a/mypyc/lib-rt/function_wrapper.c +++ b/mypyc/lib-rt/function_wrapper.c @@ -196,6 +196,7 @@ static PyObject* CPyFunction_Vectorcall(PyObject *func, PyObject *const *args, s } +// Steals ml, name, and code. Borrows module. static CPyFunction* CPyFunction_Init(CPyFunction *op, PyMethodDef *ml, PyObject* name, PyObject *module, PyObject* code, bool set_self) { PyCFunctionObject *cf = (PyCFunctionObject *)op; @@ -206,12 +207,10 @@ static CPyFunction* CPyFunction_Init(CPyFunction *op, PyMethodDef *ml, PyObject* Py_XINCREF(module); cf->m_module = module; - Py_INCREF(name); op->func_name = name; ((PyCMethodObject *)op)->mm_class = NULL; - Py_XINCREF(code); op->func_code = code; CPyFunction_func_vectorcall(op) = CPyFunction_Vectorcall; @@ -243,7 +242,7 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu PyCFunction func, int func_flags, const char *func_doc, int first_line, int code_flags, bool has_self_arg) { PyMethodDef *method = NULL; - PyObject *code = NULL, *op = NULL; + PyObject *code = NULL, *name = NULL, *op = NULL; bool set_self = false; #ifdef Py_GIL_DISABLED @@ -280,18 +279,24 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu if (unlikely(!code)) { goto err; } + name = PyUnicode_FromString(funcname); + if (unlikely(!name)) { + goto err; + } // Set m_self inside the function wrapper only if the wrapped function has no self arg // to pass m_self as the self arg when the function is called. // When the function has a self arg, it will come in the args vector passed to the // vectorcall handler. set_self = !has_self_arg; - op = (PyObject *)CPyFunction_Init(PyObject_GC_New(CPyFunction, CPyFunctionType), - method, PyUnicode_FromString(funcname), module, - code, set_self); - if (unlikely(!op)) { + CPyFunction *raw = PyObject_GC_New(CPyFunction, CPyFunctionType); + if (unlikely(!raw)) { goto err; } + op = (PyObject *)CPyFunction_Init(raw, method, name, module, code, set_self); + method = NULL; + name = NULL; + code = NULL; PyObject_GC_Track(op); return op; @@ -300,5 +305,7 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu if (method) { PyMem_Free(method); } + Py_XDECREF(name); + Py_XDECREF(code); return NULL; } diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 2733a31f3af2a..cf9b54368c277 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1416,6 +1416,7 @@ def run(x: object) -> object: ... [case testAsyncIntrospection] import asyncio +import gc import inspect import sys import weakref @@ -1423,6 +1424,8 @@ import weakref from functools import wraps from typing import Any, Callable, TypeVar, cast +from testutil import is_gil_disabled + def identity(val: int) -> int: return val @@ -1589,6 +1592,37 @@ def test_nested() -> None: assert is_coroutine(nested_wrapped_async) assert asyncio.run(nested_wrapped_async()) == 4 +def test_async_function_wrapper_code_refcount() -> None: + if is_gil_disabled(): + # On free-threaded builds the code object might be immortal, so the ref count test doesn't work. + return + code = getattr(identity_async, "__code__") + getrefcount = getattr(sys, "getrefcount") + # getrefcount sees the local code variable plus the wrapper-owned reference. + assert getrefcount(code) == 2, getrefcount(code) + +def test_nested_async_function_wrapper_code_refcount() -> None: + if is_gil_disabled(): + # On free-threaded builds the code object might be immortal, so the ref count test doesn't work. + return + def make_nested() -> Any: + async def nested_refcounted() -> int: + return 1 + + return nested_refcounted + + getrefcount = getattr(sys, "getrefcount") + fn = make_nested() + code = getattr(fn, "__code__") + before = getrefcount(code) + assert asyncio.run(fn()) == 1 + + del fn + gc.collect() + after = getrefcount(code) + assert before == after + 1, (before, after) + assert after == 1, after + [file asyncio/__init__.pyi] def run(x: object) -> object: ... From 629f456429d39abdf4754f517503200a7a4f8b17 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Jun 2026 17:21:20 +0100 Subject: [PATCH 07/50] [mypyc] Support generic primitives and add some generic primitives for vec (#21656) Generic primitives have `RTypeVar` types in parameter and/or return types, and these get expanded away when the primitive is added to IR. Generic primitives let us use lowering for various `vec` operations, many of which are generic. Using higher-level operations in the IR helps with various optimizations. It's also easier to verify that the generated IR is correct when the IR is less verbose. Add generic primitives for unsafe `vec` get item op as an initial use case. We can later use these for other `vec` operations as well. Used some coding agent assist (mostly for tests). --- mypyc/ir/ops.py | 21 ++- mypyc/ir/pprint.py | 9 +- mypyc/ir/rtypes.py | 53 ++++++++ mypyc/irbuild/ll_builder.py | 27 +++- mypyc/irbuild/vec.py | 23 +++- mypyc/lower/registry.py | 2 +- mypyc/lower/vec_ops.py | 18 +++ mypyc/primitives/librt_vecs_ops.py | 23 +++- mypyc/primitives/registry.py | 8 +- mypyc/rt_expandtype.py | 32 +++++ mypyc/test-data/irbuild-vec-i64.test | 173 ++++++++---------------- mypyc/test-data/irbuild-vec-misc.test | 51 ++----- mypyc/test-data/irbuild-vec-nested.test | 160 +++++++++++++--------- mypyc/test-data/irbuild-vec-t.test | 75 +++++++--- mypyc/test-data/lowering-vec.test | 155 +++++++++++++++++++++ mypyc/test-data/refcount.test | 110 +++++---------- mypyc/test/test_expand_rtype.py | 48 +++++++ mypyc/test/test_lowering.py | 2 +- 18 files changed, 665 insertions(+), 325 deletions(-) create mode 100644 mypyc/lower/vec_ops.py create mode 100644 mypyc/rt_expandtype.py create mode 100644 mypyc/test-data/lowering-vec.test create mode 100644 mypyc/test/test_expand_rtype.py diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 485aa84886b32..4bc7671b82082 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -39,6 +39,7 @@ class to enable the new behavior. Sometimes adding a new abstract RStruct, RTuple, RType, + RTypeVar, RUnion, RVec, RVoid, @@ -703,7 +704,7 @@ def __init__( self, name: str, arg_types: list[RType], - return_type: RType, # TODO: What about generic? + return_type: RType, var_arg_type: RType | None, truncated_type: RType | None, c_function_name: str | None, @@ -716,6 +717,7 @@ def __init__( is_pure: bool, experimental: bool, dependencies: list[Dependency] | None, + type_params: list[RTypeVar] | None, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -749,6 +751,7 @@ def __init__( # If this flag is set, the primitive has native integer types and must # be matched using more complex rules. self.is_ambiguous = any(has_fixed_width_int(t) for t in arg_types) + self.type_params = None if not type_params else type_params def __repr__(self) -> str: return f"" @@ -776,11 +779,23 @@ class PrimitiveOp(RegisterOp): code paths for short and long representations. """ - def __init__(self, args: list[Value], desc: PrimitiveDescription, line: int = -1) -> None: + def __init__( + self, + args: list[Value], + desc: PrimitiveDescription, + line: int = -1, + *, + arg_types: list[RType] | None = None, + return_type: RType | None = None, + type_args: list[RType] | None = None, + ) -> None: self.error_kind = desc.error_kind super().__init__(line) self.args = args - self.type = desc.return_type + self.arg_types = arg_types if arg_types is not None else desc.arg_types + self.type = return_type if return_type is not None else desc.return_type + self.is_borrowed = desc.is_borrowed + self.type_args = type_args self.desc = desc def sources(self) -> list[Value]: diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index d0db9f2460a1d..734426ca42de9 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -231,10 +231,15 @@ def visit_call_c(self, op: CallC) -> str: def visit_primitive_op(self, op: PrimitiveOp) -> str: args_str = ", ".join(self.format("%r", arg) for arg in op.args) + if op.type_args: + joined = ", ".join(str(arg) for arg in op.type_args) + type_args = f"[{joined}]" + else: + type_args = "" if op.is_void: - return self.format("%s %s", op.desc.name, args_str) + return self.format("%s%s %s", op.desc.name, type_args, args_str) else: - return self.format("%r = %s %s", op, op.desc.name, args_str) + return self.format("%r = %s%s %s", op, op.desc.name, type_args, args_str) def visit_truncate(self, op: Truncate) -> str: return self.format("%r = truncate %r: %t to %t", op, op.src, op.src_type, op.type) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index db29f9e304d8d..9d13ecc83175d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -152,6 +152,9 @@ def visit_rprimitive(self, typ: RPrimitive, /) -> T: def visit_rinstance(self, typ: RInstance, /) -> T: raise NotImplementedError + def visit_rtypevar(self, typ: RTypeVar, /) -> T: + raise RuntimeError("RTypeVar should not be encountered here") + @abstractmethod def visit_rvec(self, typ: RVec, /) -> T: raise NotImplementedError @@ -747,6 +750,12 @@ def visit_rarray(self, t: RArray) -> str: def visit_rvoid(self, t: RVoid) -> str: assert False, "rvoid in tuple?" + def visit_rtypevar(self, typ: RTypeVar) -> str: + # We need to return something to support generic RTuples, etc. Make sure + # the return value is invalid C so that generic RTuples must be expanded + # before they can be used in IR. + return f"!RTypeVar {typ.id} invalid!" + @final class RTuple(RType): @@ -1013,6 +1022,50 @@ def serialize(self) -> str: return self.name +@final +class RTypeVar(RType): + """Type variable type used for generic primitive ops. + + This allows having generic primitive operations like vec get item, which is + parametrized by the vec item type. + + These types are not valid in any other context outside PrimitiveDescription, + and they will always be substituted during the construction of a PrimitiveOp. + + NOTE: This is not related to mypy's TypeVarType! + """ + + def __init__(self, id: int) -> None: + self.id = id + + @property + def may_be_immortal(self) -> bool: + # RTypeVar must always be substituted before use, so this should never matter. + return False + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rtypevar(self) + + def __str__(self) -> str: + return f"" + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> TypeGuard[RTypeVar]: + return isinstance(other, RTypeVar) and other.id == self.id + + def __hash__(self) -> int: + return self.id ^ 12345 + + def serialize(self) -> JsonDict: + return {".class": "RTypeVar", "id": self.id} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RTypeVar: + return RTypeVar(data["id"]) + + @final class RVec(RType): """librt.vecs.vec[T]""" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c19eded77464e..c0ad1cf1f8264 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -206,6 +206,7 @@ new_tuple_with_length_op, sequence_tuple_op, ) +from mypyc.rt_expandtype import expand_rtype from mypyc.rt_subtype import is_runtime_subtype from mypyc.sametype import is_same_type from mypyc.subtype import is_subtype @@ -2334,6 +2335,7 @@ def primitive_op( args: list[Value], line: int, result_type: RType | None = None, + type_args: list[RType] | None = None, ) -> Value: """Add a primitive op.""" # Does this primitive map into calling a Python C API @@ -2363,10 +2365,20 @@ def primitive_op( # This primitive gets transformed in a lowering pass to # lower-level IR ops using a custom transform function. + # Evaluate argument and return types for generic primitives + return_type = None + if desc.type_params is not None: + assert type_args is not None, "Generic primitive op requires explicit type arguments" + assert len(type_args) == len(desc.type_params) + arg_types = [expand_rtype(arg_type, type_args) for arg_type in desc.arg_types] + return_type = expand_rtype(desc.return_type, type_args) + else: + arg_types = desc.arg_types + coerced = [] # Coerce fixed number arguments - for i in range(min(len(args), len(desc.arg_types))): - formal_type = desc.arg_types[i] + for i in range(min(len(args), len(arg_types))): + formal_type = arg_types[i] arg = args[i] assert formal_type is not None # TODO arg = self.coerce(arg, formal_type, line) @@ -2374,7 +2386,16 @@ def primitive_op( assert desc.ordering is None assert desc.var_arg_type is None assert not desc.extra_int_constants - target = self.add(PrimitiveOp(coerced, desc, line=line)) + target = self.add( + PrimitiveOp( + coerced, + desc, + line=line, + arg_types=arg_types, + return_type=return_type, + type_args=type_args, + ) + ) if desc.is_borrowed: # If the result is borrowed, force the arguments to be # kept alive afterwards, as otherwise the result might be diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index bfcfabee45c21..38615ebe16026 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -52,6 +52,7 @@ vec_api_by_item_type, vec_item_type_tags, ) +from mypyc.primitives.librt_vecs_ops import vec_get_item_unsafe_borrow_op, vec_get_item_unsafe_op if TYPE_CHECKING: from mypyc.irbuild.ll_builder import LowLevelIRBuilder @@ -316,9 +317,10 @@ def vec_get_item( ) -> Value: """Generate inlined vec __getitem__ call. - We inline this, since it's simple but performance-critical. + We inline the length and bounds check, since they are simple but + performance-critical. The actual item load is emitted as a generic primitive + op that is lowered later. """ - # TODO: Support more item types # TODO: Support more index types len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) @@ -328,7 +330,22 @@ def vec_get_item( def vec_get_item_unsafe( builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False ) -> Value: - """Get vec item, assuming index is non-negative and within bounds.""" + """Get vec item, assuming index is non-negative and within bounds. + + This emits a generic primitive op that is inlined during lowering. + """ + assert isinstance(base.type, RVec) + if can_borrow: + desc = vec_get_item_unsafe_borrow_op + else: + desc = vec_get_item_unsafe_op + return builder.primitive_op(desc, [base, index], line, type_args=[base.type.item_type]) + + +def vec_get_item_unsafe_lower( + builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False +) -> Value: + """Generate the low-level IR for an unsafe vec item load.""" assert isinstance(base.type, RVec) index = as_platform_int(builder, index, line) vtype = base.type diff --git a/mypyc/lower/registry.py b/mypyc/lower/registry.py index dec6a24b9417a..36262c4a011a3 100644 --- a/mypyc/lower/registry.py +++ b/mypyc/lower/registry.py @@ -26,4 +26,4 @@ def wrapper(f: LF) -> LF: # Import various modules that set up global state. -from mypyc.lower import int_ops, list_ops, misc_ops # noqa: F401 +from mypyc.lower import int_ops, list_ops, misc_ops, vec_ops # noqa: F401 diff --git a/mypyc/lower/vec_ops.py b/mypyc/lower/vec_ops.py new file mode 100644 index 0000000000000..768c8e0073af8 --- /dev/null +++ b/mypyc/lower/vec_ops.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from mypyc.ir.ops import Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.irbuild.vec import vec_get_item_unsafe_lower +from mypyc.lower.registry import lower_primitive_op + + +@lower_primitive_op("vec_get_item_unsafe") +def vec_get_item_unsafe(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + base, index = args + return vec_get_item_unsafe_lower(builder, base, index, line, can_borrow=False) + + +@lower_primitive_op("vec_get_item_unsafe_borrow") +def vec_get_item_unsafe_borrow(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + base, index = args + return vec_get_item_unsafe_lower(builder, base, index, line, can_borrow=True) diff --git a/mypyc/primitives/librt_vecs_ops.py b/mypyc/primitives/librt_vecs_ops.py index e4852d5387069..901c779a2bfd2 100644 --- a/mypyc/primitives/librt_vecs_ops.py +++ b/mypyc/primitives/librt_vecs_ops.py @@ -1,13 +1,15 @@ from mypyc.ir.deps import LIBRT_VECS, VECS_EXTRA_OPS from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + RTypeVar, RVec, bit_rprimitive, bytes_rprimitive, + int64_rprimitive, object_rprimitive, uint8_rprimitive, ) -from mypyc.primitives.registry import function_op +from mypyc.primitives.registry import custom_primitive_op, function_op # isinstance(obj, vec) isinstance_vec = function_op( @@ -28,3 +30,22 @@ error_kind=ERR_MAGIC, dependencies=[LIBRT_VECS, VECS_EXTRA_OPS], ) + +# Get vec item, assuming the index is valid (no bounds check) +vec_get_item_unsafe_op = custom_primitive_op( + name="vec_get_item_unsafe", + arg_types=[RVec(RTypeVar(0)), int64_rprimitive], + return_type=RTypeVar(0), + error_kind=ERR_NEVER, + type_params=[RTypeVar(0)], +) + +# Like vec_get_item_unsafe, but the result is a borrowed reference +vec_get_item_unsafe_borrow_op = custom_primitive_op( + name="vec_get_item_unsafe_borrow", + arg_types=[RVec(RTypeVar(0)), int64_rprimitive], + is_borrowed=True, + return_type=RTypeVar(0), + error_kind=ERR_NEVER, + type_params=[RTypeVar(0)], +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index e22a044d9bb27..22422987b4277 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -41,7 +41,7 @@ from mypyc.ir.deps import Dependency from mypyc.ir.ops import PrimitiveDescription, StealsDescription -from mypyc.ir.rtypes import RType +from mypyc.ir.rtypes import RType, RTypeVar # Error kind for functions that return negative integer on exception. This # is only used for primitives. We translate it away during IR building. @@ -154,6 +154,7 @@ def method_op( is_pure=is_pure, experimental=experimental, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -204,6 +205,7 @@ def function_op( is_pure=False, experimental=experimental, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -253,6 +255,7 @@ def binary_op( is_pure=False, experimental=False, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -313,6 +316,7 @@ def custom_primitive_op( is_pure: bool = False, experimental: bool = False, dependencies: list[Dependency] | None = None, + type_params: list[RTypeVar] | None = None, ) -> PrimitiveDescription: """Define a primitive op that can't be automatically generated based on the AST. @@ -336,6 +340,7 @@ def custom_primitive_op( is_pure=is_pure, experimental=experimental, dependencies=dependencies, + type_params=type_params, ) @@ -380,6 +385,7 @@ def unary_op( is_pure=is_pure, experimental=False, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc diff --git a/mypyc/rt_expandtype.py b/mypyc/rt_expandtype.py new file mode 100644 index 0000000000000..8537e6777ccb5 --- /dev/null +++ b/mypyc/rt_expandtype.py @@ -0,0 +1,32 @@ +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RStruct, + RTuple, + RType, + RTypeVar, + RUnion, + RVec, + RVoid, +) + + +def expand_rtype(typ: RType, type_args: list[RType]) -> RType: + if isinstance(typ, (RPrimitive, RInstance, RVoid)): + # Atomic types can't contain type variables + return typ + elif isinstance(typ, RTypeVar): + return type_args[typ.id] + elif isinstance(typ, RVec): + return RVec(expand_rtype(typ.item_type, type_args)) + elif isinstance(typ, RUnion): + return RUnion([expand_rtype(item, type_args) for item in typ.items]) + elif isinstance(typ, RTuple): + return RTuple([expand_rtype(item, type_args) for item in typ.types]) + elif isinstance(typ, RStruct): + assert False, "Generic RStruct type not supported" + elif isinstance(typ, RArray): + assert False, "Generic RArray type not supported" + else: + assert False, r"Unexpected type {typ!r}" diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index aeab3ed9f2a8b..af69f30924d46 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -64,11 +64,7 @@ def f(v, i): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: i64 + r5, r6 :: i64 L0: r0 = v.len r1 = i < r0 :: unsigned @@ -86,15 +82,10 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[i64] v, r5 + return r6 [case testVecI64GetItem_32bit] -# The IR is quite verbose, but it's acceptable since 32-bit targets are not common any more from librt.vecs import vec from mypy_extensions import i64 @@ -110,13 +101,7 @@ def f(v, i): r3 :: i64 r4 :: bit r5 :: bool - r6 :: i64 - r7, r8 :: bit - r9 :: native_int - r10 :: ptr - r11 :: native_int - r12 :: ptr - r13 :: i64 + r6, r7 :: i64 L0: r0 = v.len r1 = extend signed r0: native_int to i64 @@ -135,24 +120,8 @@ L3: L4: r6 = i L5: - r7 = r6 < 2147483648 :: signed - if r7 goto L6 else goto L8 :: bool -L6: - r8 = r6 >= -2147483648 :: signed - if r8 goto L7 else goto L8 :: bool -L7: - r9 = truncate r6: i64 to native_int - goto L9 -L8: - CPyInt32_Overflow() - unreachable -L9: - r10 = v.items - r11 = r9 * 8 - r12 = r10 + r11 - r13 = load_mem r12 :: i64* - keep_alive v - return r13 + r7 = vec_get_item_unsafe[i64] v, r6 + return r7 [case testVecI64Append] from librt.vecs import vec, append @@ -411,7 +380,7 @@ L3: L4: return r1 -[case testVecI64FastComprehensionFromVec] +[case testVecI64FastComprehensionFromVec_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -426,14 +395,11 @@ def f(n, v): r1 :: vec[i64] r2, r3 :: native_int r4 :: bit - r5 :: ptr - r6 :: native_int + r5, x, r6 :: i64 r7 :: ptr - r8, x, r9 :: i64 - r10 :: ptr - r11 :: native_int - r12 :: ptr - r13 :: native_int + r8 :: native_int + r9 :: ptr + r10 :: native_int L0: r0 = v.len r1 = VecI64Api.alloc(r0, r0) @@ -443,21 +409,17 @@ L1: r4 = r2 < r3 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = v.items - r6 = r2 * 8 - r7 = r5 + r6 - r8 = load_mem r7 :: i64* - x = r8 - keep_alive v - r9 = x + 1 - r10 = r1.items - r11 = r2 * 8 - r12 = r10 + r11 - set_mem r12, r9 :: i64* + r5 = vec_get_item_unsafe[i64] v, r2 + x = r5 + r6 = x + 1 + r7 = r1.items + r8 = r2 * 8 + r9 = r7 + r8 + set_mem r9, r6 :: i64* keep_alive r1 L3: - r13 = r2 + 1 - r2 = r13 + r10 = r2 + 1 + r2 = r10 goto L1 L4: return r1 @@ -499,7 +461,7 @@ L3: L4: return r1 -[case testVecI64ForLoop] +[case testVecI64ForLoop_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -514,11 +476,8 @@ def f(v): t :: i64 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, x, r7 :: i64 - r8 :: native_int + r3, x, r4 :: i64 + r5 :: native_int L0: t = 0 r0 = 0 @@ -527,17 +486,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 8 - r5 = r3 + r4 - r6 = load_mem r5 :: i64* - x = r6 - keep_alive v - r7 = t + 1 - t = r7 + r3 = vec_get_item_unsafe[i64] v, r0 + x = r3 + r4 = t + 1 + t = r4 L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return t @@ -601,11 +556,7 @@ def f(v): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: i64 + r5, r6 :: i64 L0: r0 = v.len r1 = 0 < r0 :: unsigned @@ -623,12 +574,8 @@ L3: L4: r5 = 0 L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[i64] v, r5 + return r6 [case testVecI64Slicing_64bit] from librt.vecs import vec @@ -720,20 +667,16 @@ def inplace(v, n, m): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9, r10 :: i64 - r11 :: native_int - r12 :: bit + r5, r6, r7 :: i64 + r8 :: native_int + r9 :: bit + r10 :: i64 + r11 :: bit + r12 :: bool r13 :: i64 - r14 :: bit - r15 :: bool - r16 :: i64 - r17 :: ptr - r18 :: i64 - r19 :: ptr + r14 :: ptr + r15 :: i64 + r16 :: ptr L0: r0 = v.len r1 = n < r0 :: unsigned @@ -751,32 +694,28 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - r10 = r9 + m - r11 = v.len - r12 = n < r11 :: unsigned - if r12 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe[i64] v, r5 + r7 = r6 + m + r8 = v.len + r9 = n < r8 :: unsigned + if r9 goto L9 else goto L6 :: bool L6: - r13 = n + r11 - r14 = r13 < r11 :: unsigned - if r14 goto L8 else goto L7 :: bool + r10 = n + r8 + r11 = r10 < r8 :: unsigned + if r11 goto L8 else goto L7 :: bool L7: - r15 = raise IndexError + r12 = raise IndexError unreachable L8: - r16 = r13 + r13 = r10 goto L10 L9: - r16 = n + r13 = n L10: - r17 = v.items - r18 = r16 * 8 - r19 = r17 + r18 - set_mem r19, r10 :: i64* + r14 = v.items + r15 = r13 * 8 + r16 = r14 + r15 + set_mem r16, r7 :: i64* keep_alive v return 1 diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index c0d4325e38fcc..22037d8597a73 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -125,10 +125,7 @@ def get_item_bool(v, i): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: bool + r6 :: bool L0: r0 = v.len r1 = i < r0 :: unsigned @@ -146,12 +143,8 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 1 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.bool* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[bool] v, r5 + return r6 [case testVecMiscPop] from librt.vecs import vec, pop @@ -209,7 +202,7 @@ L0: r0 = VecFloatApi.slice(v, x, y) return r0 -[case testVecMiscForLoop] +[case testVecMiscForLoop_64bit] from librt.vecs import vec, remove from mypy_extensions import i64, i16 @@ -225,11 +218,8 @@ def for_bool(v): s :: i16 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, x, r7 :: i16 - r8 :: native_int + r3, x, r4 :: i16 + r5 :: native_int L0: s = 0 r0 = 0 @@ -238,17 +228,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 2 - r5 = r3 + r4 - r6 = load_mem r5 :: i16* - x = r6 - keep_alive v - r7 = s + x - s = r7 + r3 = vec_get_item_unsafe[i16] v, r0 + x = r3 + r4 = s + x + s = r4 L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return s @@ -270,10 +256,7 @@ def get_item_nested(v, i): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i32] + r6 :: vec[i32] L0: r0 = v.len r1 = i < r0 :: unsigned @@ -291,12 +274,8 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[i32]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[i32]] v, r5 + return r6 [case testVecMiscNestedPop_64bit] from librt.vecs import vec, pop diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 1fe42a880d5b0..dd49d9475812f 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -206,10 +206,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[str] + r6 :: vec[str] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -227,12 +224,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[str]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[str]] v, r5 + return r6 [case testVecNestedI64GetItem_64bit] from librt.vecs import vec @@ -250,10 +243,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] + r6 :: vec[i64] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -271,12 +261,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[i64]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[i64]] v, r5 + return r6 [case testVecNestedI64GetItemWithBorrow_64bit] from librt.vecs import vec @@ -294,20 +280,13 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] - r10 :: native_int - r11 :: bit - r12 :: i64 - r13 :: bit - r14 :: bool - r15 :: i64 - r16 :: ptr - r17 :: i64 - r18 :: ptr - r19 :: i64 + r6 :: vec[i64] + r7 :: native_int + r8 :: bit + r9 :: i64 + r10 :: bit + r11 :: bool + r12, r13 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned @@ -325,32 +304,94 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = borrow load_mem r8 :: vec[i64]* - r10 = r9.len - r11 = n < r10 :: unsigned - if r11 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe_borrow[vec[i64]] v, r5 + r7 = r6.len + r8 = n < r7 :: unsigned + if r8 goto L9 else goto L6 :: bool +L6: + r9 = n + r7 + r10 = r9 < r7 :: unsigned + if r10 goto L8 else goto L7 :: bool +L7: + r11 = raise IndexError + unreachable +L8: + r12 = r9 + goto L10 +L9: + r12 = n +L10: + r13 = vec_get_item_unsafe[i64] r6, r12 + keep_alive v, r5 + return r13 + +[case testVecNestedStrGetItemWithBorrow_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +class C: + v: vec[vec[str]] + + def f(self, n: i64) -> str: + # The intermediate vec is borrowed, but the result must be owned. + return self.v[n][n] +[out] +def C.f(self, n): + self :: __main__.C + n :: i64 + r0 :: vec[vec[str]] + r1 :: native_int + r2 :: bit + r3 :: i64 + r4 :: bit + r5 :: bool + r6 :: i64 + r7 :: vec[str] + r8 :: native_int + r9 :: bit + r10 :: i64 + r11 :: bit + r12 :: bool + r13 :: i64 + r14 :: str +L0: + r0 = borrow self.v + r1 = r0.len + r2 = n < r1 :: unsigned + if r2 goto L4 else goto L1 :: bool +L1: + r3 = n + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L3 else goto L2 :: bool +L2: + r5 = raise IndexError + unreachable +L3: + r6 = r3 + goto L5 +L4: + r6 = n +L5: + r7 = vec_get_item_unsafe_borrow[vec[str]] r0, r6 + r8 = r7.len + r9 = n < r8 :: unsigned + if r9 goto L9 else goto L6 :: bool L6: - r12 = n + r10 - r13 = r12 < r10 :: unsigned - if r13 goto L8 else goto L7 :: bool + r10 = n + r8 + r11 = r10 < r8 :: unsigned + if r11 goto L8 else goto L7 :: bool L7: - r14 = raise IndexError + r12 = raise IndexError unreachable L8: - r15 = r12 + r13 = r10 goto L10 L9: - r15 = n + r13 = n L10: - r16 = r9.items - r17 = r15 * 8 - r18 = r16 + r17 - r19 = load_mem r18 :: i64* - keep_alive v, r9 - return r19 + r14 = vec_get_item_unsafe[str] r7, r13 + keep_alive self, r0, r6 + return r14 [case testVecDoublyNestedGetItem_64bit] from librt.vecs import vec @@ -368,10 +409,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[vec[str]] + r6 :: vec[vec[str]] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -389,12 +427,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[vec[str]]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[vec[str]]] v, r5 + return r6 [case testVecNestedCreateWithCap_64bit] from librt.vecs import vec diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 63ad14bc2d7a2..ee48d81fc29c8 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -215,10 +215,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: str + r6 :: str L0: r0 = v.len r1 = n < r0 :: unsigned @@ -236,12 +233,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.str* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[str] v, r5 + return r6 [case testVecTOptionalGetItem_64bit] from librt.vecs import vec @@ -260,10 +253,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: union[str, None] + r6 :: union[str, None] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -281,12 +271,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: union* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[union[str, None]] v, r5 + return r6 [case testNewTPopLast] from typing import Tuple @@ -524,3 +510,52 @@ L0: r1 = r0 r2 = VecTApi.from_iterable(r1, a, 0) return r2 + +[case testVecTBorrowGetItem_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +class A: + x: str + +def f(v: vec[A], n: i64) -> int: + return len(v[n].x) +[out] +def f(v, n): + v :: vec[__main__.A] + n :: i64 + r0 :: native_int + r1 :: bit + r2 :: i64 + r3 :: bit + r4 :: bool + r5 :: i64 + r6 :: __main__.A + r7 :: str + r8 :: native_int + r9 :: bit + r10 :: short_int +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L4 else goto L1 :: bool +L1: + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool +L2: + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = vec_get_item_unsafe_borrow[__main__.A] v, r5 + r7 = r6.x + keep_alive v, r5 + r8 = CPyStr_Size_size_t(r7) + r9 = r8 >= 0 :: signed + r10 = r8 << 1 + return r10 diff --git a/mypyc/test-data/lowering-vec.test b/mypyc/test-data/lowering-vec.test new file mode 100644 index 0000000000000..374d336b98d8e --- /dev/null +++ b/mypyc/test-data/lowering-vec.test @@ -0,0 +1,155 @@ +[case testLowerVecI64GetItem_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(i: i64) -> i64: + v = vec[i64]() + return v[i] +[out] +def f(i): + i :: i64 + r0, v :: vec[i64] + r1 :: native_int + r2 :: bit + r3 :: i64 + r4 :: bit + r5 :: bool + r6 :: i64 + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10, r11 :: i64 +L0: + r0 = VecI64Api.alloc(0, 0) + if is_error(r0) goto L8 (error at f:5) else goto L1 +L1: + v = r0 + r1 = v.len + r2 = i < r1 :: unsigned + if r2 goto L6 else goto L2 :: bool +L2: + r3 = i + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L5 else goto L9 :: bool +L3: + r5 = raise IndexError + if not r5 goto L8 (error at f:6) else goto L4 :: bool +L4: + unreachable +L5: + r6 = r3 + goto L7 +L6: + r6 = i +L7: + r7 = v.items + r8 = r6 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: i64* + dec_ref v + return r10 +L8: + r11 = :: i64 + return r11 +L9: + dec_ref v + goto L3 + +[case testLowerVecNestedGetItem_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(i: i64, j: i64) -> str: + v = vec[vec[str]]([]) + return v[i][j] +[out] +def f(i, j): + i, j :: i64 + r0 :: object + r1 :: ptr + r2 :: vec[vec[str]] + r3 :: ptr + v :: vec[vec[str]] + r4 :: native_int + r5 :: bit + r6 :: i64 + r7 :: bit + r8 :: bool + r9 :: i64 + r10 :: ptr + r11 :: i64 + r12 :: ptr + r13 :: vec[str] + r14 :: native_int + r15 :: bit + r16 :: i64 + r17 :: bit + r18 :: bool + r19 :: i64 + r20 :: ptr + r21 :: i64 + r22 :: ptr + r23, r24 :: str +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecNestedApi.alloc(0, 0, r1, 1) + if is_error(r2) goto L14 (error at f:5) else goto L1 +L1: + r3 = r2.items + v = r2 + r4 = v.len + r5 = i < r4 :: unsigned + if r5 goto L6 else goto L2 :: bool +L2: + r6 = i + r4 + r7 = r6 < r4 :: unsigned + if r7 goto L5 else goto L15 :: bool +L3: + r8 = raise IndexError + if not r8 goto L14 (error at f:6) else goto L4 :: bool +L4: + unreachable +L5: + r9 = r6 + goto L7 +L6: + r9 = i +L7: + r10 = v.items + r11 = r9 * 16 + r12 = r10 + r11 + r13 = borrow load_mem r12 :: vec[str]* + r14 = r13.len + r15 = j < r14 :: unsigned + if r15 goto L12 else goto L8 :: bool +L8: + r16 = j + r14 + r17 = r16 < r14 :: unsigned + if r17 goto L11 else goto L16 :: bool +L9: + r18 = raise IndexError + if not r18 goto L14 (error at f:6) else goto L10 :: bool +L10: + unreachable +L11: + r19 = r16 + goto L13 +L12: + r19 = j +L13: + r20 = r13.items + r21 = r19 * 8 + r22 = r20 + r21 + r23 = load_mem r22 :: builtins.str* + dec_ref v + return r23 +L14: + r24 = :: str + return r24 +L15: + dec_ref v + goto L3 +L16: + dec_ref v + goto L9 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 918c84ee3b0ad..cefa050e8f0e9 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1606,12 +1606,9 @@ def f(v): t :: i64 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, s :: str - r7 :: None - r8 :: native_int + r3, s :: str + r4 :: None + r5 :: native_int L0: t = 0 r0 = 0 @@ -1620,16 +1617,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 8 - r5 = r3 + r4 - r6 = load_mem r5 :: builtins.str* - s = r6 - r7 = g(s) + r3 = vec_get_item_unsafe[str] v, r0 + s = r3 + r4 = g(s) dec_ref s L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return t @@ -1684,11 +1678,7 @@ def C.f(self, x): r3 :: i64 r4 :: bit r5 :: bool - r6 :: i64 - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: i64 + r6, r7 :: i64 L0: r0 = borrow self.v r1 = r0.len @@ -1707,11 +1697,8 @@ L3: L4: r6 = x L5: - r7 = r0.items - r8 = r6 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: i64* - return r10 + r7 = vec_get_item_unsafe[i64] r0, r6 + return r7 [case testVecI64LenBorrowVec_64bit] from librt.vecs import vec @@ -1771,10 +1758,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: str + r6 :: str L0: r0 = v.len r1 = n < r0 :: unsigned @@ -1792,11 +1776,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.str* - return r9 + r6 = vec_get_item_unsafe[str] v, r5 + return r6 [case testVecNestedGetItem_64bit] from librt.vecs import vec @@ -1817,10 +1798,7 @@ def f(v, n): r6 :: bit r7 :: bool r8 :: i64 - r9 :: ptr - r10 :: i64 - r11 :: ptr - r12, vv :: vec[str] + r9, vv :: vec[str] L0: r0 = load_address PyUnicode_Type r1 = r0 @@ -1841,12 +1819,9 @@ L3: L4: r8 = n L5: - r9 = r2.items - r10 = r8 * 16 - r11 = r9 + r10 - r12 = load_mem r11 :: vec[str]* + r9 = vec_get_item_unsafe[vec[str]] r2, r8 dec_ref r2 - vv = r12 + vv = r9 dec_ref vv return 1 L6: @@ -1869,20 +1844,13 @@ def f(v, n, m): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] - r10 :: native_int - r11 :: bit - r12 :: i64 - r13 :: bit - r14 :: bool - r15 :: i64 - r16 :: ptr - r17 :: i64 - r18 :: ptr - r19 :: i64 + r6 :: vec[i64] + r7 :: native_int + r8 :: bit + r9 :: i64 + r10 :: bit + r11 :: bool + r12, r13 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned @@ -1900,31 +1868,25 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = borrow load_mem r8 :: vec[i64]* - r10 = r9.len - r11 = m < r10 :: unsigned - if r11 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe_borrow[vec[i64]] v, r5 + r7 = r6.len + r8 = m < r7 :: unsigned + if r8 goto L9 else goto L6 :: bool L6: - r12 = m + r10 - r13 = r12 < r10 :: unsigned - if r13 goto L8 else goto L7 :: bool + r9 = m + r7 + r10 = r9 < r7 :: unsigned + if r10 goto L8 else goto L7 :: bool L7: - r14 = raise IndexError + r11 = raise IndexError unreachable L8: - r15 = r12 + r12 = r9 goto L10 L9: - r15 = m + r12 = m L10: - r16 = r9.items - r17 = r15 * 8 - r18 = r16 + r17 - r19 = load_mem r18 :: i64* - return r19 + r13 = vec_get_item_unsafe[i64] r6, r12 + return r13 [case testVecPop] from librt.vecs import vec, pop, append diff --git a/mypyc/test/test_expand_rtype.py b/mypyc/test/test_expand_rtype.py new file mode 100644 index 0000000000000..90acc61f625d9 --- /dev/null +++ b/mypyc/test/test_expand_rtype.py @@ -0,0 +1,48 @@ +import unittest + +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + RTypeVar, + RUnion, + RVec, + int_rprimitive, + str_rprimitive, + void_rtype, +) +from mypyc.rt_expandtype import expand_rtype + + +class TestExpandRType(unittest.TestCase): + def test_trivial(self) -> None: + assert expand_rtype(str_rprimitive, []) == str_rprimitive + assert expand_rtype(str_rprimitive, [int_rprimitive]) == str_rprimitive + assert expand_rtype(void_rtype, []) == void_rtype + + def test_instance(self) -> None: + inst = RInstance(ClassIR("A", "__main__")) + assert expand_rtype(inst, [int_rprimitive]) == inst + + def test_simple_expansion(self) -> None: + assert expand_rtype(RTypeVar(0), [str_rprimitive]) == str_rprimitive + + def test_tuple_expansion(self) -> None: + assert expand_rtype( + RTuple([RTypeVar(0), RTypeVar(1)]), [str_rprimitive, int_rprimitive] + ) == RTuple([str_rprimitive, int_rprimitive]) + + def test_union_expansion(self) -> None: + assert expand_rtype( + RUnion([RTypeVar(0), RTypeVar(1)]), [str_rprimitive, int_rprimitive] + ) == RUnion([str_rprimitive, int_rprimitive]) + + def test_vec_expansion(self) -> None: + assert expand_rtype(RVec(RTypeVar(0)), [str_rprimitive]) == RVec(str_rprimitive) + + def test_nested_expansion(self) -> None: + typ = RUnion([RTuple([RVec(RTypeVar(0)), RTypeVar(1)]), RVec(RVec(RTypeVar(0)))]) + expected = RUnion( + [RTuple([RVec(str_rprimitive), int_rprimitive]), RVec(RVec(str_rprimitive))] + ) + assert expand_rtype(typ, [str_rprimitive, int_rprimitive]) == expected diff --git a/mypyc/test/test_lowering.py b/mypyc/test/test_lowering.py index e27b4e77eea8a..3eb4698c68793 100644 --- a/mypyc/test/test_lowering.py +++ b/mypyc/test/test_lowering.py @@ -28,7 +28,7 @@ class TestLowering(MypycDataSuite): - files = ["lowering-int.test", "lowering-list.test"] + files = ["lowering-int.test", "lowering-list.test", "lowering-vec.test"] base_path = test_temp_dir def run_case(self, testcase: DataDrivenTestCase) -> None: From 79a769ac7834aeada0a9af516f8bd8cdacd28542 Mon Sep 17 00:00:00 2001 From: Tom Bannink Date: Tue, 30 Jun 2026 13:06:28 +0200 Subject: [PATCH 08/50] [mypyc] Fix reference leak when setting unboxed refcounted attrs (#21657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I ran into this memory leak which can reproduced with: ```python from dataclasses import dataclass @dataclass class MyClass: v: int c = MyClass(1 << 70) ``` This PR adds a fix and a test that fails without the fix. I am no expert on mypyc internals, so I asked Claude Code to fix this bug. It is a one-line change so I believe it will be easy to review. From quick inspection it seems that the rest of the code has fewer comments than what Claude wrote, so if you prefer, I'll remove the verbose comment that this PR adds. Similarly for the test code, it can be shortened if wanted.
Long explanation from Claude code ### Description A native attribute setter generated by mypyc over-increfs the stored value when the attribute has a **refcounted unboxed type** — most importantly `int` (`CPyTagged`), and also tuples with refcounted items. For an unboxed type, `generate_setter` emitted: ```c tmp = CPyTagged_FromObject(value); // already creates a NEW (owned) reference CPyTagged_INCREF(tmp); // ...and then takes a second one self->_v = tmp; ``` CPyTagged_FromObject already increfs in the heap-boxed case, so the setter takes two references while the deallocator releases only one — leaking one reference on every set through the setter. The other two branches of generate_setter (object and the emit_cast path) are correct because they produce a borrowed value and rely on the single emit_inc_ref to take ownership; only the unboxed branch was inconsistent. Why this shows up with dataclasses This is reached whenever an attribute is set from interpreted code. The clearest real-world case is the __init__ that the stdlib dataclasses module synthesizes for a mypyc-compiled @dataclass: its self.v = v runs as interpreted code and goes through the generated descriptor setter. So every constructed instance of a compiled dataclass with a heap-boxed int field (value ≥ 2**62) leaked one PyLong — silent, unbounded growth in long-lived programs that build many such objects (e.g. 64-bit ids). It does not depend on slots, frozen, eq, field count, or field position; a hand-written native __init__ is unaffected because it stores via SetAttr rather than the descriptor. Small (inline-tagged) ints, floats, and object-typed fields are also unaffected since they aren't refcounted through this path. Reproducer ```python from dataclasses import dataclass import sys @dataclass class C: v: int big = 1 << 70 # heap-boxed int (>= 2**62), so it is refcounted base = sys.getrefcount(big) xs = [C(big) for _ in range(1000)] del xs print(sys.getrefcount(big) - base) # before: 1000 after: 0 ``` Fix Unbox with borrow=True in the setter's unboxed branch, so all three branches of generate_setter produce a borrowed value and the single emit_inc_ref takes exactly one owned reference. borrow=True is a no-op for non-refcounted unboxed types (float, fixed-width ints, bool) and is propagated correctly through RTuple unboxing, which fixes the analogous leak for tuple-typed attributes too. Tests Added testNativeAttrSetterRefcountLeak to mypyc/test-data/run-classes.test, covering a boxed-int dataclass field, an unboxed Tuple[int, int] field, and setter re-assignment. It fails before the fix (expected 100 live refs, got 200) and passes after.
--------- Co-authored-by: Claude Opus 4.8 (1M context) --- mypyc/codegen/emitclass.py | 10 +++++- mypyc/test-data/run-classes.test | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index db94f1de9406e..6054a934b25e1 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -1211,7 +1211,15 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("if (value != NULL) {") if rtype.is_unboxed: - emitter.emit_unbox("value", "tmp", rtype, error=ReturnHandler("-1"), declare_dest=True) + # Borrow the unboxed value: emit_inc_ref below takes the single owned + # reference, matching the borrowed-then-incref pattern of the other two + # branches. Without borrow=True, emit_unbox already creates a new + # reference for refcounted unboxed types (e.g. CPyTagged boxed ints, + # tuples with refcounted fields), so the emit_inc_ref would double the + # reference and leak the stored value on every set via this setter. + emitter.emit_unbox( + "value", "tmp", rtype, error=ReturnHandler("-1"), declare_dest=True, borrow=True + ) elif is_same_type(rtype, object_rprimitive): emitter.emit_line("PyObject *tmp = value;") else: diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 7722cf26ca910..d830b0c04fa6f 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -6006,3 +6006,61 @@ for _ in range(100): check(foo) after = sys.getrefcount(foo.obj) assert after - init == 0, f"Leaked {after - init} refs" + +[case testNativeAttrSetterRefcountLeak] +# Setting a native attribute from interpreted code goes through the generated +# getset descriptor setter. For refcounted unboxed types (heap-boxed ints, +# tuples with refcounted items) the setter must take exactly one reference to +# the stored value, not two. +from dataclasses import dataclass +from typing import Tuple + +@dataclass +class IntField: + v: int + +@dataclass +class TupleField: + v: Tuple[int, int] + +[file driver.py] +import sys +from native import IntField, TupleField + +# A heap-boxed int (>= 2**62) is stored as a refcounted PyObject*, unlike small +# inline-tagged ints, so an over-incref strands a real reference. Compute the +# values at runtime (not as folded literals): on free-threaded builds code +# constants are immortal, and getrefcount could not observe a leak on them. +shift = 70 +BIG = 1 << shift + +def check_no_leak(make, value) -> None: + base = sys.getrefcount(value) + objs = [make(value) for _ in range(100)] + alive = sys.getrefcount(value) + # Each live instance must hold exactly one reference to the field value. + assert alive - base == 100, f"expected 100 live refs, got {alive - base}" + del objs + after = sys.getrefcount(value) + assert after == base, f"leaked {after - base} refs" + +# The dataclass-generated __init__ stores self.v = v via the descriptor setter. +check_no_leak(IntField, BIG) + +# Tuple[int, int] is stored as an unboxed RTuple; its boxed-int elements are +# refcounted, so the setter must not over-incref them either. Use the same int +# in both slots so each instance holds exactly two references to it. +ELEM = 1 << (shift + 1) +base = sys.getrefcount(ELEM) +objs = [TupleField((ELEM, ELEM)) for _ in range(100)] +alive = sys.getrefcount(ELEM) +assert alive - base == 200, f"expected 200 live refs, got {alive - base}" +del objs +assert sys.getrefcount(ELEM) == base, f"tuple field leaked {sys.getrefcount(ELEM) - base} refs" + +# Re-assigning through the setter must release the previous value too. +o = IntField(BIG) +base = sys.getrefcount(BIG) +o.v = BIG +o.v = BIG +assert sys.getrefcount(BIG) == base, "reassignment leaked refs" From 47f0df577f42c726f847163f9c0a15df681a8479 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 30 Jun 2026 21:28:36 +0100 Subject: [PATCH 09/50] Bump librt to 0.12.0 (#21663) This bump includes: * A mypy sync, including recent free-threading fixes. * New experimental Emscripten wheels. --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 04381db913521..9505cf14af452 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -5,5 +5,5 @@ typing_extensions>=4.14.0; python_version>='3.15' mypy_extensions>=1.0.0 pathspec>=1.0.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.11.0; platform_python_implementation != 'PyPy' +librt>=0.12.0; platform_python_implementation != 'PyPy' ast-serialize>=0.5.0,<1.0.0 diff --git a/pyproject.toml b/pyproject.toml index a89bac5c356eb..6fb487a676d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.11.0; platform_python_implementation != 'PyPy'", + "librt>=0.12.0; platform_python_implementation != 'PyPy'", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -58,7 +58,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.11.0; platform_python_implementation != 'PyPy'", + "librt>=0.12.0; platform_python_implementation != 'PyPy'", "ast-serialize>=0.5.0,<1.0.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 254c6b8aaaa28..64092a392b296 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ identify==2.6.19 # via pre-commit iniconfig==2.3.0 # via pytest -librt==0.11.0 ; platform_python_implementation != "PyPy" +librt==0.12.0 ; platform_python_implementation != "PyPy" # via -r mypy-requirements.txt lxml==6.1.0 ; python_version < "3.15" # via -r test-requirements.in From d67a13887480ac9a06523139167a4f772c5cb49f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Jul 2026 09:30:29 +0100 Subject: [PATCH 10/50] Bump ast-serialize to 0.6.0 (#21664) No semantic changes in this PR, just the new experimental Emscripten wheels. --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 9505cf14af452..d2991309e391e 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -6,4 +6,4 @@ mypy_extensions>=1.0.0 pathspec>=1.0.0 tomli>=1.1.0; python_version<'3.11' librt>=0.12.0; platform_python_implementation != 'PyPy' -ast-serialize>=0.5.0,<1.0.0 +ast-serialize>=0.6.0,<1.0.0 diff --git a/pyproject.toml b/pyproject.toml index 6fb487a676d4e..f166a58086821 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ requires = [ "types-psutil", "types-setuptools", # required to work around a mypyc import bug - "ast-serialize>=0.5.0,<1.0.0", + "ast-serialize>=0.6.0,<1.0.0", ] build-backend = "setuptools.build_meta" @@ -59,7 +59,7 @@ dependencies = [ "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", "librt>=0.12.0; platform_python_implementation != 'PyPy'", - "ast-serialize>=0.5.0,<1.0.0", + "ast-serialize>=0.6.0,<1.0.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 64092a392b296..e79319fdbeece 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in # -ast-serialize==0.5.0 +ast-serialize==0.6.0 # via -r mypy-requirements.txt attrs==26.1.0 # via -r test-requirements.in From a6eadee843fd1c549d4c8e086d88997f2c7f525c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 1 Jul 2026 16:59:22 +0100 Subject: [PATCH 11/50] [mypyc] Make instance attribute read-only at runtime if Final (#21666) Previously this was only enforced statically. I plan to add more optimizations that take advantage of Final in the future. --- mypyc/codegen/emitclass.py | 30 +++++++++++++++--------- mypyc/ir/class_ir.py | 4 ++++ mypyc/irbuild/prepare.py | 2 ++ mypyc/test-data/run-classes.test | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 6054a934b25e1..9baaec06a4611 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -1065,12 +1065,14 @@ def generate_getseter_declarations(cl: ClassIR, emitter: Emitter) -> None: getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) ) ) - emitter.emit_line("static int") - emitter.emit_line( - "{}({} *self, PyObject *value, void *closure);".format( - setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + # Final attributes are read-only, so they have no setter. + if attr not in cl.final_attributes: + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure);".format( + setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) ) - ) for prop, (getter, setter) in cl.properties.items(): if getter.decl.implicit: @@ -1099,11 +1101,15 @@ def generate_getseters_table(cl: ClassIR, name: str, emitter: Emitter) -> None: if not cl.is_trait: for attr in cl.attributes: emitter.emit_line(f'{{"{attr}",') - emitter.emit_line( - " (getter){}, (setter){},".format( - getter_name(cl, attr, emitter.names), setter_name(cl, attr, emitter.names) + if attr in cl.final_attributes: + # Final attributes are read-only, so emit a NULL setter. + emitter.emit_line(f" (getter){getter_name(cl, attr, emitter.names)}, NULL,") + else: + emitter.emit_line( + " (getter){}, (setter){},".format( + getter_name(cl, attr, emitter.names), setter_name(cl, attr, emitter.names) + ) ) - ) emitter.emit_line(" NULL, NULL},") for prop, (getter, setter) in cl.properties.items(): if getter.decl.implicit: @@ -1129,8 +1135,10 @@ def generate_getseters(cl: ClassIR, emitter: Emitter) -> None: if not cl.is_trait: for i, (attr, rtype) in enumerate(cl.attributes.items()): generate_getter(cl, attr, rtype, emitter) - emitter.emit_line("") - generate_setter(cl, attr, rtype, emitter) + # Final attributes are read-only, so they have no setter. + if attr not in cl.final_attributes: + emitter.emit_line("") + generate_setter(cl, attr, rtype, emitter) if i < len(cl.attributes) - 1: emitter.emit_line("") for prop, (getter, setter) in cl.properties.items(): diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index f754275480edc..028df5898d920 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -145,6 +145,8 @@ def __init__( ) # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} + # Final attributes defined in the class (not inherited) + self.final_attributes: set[str] = set() # Deletable attributes self.deletable: list[str] = [] # We populate method_types with the signatures of every method before @@ -396,6 +398,7 @@ def serialize(self) -> JsonDict: "ctor": self.ctor.serialize(), # We serialize dicts as lists to ensure order is preserved "attributes": [(k, t.serialize()) for k, t in self.attributes.items()], + "final_attributes": sorted(self.final_attributes), # We try to serialize a name reference, but if the decl isn't in methods # then we can't be sure that will work so we serialize the whole decl. "method_decls": [ @@ -456,6 +459,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.builtin_base = data["builtin_base"] ir.ctor = FuncDecl.deserialize(data["ctor"], ctx) ir.attributes = {k: deserialize_type(t, ctx) for k, t in data["attributes"]} + ir.final_attributes = set(data["final_attributes"]) ir.method_decls = { k: ctx.functions[v].decl if isinstance(v, str) else FuncDecl.deserialize(v, ctx) for k, v in data["method_decls"] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 8b73b10bf8064..6e697823593a6 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -645,6 +645,8 @@ def prepare_methods_and_attributes( add_getter_declaration(ir, name, attr_rtype, module_name) add_setter_declaration(ir, name, attr_rtype, module_name) ir.attributes[name] = attr_rtype + if node.node.is_final: + ir.final_attributes.add(name) elif isinstance(node.node, (FuncDef, Decorator)): prepare_method_def(ir, module_name, cdef, mapper, node.node, options) elif isinstance(node.node, OverloadedFuncDef): diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d830b0c04fa6f..73927bb037a71 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2825,6 +2825,46 @@ def test_final_attribute() -> None: assert C.b['x'] == 'y' assert C.a is C.b +[case testFinalInstanceAttributeCannotBeRebound] +from typing import Any, Final + +from testutil import assertRaises + +class C: + def __init__(self, x: int) -> None: + self.x: Final[int] = x + +class D(C): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y: Final[int] = y + +def rebind_via_any(o: Any, value: int) -> None: + o.x = value + +def test_rebind_via_any() -> None: + c = C(1) + with assertRaises(AttributeError): + rebind_via_any(c, 2) + assert c.x == 1 + +def test_rebind_via_setattr() -> None: + c = C(1) + with assertRaises(AttributeError): + setattr(c, "x", 3) + assert c.x == 1 + +def test_rebind_inherited_via_setattr() -> None: + d = D(1, 2) + # Inherited Final attribute can't be modified. + with assertRaises(AttributeError): + setattr(d, "x", 3) + # The subclass's own Final attribute can't be modified either. + with assertRaises(AttributeError): + setattr(d, "y", 4) + assert d.x == 1 + assert d.y == 2 + [case testClassDerivedFromIntEnum] from enum import IntEnum, auto From 1462b4ec41ea12514b89e919ab15587378eb04c2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 2 Jul 2026 13:05:31 +0100 Subject: [PATCH 12/50] Fix error code of note about unbound type variable (#21668) The note used the default `misc` error code, which was different from the error message. --- mypy/checker.py | 1 + test-data/unit/check-typevar-unbound.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 33705c98e10c3..d13b927b28f2f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1825,6 +1825,7 @@ def check_unbound_return_typevar(self, typ: CallableType) -> None: "Consider using the upper bound " f"{format_type(typ.ret_type.upper_bound, self.options)} instead", context=typ.ret_type, + code=TYPE_VAR, ) def check_default_params(self, item: FuncItem, body_is_trivial: bool | None = None) -> None: diff --git a/test-data/unit/check-typevar-unbound.test b/test-data/unit/check-typevar-unbound.test index 79d326f9fedf3..2dffaee6f2fe7 100644 --- a/test-data/unit/check-typevar-unbound.test +++ b/test-data/unit/check-typevar-unbound.test @@ -13,6 +13,9 @@ def g() -> U: # E: A function returning TypeVar should receive at least one argu # N: Consider using the upper bound "int" instead ... +def g2() -> U: # type: ignore[type-var] + ... + V = TypeVar('V', int, str) def h() -> V: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar From 0c6013d74ec64b1ae4f77fbadfe4e1fb8093809d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 6 Jul 2026 00:25:56 +0100 Subject: [PATCH 13/50] Upload wasm wheels to PyPI (#21671) PyPI supports wasm platform tags since recently, see also https://github.com/mypyc/mypy_mypyc-wheels/pull/116 --- misc/upload-pypi.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/misc/upload-pypi.py b/misc/upload-pypi.py index 8ea86bbea584b..52092bd288f0f 100644 --- a/misc/upload-pypi.py +++ b/misc/upload-pypi.py @@ -31,16 +31,7 @@ def is_whl_or_tar(name: str) -> bool: def item_ok_for_pypi(name: str) -> bool: - if not is_whl_or_tar(name): - return False - - name = name.removesuffix(".tar.gz") - name = name.removesuffix(".whl") - - if name.endswith("wasm32"): - return False - - return True + return is_whl_or_tar(name) def get_release_for_tag(tag: str) -> dict[str, Any]: From c3d9b5cc52534f9908940a810596afe50b4ea3bf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 11:18:34 +0100 Subject: [PATCH 14/50] Fix star import dependencies in mypy daemon (#21673) Changes weren't propagated reliably when exporting names using `from <...> import *`. This could result in false negatives and false positives. I used coding agent assist with small incremental changes. --- mypy/server/deps.py | 7 ++ mypy/server/update.py | 49 +++++++----- test-data/unit/deps.test | 13 +++ test-data/unit/fine-grained.test | 131 +++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 21 deletions(-) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index b2c91d8db4888..29cd8ac8b4635 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -671,6 +671,13 @@ def visit_member_expr(self, e: MemberExpr) -> None: if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) + if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, MypyFile): + # Also depend on the name as accessed through this module. The + # fullname above may point to the original definition (e.g. via + # a re-export using "from ... import *"), but we must also + # recheck if the name is removed from the accessed module's + # namespace. + self.add_dependency(make_trigger(e.expr.node.fullname + "." + e.name)) else: # Reference to a non-module (or missing) attribute if e.expr not in self.type_map: diff --git a/mypy/server/update.py b/mypy/server/update.py index 64ce0fb5d3f8a..cb7ff60e14cba 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -783,31 +783,37 @@ def calculate_active_triggers( else: snapshot2 = snapshot_symbol_table(id, new.names) diff = compare_symbol_table_snapshots(id, snapshot1, snapshot2) - package_nesting_level = id.count(".") - for item in diff.copy(): - if item.count(".") <= package_nesting_level + 1 and item.split(".")[-1] not in ( - "__builtins__", - "__file__", - "__name__", - "__package__", - "__doc__", - ): - # Activate catch-all wildcard trigger for top-level module changes (used for - # "from m import *"). This also gets triggered by changes to module-private - # entries, but as these unneeded dependencies only result in extra processing, - # it's a minor problem. - # - # TODO: Some __* names cause mistriggers. Fix the underlying issue instead of - # special casing them here. - diff.add(id + WILDCARD_TAG) - if item.count(".") > package_nesting_level + 1: - # These are for changes within classes, used by protocols. - diff.add(item.rsplit(".", 1)[0] + WILDCARD_TAG) - + diff |= wildcard_triggers_for_changes(id, diff) names |= diff return {make_trigger(name) for name in names} +def wildcard_triggers_for_changes(module_id: str, diff: set[str]) -> set[str]: + """Return catch-all wildcard triggers activated by a set of changed names.""" + result: set[str] = set() + package_nesting_level = module_id.count(".") + for item in diff: + if item.count(".") <= package_nesting_level + 1 and item.split(".")[-1] not in ( + "__builtins__", + "__file__", + "__name__", + "__package__", + "__doc__", + ): + # Activate catch-all wildcard trigger for top-level module changes (used for + # "from m import *"). This also gets triggered by changes to module-private + # entries, but as these unneeded dependencies only result in extra processing, + # it's a minor problem. + # + # TODO: Some __* names cause mistriggers. Fix the underlying issue instead of + # special casing them here. + result.add(module_id + WILDCARD_TAG) + if item.count(".") > package_nesting_level + 1: + # These are for changes within classes, used by protocols. + result.add(item.rsplit(".", 1)[0] + WILDCARD_TAG) + return result + + def replace_modules_with_new_variants( manager: BuildManager, graph: dict[str, State], @@ -1044,6 +1050,7 @@ def key(node: FineGrainedDeferredNode) -> int: changed = compare_symbol_table_snapshots( file_node.fullname, old_symbols_snapshot, new_symbols_snapshot ) + changed |= wildcard_triggers_for_changes(module_id, changed) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index ab2c5f1c43e74..6850eabf09b6a 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -69,6 +69,19 @@ x = 1 -> m.f -> m, m.f +[case testAccessReexportedModuleAttribute] +import pkg +pkg.f() +[file pkg/__init__.py] +from pkg.sub import * +[file pkg/sub.py] +def f() -> None: pass +[out] + -> m + -> m + -> pkg + -> m + [case testImport] import n [file n.py] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index bc02f49e5d835..ef2e8c0b343bf 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -11839,3 +11839,134 @@ def bar() -> str: return "a" main:2: note: Revealed type is "None | builtins.int" == main:2: note: Revealed type is "None | builtins.str" + +[case testStarExportedNameDeleted1] +import pkg + +pkg.loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "loads" + +[case testStarExportedNameDeleted2] +import pkg + +pkg.C() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +class C: pass + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "C" + +[case testStarExportedNameDeleted3] +from pkg import loads + +loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:1: error: Module "pkg" has no attribute "loads" + +[case testStarExportedNameDeleted4] +from pkg import sub + +a = sub.x + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +x = 1 + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "x" + +[case testStarExportedNameDeleted5] +from pkg import sub + +x: sub.N = 1 + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +N = int + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "sub.N" is not defined + +[case testStarExportedNameDeleted6] +from pkg import * + +loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "loads" is not defined + +[case testStarExportedNameDeleted7] +from pkg import * + +C() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +class C: pass + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "C" is not defined + +[case testStarExportedNameDeleted8] +from a import * + +C() + +[file a.py] +from b import * + +[file b.py] +from c import * + +[file c.py] +class C: pass + +[file c.py.2] +[out] +== +main:3: error: Name "C" is not defined From 50e04b28a207662d5a672cdd9ddc04f3600d2e61 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 14:38:31 +0100 Subject: [PATCH 15/50] [mypyc] Make vec creation from list memory safe on free-threaded builds (#21681) I used coding agent assist (in particular, to write the second variant based on the first). Work on mypyc/mypyc#1202. --- mypyc/lib-rt/vecs/vec_t.c | 30 ++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vec_template.c | 24 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3cfd1756201ce..5c0865ed7772e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -756,6 +756,36 @@ static inline VecT vec_from_sequence( VecT v = vec_alloc(alloc_size, item_type); if (VEC_IS_ERROR(v)) return vec_error(); +#ifdef Py_GIL_DISABLED + if (is_list) { + // Other threads could be mutating the list concurrently, so use strong references + Py_ssize_t actual = 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(seq) && i < n; i++) { + PyObject *item = PyList_GetItemRef(seq, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + VEC_DECREF(v); + return vec_error(); + } + if (!VecT_ItemCheck(v, item, item_type)) { + Py_DECREF(item); + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + VEC_DECREF(v); + return vec_error(); + } + v.items[actual++] = item; + } + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + vec_track_buffer(&v); + v.len = actual; + return v; + } + // Tuples are immutable, so fall through to the shared loop below +#endif for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = is_list ? PyList_GET_ITEM(seq, i) : PyTuple_GET_ITEM(seq, i); if (!VecT_ItemCheck(v, item, item_type)) { diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index b3e80261fec38..a6b09ba2a7b29 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -148,6 +148,30 @@ static inline VEC vec_from_sequence(PyObject *seq, int64_t cap, const int is_lis VEC v = vec_alloc(alloc_size); if (VEC_IS_ERROR(v)) return vec_error(); +#ifdef Py_GIL_DISABLED + if (is_list) { + // Other threads could be mutating the list concurrently, so use strong references + Py_ssize_t actual = 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(seq) && i < n; i++) { + PyObject *item = PyList_GetItemRef(seq, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + VEC_DECREF(v); + return vec_error(); + } + ITEM_C_TYPE x = UNBOX_ITEM(item); + Py_DECREF(item); + if (IS_UNBOX_ERROR(x)) { + VEC_DECREF(v); + return vec_error(); + } + v.items[actual++] = x; + } + v.len = actual; + return v; + } + // Tuples are immutable, so fall through to the shared loop below +#endif for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = is_list ? PyList_GET_ITEM(seq, i) : PyTuple_GET_ITEM(seq, i); ITEM_C_TYPE x = UNBOX_ITEM(item); From 1cc7e8fdc3c7c7086ea46db996fd7d64f13a758d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 14:38:39 +0100 Subject: [PATCH 16/50] [mypyc] Fix memory safety of list.count on free-threaded builds (#21680) Work on mypyc/mypyc#1202. --- mypyc/lib-rt/pythonsupport.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 1b0583543fe48..35f1e78df3915 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -205,6 +205,25 @@ list_count(PyListObject *self, PyObject *value) Py_ssize_t count = 0; Py_ssize_t i; +#ifdef Py_GIL_DISABLED + for (i = 0; i < PyList_GET_SIZE(self); i++) { + PyObject *item = PyList_GetItemRef((PyObject *)self, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + if (PyErr_ExceptionMatches(PyExc_IndexError)) { + PyErr_Clear(); + break; + } + return CPY_INT_TAG; + } + int cmp = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (cmp > 0) + count++; + else if (cmp < 0) + return CPY_INT_TAG; + } +#else for (i = 0; i < Py_SIZE(self); i++) { int cmp = PyObject_RichCompareBool(self->ob_item[i], value, Py_EQ); if (cmp > 0) @@ -212,6 +231,7 @@ list_count(PyListObject *self, PyObject *value) else if (cmp < 0) return CPY_INT_TAG; } +#endif return CPyTagged_ShortFromSsize_t(count); } From ee076569870fc9a281551c7f7f1e4ab8b00e33b7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 14:38:48 +0100 Subject: [PATCH 17/50] [mypyc] Don't borrow list items on free-threaded builds (#21679) Borrowing list items is unsafe on free-threaded builds, since another thread could be concurrently mutating the list. Also make it possible to have different expected test case outputs on free-threaded and GIL-enabled builds. Work on mypyc/mypyc#1202. --- mypyc/irbuild/expression.py | 14 +++-- mypyc/primitives/list_ops.py | 66 +++++++++++------------ mypyc/test-data/exceptions.test | 2 +- mypyc/test-data/irbuild-basic.test | 2 +- mypyc/test-data/irbuild-classes.test | 2 +- mypyc/test-data/irbuild-i64.test | 60 ++++++++++++++++++++- mypyc/test-data/irbuild-lists.test | 20 ++++++- mypyc/test-data/refcount.test | 78 +++++++++++++++++++++++++++- mypyc/test/test_exceptions.py | 10 +++- mypyc/test/test_irbuild.py | 3 ++ mypyc/test/test_refcount.py | 8 ++- 11 files changed, 219 insertions(+), 46 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index cd7295ef709ad..21cb2f8df1b8f 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -58,7 +58,7 @@ TypeType, get_proper_type, ) -from mypyc.common import MAX_SHORT_INT +from mypyc.common import IS_FREE_THREADED, MAX_SHORT_INT from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.ir.ops import ( @@ -773,8 +773,14 @@ def try_optimize_int_floor_divide(builder: IRBuilder, expr: OpExpr) -> OpExpr: def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: index = expr.index base_type = builder.node_type(expr.base) - can_borrow = is_list_rprimitive(base_type) or isinstance(base_type, RVec) - can_borrow_base = can_borrow and is_borrow_friendly_expr(builder, index) + # We can borrow a list item safely only if GIL is enabled. The vec type is optimized for + # performance, so we'll do unsafe borrowing. + can_borrow = (is_list_rprimitive(base_type) and not IS_FREE_THREADED) or isinstance( + base_type, RVec + ) + can_borrow_base = ( + is_list_rprimitive(base_type) or isinstance(base_type, RVec) + ) and is_borrow_friendly_expr(builder, index) # Check for dunder specialization for non-slice indexing if not isinstance(index, SliceExpr): @@ -796,7 +802,7 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: if value: return value - index_reg = builder.accept(expr.index, can_borrow=can_borrow) + index_reg = builder.accept(expr.index, can_borrow=can_borrow or can_borrow_base) return builder.builder.get_item( base, index_reg, builder.node_type(expr), expr.line, can_borrow=builder.can_borrow ) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index efd40469606f3..25c9cd4b4c319 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -2,6 +2,7 @@ from __future__ import annotations +from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, @@ -108,28 +109,6 @@ priority=2, ) -# list[index] that produces a borrowed result -method_op( - name="__getitem__", - arg_types=[list_rprimitive, int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=3, -) - -# list[index] that produces a borrowed result and index is known to be short -method_op( - name="__getitem__", - arg_types=[list_rprimitive, short_int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemShortBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=4, -) - # Version with native int index method_op( name="__getitem__", @@ -140,16 +119,39 @@ priority=5, ) -# Version with native int index -method_op( - name="__getitem__", - arg_types=[list_rprimitive, int64_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemInt64Borrow", - is_borrowed=True, - error_kind=ERR_MAGIC, - priority=6, -) +if not IS_FREE_THREADED: + # list[index] that produces a borrowed result + method_op( + name="__getitem__", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=3, + ) + + # list[index] that produces a borrowed result and index is known to be short + method_op( + name="__getitem__", + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemShortBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=4, + ) + + # Version with native int index + method_op( + name="__getitem__", + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemInt64Borrow", + is_borrowed=True, + error_kind=ERR_MAGIC, + priority=6, + ) # This is unsafe because it assumes that the index is a non-negative integer # that is in-bounds for the list. diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index c271c9ae63d5e..1fea2a1eb9203 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -98,7 +98,7 @@ L6: r5 = :: int return r5 -[case testListSum] +[case testListSum_withgil] from typing import List def sum(a: List[int], l: int) -> int: sum = 0 diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index a9d7ee2357b68..4e015c80a71d3 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -2114,7 +2114,7 @@ L0: r1 = CPyTagged_Multiply(4, r0) return r1 -[case testNativeIndex] +[case testNativeIndex_withgil] from typing import List class A: def __getitem__(self, index: int) -> int: pass diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a7fdc09399009..0310d2f69d444 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1287,7 +1287,7 @@ L0: r2 = r1.x return r2 -[case testBorrowResultOfCustomGetItemInIfStatement] +[case testBorrowResultOfCustomGetItemInIfStatement_withgil] from typing import List class C: diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index 0fcadefd46c27..44157f5a820a0 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -1136,7 +1136,7 @@ L0: keep_alive d, d return r4 -[case testBorrowOverI64ListGetItem1] +[case testBorrowOverI64ListGetItem1_withgil] from mypy_extensions import i64 def f(n: i64) -> str: @@ -1168,7 +1168,38 @@ L0: keep_alive a, n, r3 return r5 -[case testBorrowOverI64ListGetItem2] +[case testBorrowOverI64ListGetItem1_nogil] +from mypy_extensions import i64 + +def f(n: i64) -> str: + a = [C()] + return a[n].s + +class C: + s: str +[out] +def f(n): + n :: i64 + r0 :: __main__.C + r1 :: list + r2 :: ptr + a :: list + r3 :: object + r4 :: __main__.C + r5 :: str +L0: + r0 = C() + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + keep_alive r1 + a = r1 + r3 = CPyList_GetItemInt64(a, n) + r4 = cast(__main__.C, r3) + r5 = r4.s + return r5 + +[case testBorrowOverI64ListGetItem2_withgil] from typing import List from mypy_extensions import i64 @@ -1194,6 +1225,31 @@ L1: L2: return 0 +[case testBorrowOverI64ListGetItem2_nogil] +from typing import List +from mypy_extensions import i64 + +def f(a: List[i64], n: i64) -> bool: + if a[n] == 0: + return True + return False +[out] +def f(a, n): + a :: list + n :: i64 + r0 :: object + r1 :: i64 + r2 :: bit +L0: + r0 = CPyList_GetItemInt64(a, n) + r1 = unbox(i64, r0) + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + [case testCoerceShortIntToI64] from mypy_extensions import i64 from typing import List diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index f6cdbcaf3a96e..6050931497a7b 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -26,7 +26,7 @@ L0: r1 = cast(list, r0) return r1 -[case testListOfListGet2] +[case testListOfListGet2_withgil] from typing import List def f(x: List[List[int]]) -> int: return x[0][1] @@ -45,6 +45,24 @@ L0: keep_alive x, r0 return r3 +[case testListOfListGet2_nogil] +from typing import List +def f(x: List[List[int]]) -> int: + return x[0][1] +[out] +def f(x): + x :: list + r0 :: object + r1 :: list + r2 :: object + r3 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = cast(list, r0) + r2 = CPyList_GetItemShort(r1, 2) + r3 = unbox(int, r2) + return r3 + [case testListSet] from typing import List def f(x: List[int]) -> None: diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index cefa050e8f0e9..7d88cb7d11623 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -592,6 +592,28 @@ L0: dec_ref d return 1 +[case testBorrowListObjectWithLiteralIndex_nogil] +from typing import cast + +def f() -> str: + o = cast(object, []) + return cast(list[str], o)[0] +[out] +def f(): + r0 :: list + o :: object + r1 :: list + r2 :: object + r3 :: str +L0: + r0 = PyList_New(0) + o = r0 + r1 = borrow cast(list, o) + r2 = CPyList_GetItemShort(r1, 0) + r3 = cast(str, r2) + dec_ref o + return r3 + [case testUnaryBranchSpecialCase] def f(x: bool) -> int: if x: @@ -1178,7 +1200,7 @@ L0: r2 = cast(str, r1) return r2 -[case testBorrowListGetItem2] +[case testBorrowListGetItem2_withgil] from typing import List def attr_before_index(x: C) -> str: @@ -1228,6 +1250,58 @@ L0: r2 = r1.n return r2 +[case testBorrowListGetItem2_nogil] +from typing import List + +def attr_before_index(x: C) -> str: + return x.a[x.n] + +def attr_after_index(a: List[C], i: int) -> int: + return a[i].n + +def attr_after_index_literal(a: List[C]) -> int: + return a[0].n + +class C: + a: List[str] + n: int +[out] +def attr_before_index(x): + x :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = borrow x.a + r1 = borrow x.n + r2 = CPyList_GetItem(r0, r1) + r3 = cast(str, r2) + return r3 +def attr_after_index(a, i): + a :: list + i :: int + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItem(a, i) + r1 = cast(__main__.C, r0) + r2 = r1.n + dec_ref r1 + return r2 +def attr_after_index_literal(a): + a :: list + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItemShort(a, 0) + r1 = cast(__main__.C, r0) + r2 = r1.n + dec_ref r1 + return r2 + [case testCannotBorrowListGetItem] from typing import List @@ -1257,7 +1331,7 @@ def f(): L0: return 0 -[case testBorrowListGetItemKeepAlive] +[case testBorrowListGetItemKeepAlive_withgil] from typing import List def f() -> str: diff --git a/mypyc/test/test_exceptions.py b/mypyc/test/test_exceptions.py index e9d4348eb64c5..d842ea1a85eda 100644 --- a/mypyc/test/test_exceptions.py +++ b/mypyc/test/test_exceptions.py @@ -11,7 +11,7 @@ from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase from mypyc.analysis.blockfreq import frequently_executed_blocks -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -34,6 +34,14 @@ class TestExceptionTransform(MypycDataSuite): def run_case(self, testcase: DataDrivenTestCase) -> None: """Perform a runtime checking transformation test case.""" + + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 7e3993e267e74..b11e425e51d3d 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -86,6 +86,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if "_withgil" in testcase.name and IS_FREE_THREADED: # Test case should only run on a non-free-threaded build. return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) diff --git a/mypyc/test/test_refcount.py b/mypyc/test/test_refcount.py index 7efa861e784fb..896ba14e2e68b 100644 --- a/mypyc/test/test_refcount.py +++ b/mypyc/test/test_refcount.py @@ -11,7 +11,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -40,6 +40,12 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) From 0f956541afff9e956ac1658f0f927192b8c13c6b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 15:06:12 +0100 Subject: [PATCH 18/50] [mypyc] Make librt tests resilient to cleaning all .so files (#21682) If `.so` files have been deleted, don't use cached librt build as it won't work. --- mypyc/test/librt_cache.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mypyc/test/librt_cache.py b/mypyc/test/librt_cache.py index f80059ed64786..bf57b9f54196e 100644 --- a/mypyc/test/librt_cache.py +++ b/mypyc/test/librt_cache.py @@ -162,8 +162,15 @@ def get_librt_path(experimental: bool = True, opt_level: str = "0") -> str: os.makedirs(cache_root, exist_ok=True) + binary_suffix = ".pyd" if sys.platform == "win32" else ".so" + with filelock.FileLock(lock_file, timeout=300): # 5 min timeout - if os.path.exists(marker): + # Reuse the cache only if the build completed *and* the compiled + # binaries still exist. A repo-wide clean of .so/.pyd files can delete + # the cached binaries while leaving the marker behind. + if os.path.exists(marker) and any( + f.endswith(binary_suffix) for f in os.listdir(os.path.join(build_dir, "librt")) + ): return build_dir # Clean up any partial build From cd747e557aaf5d96983e32dc69ebf84d6dfc99de Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 17:02:09 +0100 Subject: [PATCH 19/50] [mypyc] Make list get/set item more memory safe on free-threaded builds (#21683) Use the CPython C API for list get item and set item, as direct item access is not memory safe in the presence of race conditions on a free-threaded Python. Note that these operations are not necessarily atomic -- this only fixes memory safety. Explicit synchronization is still generally expected for correctness. This still doesn't cover for loops over lists. I will fix them in a follow-up PR. Preserve existing optimized primitives on non-free-threaded builds. Work on mypyc/mypyc#1202. --- mypyc/lib-rt/CPy.h | 60 ++++++++++++- mypyc/lib-rt/list_ops.c | 149 ++++++++++++++++++++------------- mypyc/lib-rt/mypyc_util.h | 4 + mypyc/test-data/run-lists.test | 62 ++++++++++++++ 4 files changed, 213 insertions(+), 62 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c22c4162669bd..458db90efd530 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -664,16 +664,68 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); // List operations -PyObject *CPyList_Build(Py_ssize_t len, ...); +#ifndef Py_GIL_DISABLED + PyObject *CPyList_GetItem(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); +bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); + +#else + +PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index); +bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value); + +static inline PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { + if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) { + // Inlined fast path + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return PyList_GetItemRef(list, n); + } else { + return CPyList_GetItem_(list, index); + } +} + +static inline PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + if (n < 0) { + n += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, n); +} + +static inline PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { + if (index < 0) { + index += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, index); +} + +static inline bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { + if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) { + // Inlined fast path + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return PyList_SetItem(list, n, value) >= 0; + } else { + return CPyList_SetItem_(list, index, value); + } +} + +static inline bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { + if (index < 0) { + index += PyList_GET_SIZE(list); + } + return PyList_SetItem(list, index, value) >= 0; +} + +#endif + PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); -PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index); -bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value); -bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); +PyObject *CPyList_Build(Py_ssize_t len, ...); PyObject *CPyList_PopLast(PyObject *obj); PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); CPyTagged CPyList_Count(PyObject *obj, PyObject *value); diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 8b7eaa1acf3f0..e8fd061ada687 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -49,6 +49,8 @@ PyObject *CPyList_Copy(PyObject *list) { return PyObject_CallMethodNoArgs(list, mypyc_interned_str.copy); } +#ifndef Py_GIL_DISABLED + PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); Py_ssize_t size = PyList_GET_SIZE(list); @@ -69,24 +71,6 @@ PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { return result; } -PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - Py_ssize_t size = PyList_GET_SIZE(list); - if (n >= 0) { - if (n >= size) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } else { - n += size; - if (n < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } - return PyList_GET_ITEM(list, n); -} - PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); @@ -112,29 +96,6 @@ PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { } } -PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { - if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - Py_ssize_t size = PyList_GET_SIZE(list); - if (n >= 0) { - if (n >= size) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } else { - n += size; - if (n < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } - return PyList_GET_ITEM(list, n); - } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return NULL; - } -} - PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { size_t size = PyList_GET_SIZE(list); if (likely((uint64_t)index < size)) { @@ -156,23 +117,6 @@ PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { return result; } -PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { - size_t size = PyList_GET_SIZE(list); - if (likely((uint64_t)index < size)) { - return PyList_GET_ITEM(list, index); - } - if (index >= 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - index += size; - if (index < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - return PyList_GET_ITEM(list, index); -} - bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); @@ -220,6 +164,95 @@ bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { return true; } +#else /* Py_GIL_DISABLED */ + +PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + if (n < 0) { + n += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n < 0) { + n += size; + } + return PyList_SetItem(list, n, value) >= 0; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return false; + } +} + +#endif + +PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); +} + +PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + return PyList_GET_ITEM(list, index); + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GET_ITEM(list, index); +} + // This function should only be used to fill in brand new lists. void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value) { PyList_SET_ITEM(list, index, value); diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 4a4552d059b8c..d7a3eb3214bed 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -156,6 +156,10 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { return x << 1; } +static inline int CPyTagged_IsNegative(CPyTagged x) { + return ((Py_ssize_t)x) < 0; +} + // Are we targeting Python 3.X or newer? #define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000) #define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 40ca1b6e005f9..d06fc9ce10b0b 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -171,6 +171,68 @@ def test_list_build() -> None: l3.append('a') assert l3 == ['a'] +class C: + def __init__(self, s: str) -> None: + self.s = s + +a = [C("a"), C("b"), C("c")] + +def test_get_item() -> None: + zero = int() + assert a[zero].s == "a" + assert a[zero + 1].s == "b" + assert a[zero + 2].s == "c" + assert a[zero - 1].s == "c" + assert a[zero - 2].s == "b" + assert a[zero - 3].s == "a" + with assertRaises(IndexError): + a[zero + 3] + with assertRaises(IndexError): + a[zero - 4] + # TODO: Raise IndexError? + with assertRaises(OverflowError): + a[zero + 2**90] + with assertRaises(OverflowError): + a[zero - 2**90] + +def test_get_item_literal() -> None: + assert a[0].s == "a" + assert a[1].s == "b" + assert a[2].s == "c" + assert a[-1].s == "c" + assert a[-2].s == "b" + assert a[-3].s == "a" + with assertRaises(IndexError): + a[3] + with assertRaises(IndexError): + a[-4] + # TODO: Raise IndexError? + with assertRaises(OverflowError): + a[2**90] + with assertRaises(OverflowError): + a[-2**90] + +def test_set_item() -> None: + l = a.copy() + zero = int() + zero2 = int() + l[zero] = C("x") + assert l[zero2].s == "x" + l[zero + 2] = C("y") + assert l[zero2 + 2].s == "y" + l[zero - 1] = C("t") + assert l[zero2 - 1].s == "t" + l[zero - 3] = C("u") + assert l[zero2 - 3].s == "u" + with assertRaises(IndexError): + a[zero + 3] = C("z") + with assertRaises(IndexError): + a[zero - 4] = C("z") + with assertRaises(OverflowError): + a[zero + 2**90] = C("z") + with assertRaises(OverflowError): + a[zero - 2**90] = C("z") + def test_append() -> None: l = [1, 2] l.append(10) From 2f21f985ea7444083486febbcad1102dde8294d2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 6 Jul 2026 17:48:50 +0100 Subject: [PATCH 20/50] [mypyc] Make multiple assignment from list memory-safe with free threading (#21684) Work on mypyc/mypyc#1202. --- mypyc/irbuild/builder.py | 13 +++++++-- mypyc/primitives/list_ops.py | 2 +- mypyc/test-data/irbuild-statements.test | 39 ++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 587be873a46d9..39e2c6aa5181f 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -62,6 +62,7 @@ BITMAP_BITS, EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, + IS_FREE_THREADED, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -146,7 +147,12 @@ from mypyc.options import CompilerOptions from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op from mypyc.primitives.generic_ops import iter_op, next_op, py_setattr_op -from mypyc.primitives.list_ops import list_get_item_unsafe_op, list_pop_last, to_list +from mypyc.primitives.list_ops import ( + list_get_item_int64_op, + list_get_item_unsafe_op, + list_pop_last, + to_list, +) from mypyc.primitives.misc_ops import ( check_unpack_count_op, get_module_dict_op, @@ -893,7 +899,10 @@ def process_sequence_assignment( index: Value if is_list_rprimitive(rvalue.type): index = Integer(i, c_pyssize_t_rprimitive) - item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) + if not IS_FREE_THREADED: + item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) + else: + item_value = self.primitive_op(list_get_item_int64_op, [rvalue, index], line) elif is_tuple_rprimitive(rvalue.type): index = Integer(i, c_pyssize_t_rprimitive) item_value = self.call_c(tuple_get_item_unsafe_op, [rvalue, index], line) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index 25c9cd4b4c319..8af7e9e71df7e 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -110,7 +110,7 @@ ) # Version with native int index -method_op( +list_get_item_int64_op = method_op( name="__getitem__", arg_types=[list_rprimitive, int64_rprimitive], return_type=object_rprimitive, diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index df7eb5fc7449a..8199cfdaa699e 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test @@ -575,7 +575,7 @@ L0: z = r6 return 1 -[case testMultipleAssignmentUnpackFromSequence] +[case testMultipleAssignmentUnpackFromSequence_withgil] from typing import List, Tuple def f(l: List[int], t: Tuple[int, ...]) -> None: @@ -612,6 +612,43 @@ L0: y = r9 return 1 +[case testMultipleAssignmentUnpackFromSequence_nogil] +from typing import List, Tuple + +def f(l: List[int], t: Tuple[int, ...]) -> None: + x: object + y: int + x, y = l + x, y = t +[out] +def f(l, t): + l :: list + t :: tuple + r0 :: i32 + r1 :: bit + r2, r3, x :: object + r4, y :: int + r5 :: i32 + r6 :: bit + r7, r8 :: object + r9 :: int +L0: + r0 = CPySequence_CheckUnpackCount(l, 2) + r1 = r0 >= 0 :: signed + r2 = CPyList_GetItemInt64(l, 0) + r3 = CPyList_GetItemInt64(l, 1) + x = r2 + r4 = unbox(int, r3) + y = r4 + r5 = CPySequence_CheckUnpackCount(t, 2) + r6 = r5 >= 0 :: signed + r7 = CPySequenceTuple_GetItemUnsafe(t, 0) + r8 = CPySequenceTuple_GetItemUnsafe(t, 1) + x = r7 + r9 = unbox(int, r8) + y = r9 + return 1 + [case testAssert] from typing import Optional From d308a8dd60d92dbc0ac6e8443d0ef1a9807a21cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Mon, 6 Jul 2026 22:23:17 +0200 Subject: [PATCH 21/50] Comment spelling cleanup (#21676) Correct 3 misspellings. I've used https://github.com/crate-ci/typos to discover them, and my robobuddy to speed up work. --- CHANGELOG.md | 2 +- mypy/inspections.py | 2 +- mypy/plugins/dataclasses.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac2cb8036dc71..2a2c9191b71a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -449,7 +449,7 @@ Contributed by Marc Mueller (PR [20156](https://github.com/python/mypy/pull/2015 For best performance, mypy can be compiled to C extension modules using mypyc. This makes mypy 3-5x faster than when interpreted with pure Python. We now build and upload mypyc accelerated mypy wheels for `win_arm64` and `cp314t-...` to PyPI, making it easy for Windows -users on ARM and those using the free theading builds for Python 3.14 to realise this speedup +users on ARM and those using the free threading builds for Python 3.14 to realise this speedup -- just `pip install` the latest mypy. Contributed by Marc Mueller diff --git a/mypy/inspections.py b/mypy/inspections.py index 1a869438101fe..6e2018262944c 100644 --- a/mypy/inspections.py +++ b/mypy/inspections.py @@ -297,7 +297,7 @@ def cmp_types(x: TypeInfo, y: TypeInfo) -> int: result = {} for base in sorted_bases: if not combined_attrs[base]: - # Skip bases where everytihng was filtered out. + # Skip bases where everything was filtered out. continue result[base] = combined_attrs[base] return result diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 5b39b11a623cc..790fe618c95c7 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -914,7 +914,7 @@ def _infer_dataclass_attr_init_type( # Perform a simple-minded inference from the signature of __set__, if present. # We can't use mypy.checkmember here, since this plugin runs before type checking. - # We only support some basic scanerios here, which is hopefully sufficient for + # We only support some basic scenarios here, which is hopefully sufficient for # the vast majority of use cases. if not isinstance(t, Instance): return default From a805f692014921d840e70ad8e511e3a6a10c28cb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:48:48 -0700 Subject: [PATCH 22/50] Sync typeshed (#21665) Source commit: https://github.com/python/typeshed/commit/b9090e99745ac1511d8efd828622b11a9a3623e8 --- mypy/typeshed/stdlib/VERSIONS | 2 - mypy/typeshed/stdlib/_curses.pyi | 32 +++--- mypy/typeshed/stdlib/ast.pyi | 21 +++- mypy/typeshed/stdlib/builtins.pyi | 14 +-- mypy/typeshed/stdlib/fnmatch.pyi | 7 +- mypy/typeshed/stdlib/html/__init__.pyi | 3 + mypy/typeshed/stdlib/inspect.pyi | 10 +- .../profiling/sampling/gecko_collector.pyi | 102 +++++++++++++++++- mypy/typeshed/stdlib/types.pyi | 6 +- mypy/typeshed/stdlib/typing_extensions.pyi | 84 ++++++++++++--- 10 files changed, 229 insertions(+), 52 deletions(-) diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index e9c8d91fdbd71..96eb98131db63 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -245,8 +245,6 @@ posixpath: 3.0- pprint: 3.0- profile: 3.0- profiling: 3.15- -profiling.sampling: 3.15- -profiling.tracing: 3.15- pstats: 3.0- pty: 3.0- pwd: 3.0- diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index 449cf75dad422..fcd0da4c465c7 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -423,7 +423,7 @@ class window: # undocumented def chgat(self, y: int, x: int, num: int, attr: int) -> None: ... def clear(self) -> None: ... - def clearok(self, yes: int) -> None: ... + def clearok(self, flag: bool, /) -> None: ... def clrtobot(self) -> None: ... def clrtoeol(self) -> None: ... def cursyncup(self) -> None: ... @@ -480,9 +480,9 @@ class window: # undocumented @overload def hline(self, y: int, x: int, ch: _ChType, n: int) -> None: ... - def idcok(self, flag: bool) -> None: ... - def idlok(self, yes: bool) -> None: ... - def immedok(self, flag: bool) -> None: ... + def idcok(self, flag: bool, /) -> None: ... + def idlok(self, flag: bool, /) -> None: ... + def immedok(self, flag: bool, /) -> None: ... @overload def inch(self) -> int: ... @@ -494,7 +494,7 @@ class window: # undocumented @overload def insch(self, y: int, x: int, ch: _ChType, attr: int = ...) -> None: ... - def insdelln(self, nlines: int) -> None: ... + def insdelln(self, nlines: int, /) -> None: ... def insertln(self) -> None: ... @overload @@ -514,13 +514,13 @@ class window: # undocumented def is_linetouched(self, line: int, /) -> bool: ... def is_wintouched(self) -> bool: ... - def keypad(self, yes: bool, /) -> None: ... - def leaveok(self, yes: bool) -> None: ... - def move(self, new_y: int, new_x: int) -> None: ... - def mvderwin(self, y: int, x: int) -> None: ... - def mvwin(self, new_y: int, new_x: int) -> None: ... - def nodelay(self, yes: bool) -> None: ... - def notimeout(self, yes: bool) -> None: ... + def keypad(self, flag: bool, /) -> None: ... + def leaveok(self, flag: bool, /) -> None: ... + def move(self, new_y: int, new_x: int, /) -> None: ... + def mvderwin(self, y: int, x: int, /) -> None: ... + def mvwin(self, new_y: int, new_x: int, /) -> None: ... + def nodelay(self, flag: bool, /) -> None: ... + def notimeout(self, flag: bool, /) -> None: ... @overload def noutrefresh(self) -> None: ... @@ -550,9 +550,9 @@ class window: # undocumented @overload def refresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... - def resize(self, nlines: int, ncols: int) -> None: ... + def resize(self, nlines: int, ncols: int, /) -> None: ... def scroll(self, lines: int = 1) -> None: ... - def scrollok(self, flag: bool) -> None: ... + def scrollok(self, flag: bool, /) -> None: ... def setscrreg(self, top: int, bottom: int, /) -> None: ... def standend(self) -> None: ... def standout(self) -> None: ... @@ -568,9 +568,9 @@ class window: # undocumented def subwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... def syncdown(self) -> None: ... - def syncok(self, flag: bool) -> None: ... + def syncok(self, flag: bool, /) -> None: ... def syncup(self) -> None: ... - def timeout(self, delay: int) -> None: ... + def timeout(self, delay: int, /) -> None: ... def touchline(self, start: int, count: int, changed: bool = True) -> None: ... def touchwin(self) -> None: ... def untouchwin(self) -> None: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 14a98b9a5fcaf..a1993773ee25f 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -956,14 +956,27 @@ class DictComp(expr): else: value: expr generators: list[comprehension] - if sys.version_info >= (3, 13): + if sys.version_info >= (3, 15): + def __init__( + self, key: expr, value: expr | None = None, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + elif sys.version_info >= (3, 13): def __init__( self, key: expr, value: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] ) -> None: ... else: def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - if sys.version_info >= (3, 14): + if sys.version_info >= (3, 15): + def __replace__( + self, + *, + key: expr = ..., + value: expr | None = ..., + generators: list[comprehension] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + elif sys.version_info >= (3, 14): def __replace__( self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... @@ -2147,6 +2160,10 @@ class NodeVisitor: def visit_TypeVarTuple(self, node: TypeVarTuple) -> Any: ... def visit_TypeAlias(self, node: TypeAlias) -> Any: ... + if sys.version_info >= (3, 14): + def visit_TemplateStr(self, node: TemplateStr) -> Any: ... + def visit_Interpolation(self, node: Interpolation) -> Any: ... + # visit methods for deprecated nodes def visit_ExtSlice(self, node: ExtSlice) -> Any: ... def visit_Index(self, node: Index) -> Any: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index d773f98e90b6b..7b659e5476d59 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -33,7 +33,7 @@ from _typeshed import ( from collections.abc import Awaitable, Callable, Iterable, Iterator, MutableSet, Reversible, Set as AbstractSet, Sized from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper from os import PathLike -from types import CellType, CodeType, EllipsisType, GenericAlias, NotImplementedType, TracebackType +from types import CellType, CodeType, EllipsisType, GenericAlias, NotImplementedType, TracebackType, UnionType # mypy crashes if any of {ByteString, Sequence, MutableSequence, Mapping, MutableMapping} # are imported from collections.abc in builtins.pyi @@ -2080,11 +2080,13 @@ if sys.version_info >= (3, 15): class sentinel: __name__: str __module__: str - def __new__(cls, name: str, /, *, repr: str | None = None) -> Self: ... - def __copy__(self, /) -> Self: ... - def __deepcopy__(self, memo: Any, /) -> Self: ... - def __or__(self, other: Any, /) -> Any: ... - def __ror__(self, other: Any, /) -> Any: ... + def __new__(cls, name: str, /, *, repr: str | None = None) -> sentinel: ... + def __copy__(self, /) -> sentinel: ... + def __deepcopy__(self, memo: Any, /) -> sentinel: ... + # `other` can be any legal form for unions. + # `x | x` creates a `sentinel` instance if `x` is a sentinel, not a `UnionType` instance. + def __or__(self, other: Any, /) -> UnionType | sentinel: ... + def __ror__(self, other: Any, /) -> UnionType | sentinel: ... @overload def sorted( diff --git a/mypy/typeshed/stdlib/fnmatch.pyi b/mypy/typeshed/stdlib/fnmatch.pyi index 345c4576497de..018139dc482bf 100644 --- a/mypy/typeshed/stdlib/fnmatch.pyi +++ b/mypy/typeshed/stdlib/fnmatch.pyi @@ -1,15 +1,16 @@ import sys from collections.abc import Iterable +from os import PathLike from typing import AnyStr __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] if sys.version_info >= (3, 14): __all__ += ["filterfalse"] -def fnmatch(name: AnyStr, pat: AnyStr) -> bool: ... +def fnmatch(name: AnyStr | PathLike[AnyStr], pat: AnyStr | PathLike[AnyStr]) -> bool: ... def fnmatchcase(name: AnyStr, pat: AnyStr) -> bool: ... -def filter(names: Iterable[AnyStr], pat: AnyStr) -> list[AnyStr]: ... +def filter(names: Iterable[AnyStr | PathLike[AnyStr]], pat: AnyStr | PathLike[AnyStr]) -> list[AnyStr]: ... def translate(pat: str) -> str: ... if sys.version_info >= (3, 14): - def filterfalse(names: Iterable[AnyStr], pat: AnyStr) -> list[AnyStr]: ... + def filterfalse(names: Iterable[AnyStr | PathLike[AnyStr]], pat: AnyStr | PathLike[AnyStr]) -> list[AnyStr]: ... diff --git a/mypy/typeshed/stdlib/html/__init__.pyi b/mypy/typeshed/stdlib/html/__init__.pyi index 8ad72f1265882..71e971d150449 100644 --- a/mypy/typeshed/stdlib/html/__init__.pyi +++ b/mypy/typeshed/stdlib/html/__init__.pyi @@ -1,4 +1,7 @@ +import re + __all__ = ["escape", "unescape"] def escape(s: str, quote: bool = True) -> str: ... def unescape(s: str) -> str: ... +def _replace_charref(s: re.Match[str]) -> str: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index 003ecc10f072e..1bd2b9dee325b 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -38,7 +38,7 @@ from typing import ( overload, type_check_only, ) -from typing_extensions import Self, TypeIs, deprecated, disjoint_base +from typing_extensions import Never, Self, TypeIs, deprecated, disjoint_base if sys.version_info >= (3, 14): from annotationlib import Format @@ -217,7 +217,7 @@ if sys.version_info >= (3, 11): def getmodulename(path: StrPath) -> str | None: ... def ismodule(object: object) -> TypeIs[ModuleType]: ... -def isclass(object: object) -> TypeIs[type[Any]]: ... +def isclass(object: object) -> TypeIs[type[object]]: ... def ismethod(object: object) -> TypeIs[MethodType]: ... if sys.version_info >= (3, 14): @@ -245,7 +245,7 @@ def iscoroutinefunction(obj: Callable[_P, object]) -> TypeGuard[Callable[_P, Cor @overload def iscoroutinefunction(obj: object) -> TypeGuard[Callable[..., CoroutineType[Any, Any, Any]]]: ... -def isgenerator(object: object) -> TypeIs[GeneratorType[Any, Any, Any]]: ... +def isgenerator(object: object) -> TypeIs[GeneratorType[object, Never, object]]: ... def iscoroutine(object: object) -> TypeIs[CoroutineType[Any, Any, Any]]: ... def isawaitable(object: object) -> TypeIs[Awaitable[Any]]: ... @@ -264,7 +264,7 @@ class _SupportsSet(Protocol[_T_contra, _V_contra]): class _SupportsDelete(Protocol[_T_contra]): def __delete__(self, instance: _T_contra, /) -> None: ... -def isasyncgen(object: object) -> TypeIs[AsyncGeneratorType[Any, Any]]: ... +def isasyncgen(object: object) -> TypeIs[AsyncGeneratorType[object, Never]]: ... def istraceback(object: object) -> TypeIs[TracebackType]: ... def isframe(object: object) -> TypeIs[FrameType]: ... def iscode(object: object) -> TypeIs[CodeType]: ... @@ -289,7 +289,7 @@ def ismethoddescriptor(object: object) -> TypeIs[MethodDescriptorType]: ... def ismemberdescriptor(object: object) -> TypeIs[MemberDescriptorType]: ... def isabstract(object: object) -> bool: ... def isgetsetdescriptor(object: object) -> TypeIs[GetSetDescriptorType]: ... -def isdatadescriptor(object: object) -> TypeIs[_SupportsSet[Any, Any] | _SupportsDelete[Any]]: ... +def isdatadescriptor(object: object) -> TypeIs[_SupportsSet[Never, Never] | _SupportsDelete[Never]]: ... # # Retrieving source code diff --git a/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi b/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi index 6072d4f2359af..666fd42c3ea5f 100644 --- a/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi +++ b/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi @@ -1,11 +1,109 @@ -from _typeshed import StrOrBytesPath -from collections.abc import Sequence +from _typeshed import Incomplete, StrOrBytesPath, StrPath +from collections.abc import Generator, Sequence +from tempfile import TemporaryDirectory +from typing import Any, ClassVar, Final, TypedDict, type_check_only from _remote_debugging import AwaitedInfo, InterpreterInfo from .collector import Collector, _Timestamps +@type_check_only +class _GeckoCategory(TypedDict): + name: str + color: str + subcategories: list[str] + +THREAD_STATUS_HAS_GIL: Final[int] +THREAD_STATUS_ON_CPU: Final[int] +THREAD_STATUS_UNKNOWN: Final[int] +THREAD_STATUS_GIL_REQUESTED: Final[int] +THREAD_STATUS_HAS_EXCEPTION: Final[int] +THREAD_STATUS_MAIN_THREAD: Final[int] + +GECKO_CATEGORIES: Final[list[_GeckoCategory]] + +CATEGORY_OTHER: Final = 0 +CATEGORY_PYTHON: Final = 1 +CATEGORY_NATIVE: Final = 2 +CATEGORY_GC: Final = 3 +CATEGORY_GIL: Final = 4 +CATEGORY_CPU: Final = 5 +CATEGORY_CODE_TYPE: Final = 6 +CATEGORY_OPCODES: Final = 7 +CATEGORY_EXCEPTION: Final = 8 + +DEFAULT_SUBCATEGORY: Final = 0 + +GECKO_FORMAT_VERSION: Final = 32 +GECKO_PREPROCESSED_VERSION: Final = 57 + +RESOURCE_TYPE_LIBRARY: Final = 1 + +FRAME_ADDRESS_NONE: Final = -1 +FRAME_INLINE_DEPTH_ROOT: Final = 0 + +PROCESS_TYPE_MAIN: Final = 0 +STACKWALK_DISABLED: Final = 0 + +DEFAULT_SPILL_BUFFER_BYTES: Final[int] + +class SpillColumn: + path: str + buffer: bytearray + + def __init__(self, directory: StrPath, basename: StrPath, *, buffer_bytes: int | None = None) -> None: ... + # "value" accepts the same types as json.JSONEncoder.encode() + def append(self, value: Any) -> None: ... + def flush(self) -> None: ... + def iter_tokens(self) -> Generator[str]: ... + +class GeckoThreadSpill: + sample_count: int + marker_count: int + def __init__(self, directory: StrPath, tid: int) -> None: ... + def append_sample(self, stack_index: int, time_ms: float) -> None: ... + def append_marker( + self, name_idx: int, start_time: float, end_time: float, phase: int, category: int, data: dict[str, Any] + ) -> None: ... + def prepare_read(self) -> None: ... + class GeckoCollector(Collector): + aggregating: ClassVar[bool] + + sample_interval_usec: int + skip_idle: bool + opcodes_enabled: bool + start_time: float + + global_strings: list[str] + global_string_map: dict[str, int] + + threads: dict[int, dict[str, Any]] + spill_dir: TemporaryDirectory[str] | None + exported: bool + + libs: list[Incomplete] + + sample_count: int + last_sample_time: float + interval: float + + has_gil_start: dict[Incomplete, Incomplete] + no_gil_start: dict[Incomplete, Incomplete] + on_cpu_start: dict[Incomplete, Incomplete] + off_cpu_start: dict[Incomplete, Incomplete] + python_code_start: dict[Incomplete, Incomplete] + native_code_start: dict[Incomplete, Incomplete] + gil_wait_start: dict[Incomplete, Incomplete] + exception_start: dict[Incomplete, Incomplete] + no_exception_start: dict[Incomplete, Incomplete] + + gc_start_per_thread: dict[int, float] + + initialized_threads: set[Incomplete] + + opcode_state: dict[int, tuple[Incomplete, int, int, str, str, float]] + def __init__(self, sample_interval_usec: int, *, skip_idle: bool = False, opcodes: bool = False) -> None: ... def collect( self, stack_frames: Sequence[InterpreterInfo] | Sequence[AwaitedInfo], timestamps_us: _Timestamps = None diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index b9771ffc72dad..68b6b3fbe41d7 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -703,8 +703,10 @@ class GenericAlias: @property def __typing_unpacked_tuple_args__(self) -> tuple[Any, ...] | None: ... - def __or__(self, value: Any, /) -> UnionType: ... - def __ror__(self, value: Any, /) -> UnionType: ... + # `other` can be any legal form for unions. + # `list[int] | list[int]` creates a `GenericAlias` instance, not a `UnionType` instance + def __or__(self, value: Any, /) -> UnionType | GenericAlias: ... + def __ror__(self, value: Any, /) -> UnionType | GenericAlias: ... # GenericAlias delegates attr access to `__origin__` def __getattr__(self, name: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index fdbba495c579f..80341175e6e14 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -59,7 +59,6 @@ from typing import ( # noqa: Y022,Y037,Y038,Y039,UP035 Tuple as Tuple, Type as Type, TypeAlias as TypeAlias, - TypedDict as TypedDict, TypeGuard as TypeGuard, TypeVar as _TypeVar, Union as Union, @@ -72,6 +71,9 @@ from typing import ( # noqa: Y022,Y037,Y038,Y039,UP035 type_check_only, ) +if sys.version_info >= (3, 14): + from _typeshed import EvaluateFunc + # Please keep order the same as at runtime. __all__ = [ # Super-special typing primitives. @@ -143,6 +145,7 @@ __all__ = [ "override", "Protocol", "Sentinel", + "sentinel", "reveal_type", "runtime", "runtime_checkable", @@ -232,6 +235,11 @@ Literal: _SpecialForm def IntVar(name: str) -> Any: ... # returns a new TypeVar +# Kept as a distinct symbol to `typing.TypedDict` so that type checkers can more easily +# distinguish between the two on Python 3.14, on which `typing_extensions.TypedDict` +# exposes `__closed__` and `__extra_items__` but `typing.TypedDict` does not +TypedDict: _SpecialForm + # Internal mypy fallback type for all typed dicts (does not exist at runtime) # N.B. Keep this mostly in sync with typing._TypedDict/mypy_extensions._TypedDict @type_check_only @@ -456,7 +464,6 @@ if sys.version_info >= (3, 13): ReadOnly as ReadOnly, TypeIs as TypeIs, TypeVar as TypeVar, - TypeVarTuple as TypeVarTuple, get_protocol_members as get_protocol_members, is_protocol as is_protocol, ) @@ -545,19 +552,58 @@ else: def has_default(self) -> bool: ... def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + ReadOnly: _SpecialForm + TypeIs: _SpecialForm + +if sys.version_info >= (3, 15): + from typing import TypeVarTuple as TypeVarTuple +else: @final class TypeVarTuple: @property def __name__(self) -> str: ... @property + def __bound__(self) -> AnnotationForm | None: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + @property + def __infer_variance__(self) -> bool: ... + @property def __default__(self) -> AnnotationForm: ... - def __init__(self, name: str, *, default: AnnotationForm = ...) -> None: ... + if sys.version_info >= (3, 11): + def __new__( + cls, + name: str, + *, + bound: AnnotationForm | None = None, + covariant: bool = False, + contravariant: bool = False, + infer_variance: bool = False, + default: AnnotationForm = ..., + ) -> Self: ... + else: + def __init__( + self, + name: str, + *, + bound: AnnotationForm | None = None, + covariant: bool = False, + contravariant: bool = False, + infer_variance: bool = False, + default: AnnotationForm = ..., + ) -> None: ... + def __iter__(self) -> Any: ... # Unpack[Self] def has_default(self) -> bool: ... - def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Never, /) -> Never: ... - ReadOnly: _SpecialForm - TypeIs: _SpecialForm + def __typing_prepare_subst__(self, alias: Any, args: Any, /) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 14): + @property + def evaluate_default(self) -> EvaluateFunc | None: ... # TypeAliasType was added in Python 3.12, but had significant changes in 3.14. if sys.version_info >= (3, 14): @@ -680,11 +726,21 @@ else: def type_repr(value: object) -> str: ... # PEP 661 -class Sentinel: - def __init__(self, name: str, repr: str | None = None) -> None: ... - if sys.version_info >= (3, 14): - def __or__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions - def __ror__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions - else: - def __or__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions - def __ror__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions +if sys.version_info >= (3, 15): + from builtins import sentinel as sentinel +else: + class sentinel: + def __init__(self, name: str, /, *, repr: str | None = None) -> None: ... + __name__: str + __module__: str + if sys.version_info >= (3, 14): + # `other`` can be any type form legal for unions. + # `x | x` creates a `sentinel` instance if `x` is a sentinel, not a `UnionType` instance + def __or__(self, other: Any) -> UnionType | sentinel: ... + def __ror__(self, other: Any) -> UnionType | sentinel: ... + else: + # other can be any type form legal for unions + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +Sentinel = sentinel From 523acf6d19bcf45bb4189cc3542c9a7f8fa6f3d3 Mon Sep 17 00:00:00 2001 From: esarp <11684270+esarp@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:36:30 -0500 Subject: [PATCH 23/50] Update version in prep for new release (#21685) --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index a33ca938708f4..13a4418314c25 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "2.2.0+dev" +__version__ = "2.3.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From c07d7888e7b61983924632036a864c4d210848fe Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 Jul 2026 10:37:11 +0100 Subject: [PATCH 24/50] [mypyc] Make for loop over list memory-safe on free-threaded builds (#21686) Don't access list items directly but use CPython C API functions for item access. Also, reading the list length should use an atomic relaxed memory order read, but changing this is lower priority and won't be fixed until later. There is no change on non-free-threaded Python builds. Work on mypyc/mypyc#1202. --- mypyc/irbuild/for_helpers.py | 13 ++++++-- mypyc/test-data/irbuild-basic.test | 6 ++-- mypyc/test-data/irbuild-lists.test | 8 ++--- mypyc/test-data/irbuild-set.test | 4 +-- mypyc/test-data/irbuild-statements.test | 43 +++++++++++++++++++++++-- mypyc/test-data/irbuild-tuple.test | 4 +-- mypyc/test-data/irbuild-vec-i64.test | 4 +-- mypyc/test-data/lowering-int.test | 2 +- mypyc/test/test_lowering.py | 8 ++++- 9 files changed, 72 insertions(+), 20 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 95894dcedaba8..fc539a1fc090b 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -28,6 +28,7 @@ Var, ) from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types +from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -81,7 +82,12 @@ ) from mypyc.primitives.exc_ops import no_err_occurred_op, propagate_if_error_op from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op -from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op +from mypyc.primitives.list_ops import ( + list_append_op, + list_get_item_int64_op, + list_get_item_unsafe_op, + new_list_set_item_op, +) from mypyc.primitives.misc_ops import stop_async_iteration_op from mypyc.primitives.registry import CFunctionDescription from mypyc.primitives.set_ops import set_add_op @@ -866,7 +872,10 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> # since we want to use __getitem__ if we don't have an unsafe version, # so we just check manually. if is_list_rprimitive(target.type): - return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) + if not IS_FREE_THREADED: + return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) + else: + return builder.primitive_op(list_get_item_int64_op, [target, index], line) elif is_tuple_rprimitive(target.type): return builder.call_c(tuple_get_item_unsafe_op, [target, index], line) elif is_str_rprimitive(target.type): diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 4e015c80a71d3..e0c86e4cfb93f 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1870,7 +1870,7 @@ L0: r5 = a.f(12, 6, r4) return 1 -[case testListComprehension] +[case testListComprehension_withgil] from typing import List def f() -> List[int]: @@ -1931,7 +1931,7 @@ L7: L8: return r0 -[case testDictComprehension] +[case testDictComprehension_withgil] from typing import Dict def f() -> Dict[int, int]: return {x: x*x for x in [1,2,3] if x != 2 if x != 3} @@ -1993,7 +1993,7 @@ L7: L8: return r0 -[case testLoopsMultipleAssign] +[case testLoopsMultipleAssign_withgil] from typing import List, Tuple def f(l: List[Tuple[int, int, int]]) -> List[int]: for x, y, z in l: diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 6050931497a7b..e70d81c96bf7a 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -379,7 +379,7 @@ L0: r2 = r1 >= 0 :: signed return 1 -[case testListBuiltFromGenerator] +[case testListBuiltFromGenerator_withgil] from typing import List def f(source: List[int]) -> None: a = list(x + 1 for x in source) @@ -448,7 +448,7 @@ L8: b = r11 return 1 -[case testGeneratorNext] +[case testGeneratorNext_withgil] from typing import List, Optional def test(x: List[int]) -> None: @@ -489,7 +489,7 @@ L5: res = r6 return 1 -[case testSimplifyListUnion] +[case testSimplifyListUnion_withgil] from typing import List, Union, Optional def narrow(a: Union[List[str], List[bytes], int]) -> int: @@ -923,7 +923,7 @@ L10: a = r3 return 1 -[case testListBuiltFromStars] +[case testListBuiltFromStars_withgil] from typing import Final abc: Final = "abc" diff --git a/mypyc/test-data/irbuild-set.test b/mypyc/test-data/irbuild-set.test index 00bcff68d4ff5..ac01c58401118 100644 --- a/mypyc/test-data/irbuild-set.test +++ b/mypyc/test-data/irbuild-set.test @@ -53,7 +53,7 @@ L0: r0 = PySet_New(l) return r0 -[case testNewSetFromIterable2] +[case testNewSetFromIterable2_withgil] def f(x: int) -> int: return x @@ -273,7 +273,7 @@ L4: e = r0 return 1 -[case testNewSetFromIterable3] +[case testNewSetFromIterable3_withgil] def f1(x: int) -> int: return x diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index 8199cfdaa699e..8c5c95c8223ec 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test @@ -218,7 +218,7 @@ L5: L6: return 1 -[case testForList] +[case testForList_withgil] from typing import List def f(ls: List[int]) -> int: @@ -255,6 +255,43 @@ L3: L4: return y +[case testForList_nogil] +from typing import List + +def f(ls: List[int]) -> int: + y = 0 + for x in ls: + y = y + x + return y +[out] +def f(ls): + ls :: list + y :: int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x, r5 :: int + r6 :: native_int +L0: + y = 0 + r0 = 0 +L1: + r1 = var_object_size ls + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = CPyList_GetItemInt64(ls, r0) + r4 = unbox(int, r3) + x = r4 + r5 = CPyTagged_Add(y, x) + y = r5 +L3: + r6 = r0 + 1 + r0 = r6 + goto L1 +L4: + return y + [case testForDictBasic] from typing import Dict @@ -899,7 +936,7 @@ L0: r6 = r5 >= 0 :: signed return 1 -[case testForEnumerate] +[case testForEnumerate_withgil] from typing import List, Iterable def f(a: List[int]) -> None: @@ -967,7 +1004,7 @@ L4: L5: return 1 -[case testForZip] +[case testForZip_withgil] from typing import List, Iterable, Sequence def f(a: List[int], b: Sequence[bool]) -> None: diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 9f6b0f5d07390..808b5b9e0d4ab 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -281,7 +281,7 @@ L5: L6: return r6 -[case testTupleBuiltFromList] +[case testTupleBuiltFromList_withgil] def f(val: int) -> bool: return val % 2 == 0 @@ -1018,7 +1018,7 @@ L5: a = r42 return 1 -[case testTupleBuiltFromStars] +[case testTupleBuiltFromStars_withgil] from typing import Final abc: Final = "abc" diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index af69f30924d46..176e2616a4856 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -334,7 +334,7 @@ L3: L4: return r1 -[case testVecI64FastComprehensionFromList] +[case testVecI64FastComprehensionFromList_nogil] from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -364,7 +364,7 @@ L1: r4 = r2 < r3 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = list_get_item_unsafe l, r2 + r5 = CPyList_GetItemInt64(l, r2) r6 = unbox(i64, r5) x = r6 r7 = x + 1 diff --git a/mypyc/test-data/lowering-int.test b/mypyc/test-data/lowering-int.test index c2bcba54e444d..72c4a82dcaea3 100644 --- a/mypyc/test-data/lowering-int.test +++ b/mypyc/test-data/lowering-int.test @@ -332,7 +332,7 @@ L4: L5: return 4 -[case testLowerIntForLoop_64bit] +[case testLowerIntForLoop_withgil_64bit] from __future__ import annotations def f(l: list[int]) -> None: diff --git a/mypyc/test/test_lowering.py b/mypyc/test/test_lowering.py index 3eb4698c68793..75351cd50e4a7 100644 --- a/mypyc/test/test_lowering.py +++ b/mypyc/test/test_lowering.py @@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions from mypyc.test.testutil import ( @@ -36,6 +36,12 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) From cfc07346408d92ac91c18e3698c74b9c0d3a4a65 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 Jul 2026 12:39:10 +0100 Subject: [PATCH 25/50] [mypyc] Fix unsafe borrowing of instance attributes with free-threading (#21688) Don't borrow native attributes any more, except for final native attributes (these can't be rebound, so they are safe to borrow), and attributes with `vec` types (`vec` requires explicit synchronization by design). Work on mypyc/mypyc#1203. I used coding agent assist for this, but manually reviewed changes. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/builder.py | 16 ++ mypyc/irbuild/expression.py | 11 +- mypyc/test-data/exceptions.test | 28 ++- mypyc/test-data/irbuild-basic.test | 2 +- mypyc/test-data/irbuild-classes.test | 24 ++- mypyc/test-data/irbuild-generics.test | 2 +- mypyc/test-data/irbuild-glue-methods.test | 2 +- mypyc/test-data/irbuild-i64.test | 6 +- mypyc/test-data/irbuild-isinstance.test | 2 +- mypyc/test-data/irbuild-optional.test | 2 +- mypyc/test-data/irbuild-vec-t.test | 12 +- mypyc/test-data/opt-copy-propagation.test | 2 +- mypyc/test-data/refcount.test | 211 ++++++++++++++++++++-- mypyc/test/test_optimizations.py | 8 +- 14 files changed, 294 insertions(+), 34 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 39e2c6aa5181f..56a3f944fe77f 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1565,6 +1565,22 @@ def is_native_attr_ref(self, expr: MemberExpr) -> bool: and any(expr.name in ir.attributes for ir in obj_rtype.class_ir.mro) ) + def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: + """Is expr a direct reference to a Final native (struct) attribute of an instance? + + A Final attribute is read-only at runtime (it has no setter), so it can never be + reassigned after construction. This makes it safe to borrow even on free-threaded + builds, since no concurrent store can invalidate the borrowed reference. + """ + obj_rtype = self.node_type(expr.expr) + if not (isinstance(obj_rtype, RInstance) and obj_rtype.class_ir.is_ext_class): + return False + # Find the class that defines the attribute and check whether it's Final there. + for ir in obj_rtype.class_ir.mro: + if expr.name in ir.attributes: + return expr.name in ir.final_attributes + return False + def mark_block_unreachable(self) -> None: """Mark statements in the innermost block being processed as unreachable. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 21cb2f8df1b8f..b99e8161c08c5 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -272,9 +272,16 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if isinstance(expr.node, MypyFile) and expr.node.fullname in builder.imports: return builder.load_module(expr.node.fullname) - can_borrow = builder.is_native_attr_ref(expr) - obj = builder.accept(expr.expr, can_borrow=can_borrow) rtype = builder.node_type(expr) + # Borrowing a native attribute read is unsafe on free-threaded builds, since another + # thread could concurrently reassign the attribute and free the old value. We still borrow + # in two cases: + # - Native Final attributes are read-only at runtime, so they can never be reassigned. + # - Vec-typed attributes require manual synchronization, so we borrow them liberally. + can_borrow = builder.is_native_attr_ref(expr) and ( + not IS_FREE_THREADED or isinstance(rtype, RVec) or builder.is_final_native_attr_ref(expr) + ) + obj = builder.accept(expr.expr, can_borrow=can_borrow) if ( is_object_rprimitive(obj.type) diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 1fea2a1eb9203..b8d8b4727363c 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -553,7 +553,7 @@ L3: r3 = :: i64 return r3 -[case testExceptionWithNativeAttributeGetAndSet] +[case testExceptionWithNativeAttributeGetAndSet_withgil] class C: def __init__(self, x: int) -> None: self.x = x @@ -578,6 +578,32 @@ L0: c.x = r1 return 1 +[case testExceptionWithNativeAttributeGetAndSet_nogil] +class C: + def __init__(self, x: int) -> None: + self.x = x + +def foo(c: C, x: int) -> None: + c.x = x - c.x +[out] +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def foo(c, x): + c :: __main__.C + x, r0, r1 :: int + r2 :: bool +L0: + r0 = c.x + r1 = CPyTagged_Subtract(x, r0) + dec_ref r0 :: int + c.x = r1 + return 1 + [case testExceptionWithOverlappingFloatErrorValue] def f() -> float: return 0.0 diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index e0c86e4cfb93f..c6c231f0386be 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -2063,7 +2063,7 @@ L7: L8: return r10 -[case testProperty] +[case testProperty_withgil] class PropertyHolder: @property def value(self) -> int: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 0310d2f69d444..66caf0772ec40 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -26,7 +26,7 @@ L0: a.x = 2; r0 = is_error return 1 -[case testUserClassInList] +[case testUserClassInList_withgil] class C: x: int @@ -103,7 +103,7 @@ L0: r0 = a.n return r0 -[case testOptionalMember] +[case testOptionalMember_withgil] from typing import Optional class Node: next: Optional[Node] @@ -1239,7 +1239,7 @@ L0: __mypyc_self__.s = r0 return 1 -[case testBorrowAttribute] +[case testBorrowAttribute_withgil] def f(d: D) -> int: return d.c.x @@ -1258,6 +1258,24 @@ L0: keep_alive d return r1 +[case testCannotBorrowAttribute_nogil] +def f(d: D) -> int: + return d.c.x + +class C: + x: int +class D: + c: C +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = d.c + r1 = r0.x + return r1 + [case testNoBorrowOverPropertyAccess] class C: d: D diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 9ec29182e89b6..0d4dad4f97e23 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -40,7 +40,7 @@ L0: y = r3 return 1 -[case testGenericAttrAndTypeApplication] +[case testGenericAttrAndTypeApplication_withgil] from typing import TypeVar, Generic T = TypeVar('T') class C(Generic[T]): diff --git a/mypyc/test-data/irbuild-glue-methods.test b/mypyc/test-data/irbuild-glue-methods.test index 87b3bf7fee588..bdbdb7e2afa68 100644 --- a/mypyc/test-data/irbuild-glue-methods.test +++ b/mypyc/test-data/irbuild-glue-methods.test @@ -91,7 +91,7 @@ L0: r0 = x.foo(y) return r0 -[case testPropertyDerivedGen] +[case testPropertyDerivedGen_withgil] from typing import Callable class BaseProperty: @property diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index 44157f5a820a0..4cea7894052aa 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -1032,7 +1032,7 @@ L4: y = r3 return y -[case testBorrowOverI64Arithmetic] +[case testBorrowOverI64Arithmetic_withgil] from mypy_extensions import i64 def add_simple(c: C) -> i64: @@ -1084,7 +1084,7 @@ L0: keep_alive d, d return r4 -[case testBorrowOverI64Bitwise] +[case testBorrowOverI64Bitwise_withgil] from mypy_extensions import i64 def bitwise_simple(c: C) -> i64: @@ -2070,7 +2070,7 @@ L4: r6 = CPyFloat_FromTagged(r3) return r6 -[case testI64IsinstanceNarrowing] +[case testI64IsinstanceNarrowing_withgil] from typing import Union from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 36a9300350bd3..216a52ff2a25c 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -48,7 +48,7 @@ L2: L3: return r1 -[case testBorrowSpecialCaseWithIsinstance] +[case testBorrowSpecialCaseWithIsinstance_withgil] class C: s: str diff --git a/mypyc/test-data/irbuild-optional.test b/mypyc/test-data/irbuild-optional.test index fbf7cb148b089..252f9489dc57f 100644 --- a/mypyc/test-data/irbuild-optional.test +++ b/mypyc/test-data/irbuild-optional.test @@ -237,7 +237,7 @@ L3: L4: return 1 -[case testUnionType] +[case testUnionType_withgil] from typing import Union class A: diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index ee48d81fc29c8..ce162bd0806c7 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -512,15 +512,25 @@ L0: return r2 [case testVecTBorrowGetItem_64bit] +from typing import Final + from librt.vecs import vec from mypy_extensions import i64 class A: - x: str + def __init__(self, x: str) -> None: + # Final to allow borrowing on free-threaded builds + self.x: Final = x def f(v: vec[A], n: i64) -> int: return len(v[n].x) [out] +def A.__init__(self, x): + self :: __main__.A + x :: str +L0: + self.x = x + return 1 def f(v, n): v :: vec[__main__.A] n :: i64 diff --git a/mypyc/test-data/opt-copy-propagation.test b/mypyc/test-data/opt-copy-propagation.test index 49b80f4385fc4..9fa5edb579fac 100644 --- a/mypyc/test-data/opt-copy-propagation.test +++ b/mypyc/test-data/opt-copy-propagation.test @@ -185,7 +185,7 @@ L1: L2: return 2 -[case testIRTransformRegisterOps1] +[case testIRTransformRegisterOps1_withgil] from __future__ import annotations from typing import cast diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 7d88cb7d11623..7c7134dcfbfaa 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -651,7 +651,7 @@ L0: r4 = (r2, r3) return r4 -[case testDecomposeTuple] +[case testDecomposeTuple_withgil] from typing import Tuple class C: @@ -970,7 +970,7 @@ L0: dec_ref r4 return r5 -[case testBorrowAttribute] +[case testBorrowAttribute_withgil] def g() -> int: d = D() return d.c.x @@ -1003,7 +1003,99 @@ L0: r1 = r0.x return r1 -[case testBorrowAttributeTwice] +[case testBorrowAttribute_nogil] +from typing import Final + +def g() -> int: + d = D(C(1)) + return d.c.x + +def f(d: D) -> int: + return d.c.x + +class C: + def __init__(self, x: int) -> None: + self.x: Final = x +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def g(): + r0 :: __main__.C + r1, d :: __main__.D + r2 :: __main__.C + r3 :: int +L0: + r0 = C(2) + r1 = D(r0) + dec_ref r0 + d = r1 + r2 = borrow d.c + r3 = r2.x + dec_ref d + return r3 +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + return r1 +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + inc_ref c + self.c = c + return 1 + +[case testBorrowInheritedFinalAttribute_nogil] +from typing import Final + +def f(d: D) -> int: + return d.c.x + +class B: + def __init__(self, x: int) -> None: + self.x: Final = x +class C(B): + pass +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + return r1 +def B.__init__(self, x): + self :: __main__.B + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + inc_ref c + self.c = c + return 1 + +[case testBorrowAttributeTwice_withgil] def f(e: E) -> int: return e.d.c.x @@ -1025,7 +1117,7 @@ L0: r2 = r1.x return r2 -[case testBorrowAttributeIsNone] +[case testBorrowAttributeIsNone_withgil] from typing import Optional def f(c: C) -> bool: @@ -1058,7 +1150,7 @@ L0: r2 = r0 == r1 return r2 -[case testBorrowAttributeNarrowOptional] +[case testBorrowAttributeNarrowOptional_withgil] from typing import Optional def f(c: C) -> bool: @@ -1093,7 +1185,7 @@ L1: L2: return 0 -[case testBorrowLenArgument] +[case testBorrowLenArgument_withgil] from typing import List def f(x: C) -> int: @@ -1113,7 +1205,7 @@ L0: r2 = r1 << 1 return r2 -[case testBorrowIsinstanceArgument] +[case testBorrowIsinstanceArgument_withgil] from typing import List def f(x: C) -> bool: @@ -1152,7 +1244,7 @@ L1: L2: return 1 -[case testBorrowListGetItem1] +[case testBorrowListGetItem1_withgil] from typing import List def literal_index(x: C) -> str: @@ -1200,6 +1292,57 @@ L0: r2 = cast(str, r1) return r2 +[case testBorrowListGetItem1_nogil] +from typing import List + +def literal_index(x: C) -> str: + return x.a[0] + +def negative_index(x: C) -> str: + return x.a[-1] + +def lvar_index(x: C, n: int) -> str: + return x.a[n] + +class C: + a: List[str] + +[out] +def literal_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItemShort(r0, 0) + dec_ref r0 + r2 = cast(str, r1) + return r2 +def negative_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItemShort(r0, -2) + dec_ref r0 + r2 = cast(str, r1) + return r2 +def lvar_index(x, n): + x :: __main__.C + n :: int + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItem(r0, n) + dec_ref r0 + r2 = cast(str, r1) + return r2 + [case testBorrowListGetItem2_withgil] from typing import List @@ -1273,9 +1416,11 @@ def attr_before_index(x): r2 :: object r3 :: str L0: - r0 = borrow x.a - r1 = borrow x.n + r0 = x.a + r1 = x.n r2 = CPyList_GetItem(r0, r1) + dec_ref r0 + dec_ref r1 :: int r3 = cast(str, r2) return r3 def attr_after_index(a, i): @@ -1361,7 +1506,7 @@ L0: dec_ref a return r5 -[case testBorrowSetAttrObject] +[case testBorrowSetAttrObject_withgil] from typing import Optional def f(x: Optional[C]) -> None: @@ -1401,7 +1546,7 @@ L0: r0.b = 0; r1 = is_error return 1 -[case testBorrowIntEquality] +[case testBorrowIntEquality_withgil] def add(c: C) -> bool: return c.x == c.y @@ -1419,7 +1564,7 @@ L0: r2 = int_eq r0, r1 return r2 -[case testBorrowIntLessThan] +[case testBorrowIntLessThan_withgil] def add(c: C) -> bool: return c.x < c.y @@ -1437,7 +1582,7 @@ L0: r2 = int_lt r0, r1 return r2 -[case testBorrowIntCompareFinal] +[case testBorrowIntCompareFinal_withgil] from typing import Final X: Final = 10 @@ -1457,7 +1602,7 @@ L0: r1 = int_eq r0, 20 return r1 -[case testBorrowIntArithmetic] +[case testBorrowIntArithmetic_withgil] def add(c: C) -> int: return c.x + c.y @@ -1485,7 +1630,39 @@ L0: r2 = CPyTagged_Subtract(r0, r1) return r2 -[case testBorrowIntComparisonInIf] +[case testBorrowIntArithmetic_nogil] +def add(c: C) -> int: + return c.x + c.y + +def sub(c: C) -> int: + return c.x - c.y + +class C: + x: int + y: int +[out] +def add(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = c.x + r1 = c.y + r2 = CPyTagged_Add(r0, r1) + dec_ref r0 :: int + dec_ref r1 :: int + return r2 +def sub(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = c.x + r1 = c.y + r2 = CPyTagged_Subtract(r0, r1) + dec_ref r0 :: int + dec_ref r1 :: int + return r2 + +[case testBorrowIntComparisonInIf_withgil] def add(c: C, n: int) -> bool: if c.x == c.y: return True @@ -1509,7 +1686,7 @@ L1: L2: return 0 -[case testBorrowIntInPlaceOp] +[case testBorrowIntInPlaceOp_withgil] def add(c: C, n: int) -> None: c.x += n diff --git a/mypyc/test/test_optimizations.py b/mypyc/test/test_optimizations.py index 6ca53b134ba91..c6a40d08e2e67 100644 --- a/mypyc/test/test_optimizations.py +++ b/mypyc/test/test_optimizations.py @@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.func_ir import FuncIR from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions @@ -33,6 +33,12 @@ class OptimizationSuite(MypycDataSuite): base_path = test_temp_dir def run_case(self, testcase: DataDrivenTestCase) -> None: + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: From f3294a092bd2f43979bc1d83e3f27e1c2e0a1453 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 Jul 2026 18:41:19 +0100 Subject: [PATCH 26/50] [mypyc] Add librt.threading.Lock class (#21690) The new native `Lock` class is a partial replacement for `threading.Lock`. It's much faster than `threading.Lock` in compiled code. In a microbenchmark I saw ~2.5x to ~4x performance improvement over compiled code using stdlib. The Lock type has four platform-specific backends: - PyMutex on Python 3.14+ (all platforms); uses CPython's internal 1-byte atomic lock - SRWLOCK + condition variable on Windows < 3.14 - POSIX semaphore on Linux < 3.14 with GIL - pthread mutex + condvar fallback (macOS, free-threaded builds) All backends preserve threading.Lock semantics (cross-thread release allowed) and drop the GIL while blocking. Added optimized handling of `with lock:` statements (for `librt.threading` only). There is one significant known regression compared to CPython (beyond missing features): if two threads race to release the same lock, this triggers undefined behavior. The PyMutex C API doesn't provide a way to avoid this without a massive performance cost, so I decided to keep this limitation instead of sacrificing performance, as performance is the main goal of the new type. This is just a lock -- using it with `threading.Condition` is not supported. We may later decide to add a native `Condition` (and possibly other synchronization primitives). I used heavy coding agent assist with small, incremental changes, and multiple review iterations. I left the rather verbose comments generated by coding agents, since the code is legitimately tricky. --------- Co-authored-by: Piotr Sawicki --- mypy/typeshed/stubs/librt/librt/threading.pyi | 16 + mypyc/build.py | 6 + mypyc/codegen/emitmodule.py | 5 + mypyc/ir/deps.py | 1 + mypyc/ir/rtypes.py | 11 +- mypyc/irbuild/statement.py | 32 + mypyc/lib-rt/setup.py | 6 + mypyc/lib-rt/threading/librt_threading.c | 714 ++++++++++++++++++ mypyc/lib-rt/threading/librt_threading.h | 10 + mypyc/lib-rt/threading/librt_threading_api.c | 43 ++ mypyc/lib-rt/threading/librt_threading_api.h | 20 + mypyc/primitives/librt_threading_ops.py | 54 ++ mypyc/primitives/registry.py | 1 + mypyc/test-data/irbuild-threading.test | 115 +++ mypyc/test-data/run-threading.test | 156 ++++ mypyc/test/test_irbuild.py | 1 + mypyc/test/test_run.py | 1 + 17 files changed, 1190 insertions(+), 2 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/librt/threading.pyi create mode 100644 mypyc/lib-rt/threading/librt_threading.c create mode 100644 mypyc/lib-rt/threading/librt_threading.h create mode 100644 mypyc/lib-rt/threading/librt_threading_api.c create mode 100644 mypyc/lib-rt/threading/librt_threading_api.h create mode 100644 mypyc/primitives/librt_threading_ops.py create mode 100644 mypyc/test-data/irbuild-threading.test create mode 100644 mypyc/test-data/run-threading.test diff --git a/mypy/typeshed/stubs/librt/librt/threading.pyi b/mypy/typeshed/stubs/librt/librt/threading.pyi new file mode 100644 index 0000000000000..a3571ac7f4745 --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/threading.pyi @@ -0,0 +1,16 @@ +from types import TracebackType +from typing import final + +@final +class Lock: + def acquire(self, blocking: bool = True) -> bool: ... + def release(self) -> None: ... + def locked(self) -> bool: ... + def __enter__(self) -> bool: ... + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + /, + ) -> None: ... diff --git a/mypyc/build.py b/mypyc/build.py index b46365d263e6a..8c6eabead17c9 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -127,6 +127,12 @@ class ModDesc(NamedTuple): ), ModDesc("librt.time", ["time/librt_time.c"], ["time/librt_time.h"], []), ModDesc("librt.random", ["random/librt_random.c"], ["random/librt_random.h"], ["random"]), + ModDesc( + "librt.threading", + ["threading/librt_threading.c"], + ["threading/librt_threading.h"], + ["threading"], + ), ] try: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 6e3c122aa13f6..3b320b5321232 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -62,6 +62,7 @@ LIBRT_BASE64, LIBRT_RANDOM, LIBRT_STRINGS, + LIBRT_THREADING, LIBRT_TIME, LIBRT_VECS, Capsule, @@ -1261,6 +1262,10 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_time() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") + if LIBRT_THREADING in module.dependencies: + emitter.emit_line("if (import_librt_threading() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") if LIBRT_VECS in module.dependencies: emitter.emit_line("if (import_librt_vecs() < 0) {") emitter.emit_line("return -1;") diff --git a/mypyc/ir/deps.py b/mypyc/ir/deps.py index 751845d3a324c..b1a27f85fcfe6 100644 --- a/mypyc/ir/deps.py +++ b/mypyc/ir/deps.py @@ -110,6 +110,7 @@ def get_header(self) -> str: LIBRT_VECS: Final = Capsule("librt.vecs") LIBRT_TIME: Final = Capsule("librt.time") LIBRT_RANDOM: Final = Capsule("librt.random") +LIBRT_THREADING: Final = Capsule("librt.threading") BYTES_EXTRA_OPS: Final = SourceDep("bytes_extra_ops.c") BYTES_WRITER_EXTRA_OPS: Final = SourceDep("byteswriter_extra_ops.c") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 9d13ecc83175d..1a5515d5621a8 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -41,7 +41,7 @@ class to enable the new behavior. In rare cases, adding a new from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, Union, final from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name -from mypyc.ir.deps import LIBRT_RANDOM, LIBRT_STRINGS, LIBRT_VECS, Dependency +from mypyc.ir.deps import LIBRT_RANDOM, LIBRT_STRINGS, LIBRT_THREADING, LIBRT_VECS, Dependency from mypyc.namegen import NameGenerator if TYPE_CHECKING: @@ -550,12 +550,19 @@ def __hash__(self) -> int: } | { "librt.random.Random": RPrimitive( "librt.random.Random", is_unboxed=False, is_refcounted=True, dependencies=(LIBRT_RANDOM,) - ) + ), + "librt.threading.Lock": RPrimitive( + "librt.threading.Lock", + is_unboxed=False, + is_refcounted=True, + dependencies=(LIBRT_THREADING,), + ), } bytes_writer_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.strings.BytesWriter"] string_writer_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.strings.StringWriter"] random_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.random.Random"] +lock_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.threading.Lock"] def is_native_rprimitive(rtype: RType) -> bool: diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 428650a810dca..e90aa089305f2 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -80,6 +80,7 @@ c_pyssize_t_rprimitive, exc_rtuple, is_tagged, + lock_rprimitive, none_rprimitive, object_pointer_rprimitive, object_rprimitive, @@ -115,6 +116,7 @@ restore_exc_info_op, ) from mypyc.primitives.generic_ops import iter_op, next_raw_op, py_delattr_op +from mypyc.primitives.librt_threading_ops import lock_acquire_op, lock_release_op from mypyc.primitives.misc_ops import ( check_stop_op, coro_op, @@ -1108,6 +1110,10 @@ def transform_with( al = "a" if is_async else "" mgr_v = builder.accept(expr) + + if not is_async and mgr_v.type == lock_rprimitive: + transform_with_lock(builder, mgr_v, target, body, line) + return is_native = isinstance(mgr_v.type, RInstance) if is_native: value = builder.add(MethodCall(mgr_v, f"__{al}enter__", args=[], line=line)) @@ -1179,6 +1185,32 @@ def finally_body() -> None: ) +def transform_with_lock( + builder: IRBuilder, mgr_v: Value, target: Lvalue | None, body: GenFunc, line: int +) -> None: + """Optimized 'with' for librt.threading.Lock. + + Generate a simple try/finally with direct acquire/release calls. + Lock.__exit__ never suppresses exceptions, so we don't need the + full PEP 343 try/except/finally machinery. + """ + # __enter__: acquire the lock + value = builder.primitive_op(lock_acquire_op, [mgr_v], line) + + mgr = builder.maybe_spill(mgr_v) + + def try_body() -> None: + if target: + builder.assign(builder.get_assignment_target(target), value, line) + body() + + def finally_body() -> None: + # __exit__: release the lock (ignoring exception info) + builder.primitive_op(lock_release_op, [builder.read(mgr, line)], line) + + transform_try_finally_stmt(builder, try_body, finally_body, line) + + def transform_with_stmt(builder: IRBuilder, o: WithStmt) -> None: # Generate separate logic for each expr in it, left to right def generate(i: int) -> None: diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 371b322ca18b2..15d31a443ff86 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -164,5 +164,11 @@ def run(self) -> None: include_dirs=["."], extra_compile_args=cflags, ), + Extension( + "librt.threading", + ["threading/librt_threading.c"], + include_dirs=[".", "threading"], + extra_compile_args=cflags, + ), ] ) diff --git a/mypyc/lib-rt/threading/librt_threading.c b/mypyc/lib-rt/threading/librt_threading.c new file mode 100644 index 0000000000000..84d7346a75872 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading.c @@ -0,0 +1,714 @@ +#include "pythoncapi_compat.h" + +#define PY_SSIZE_T_CLEAN +#include +#include "librt_threading.h" +#include "mypyc_util.h" + +#if !defined(_WIN32) +#include +#include +#endif + +#if CPY_3_14_FEATURES + +// Python 3.14+ (all platforms, with or without the GIL): Use PyMutex (1-byte +// atomic lock with parking lot). PyMutex gives better interruptibility than the +// pthread fallback, and its fast release path is competitive with sem_t. +// PyMutex_LockFast, _PyMutex_LockTimed, and _PY_LOCK_DETACH are internal +// CPython APIs that might change across minor releases. +#define LOCK_BACKEND_PYMUTEX +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "internal/pycore_lock.h" + +#elif defined(_WIN32) + +// Python <3.14 on Windows: Use Slim Reader/Writer Lock +#define LOCK_BACKEND_SRWLOCK +#include + +#else + +// Python <3.14 on POSIX. +// +// Prefer a POSIX unnamed semaphore when the platform supports it well and the +// GIL is enabled, and fall back to a pthread mutex + condition variable +// otherwise. We use the same test CPython uses to pick its semaphore-based lock +// (see Python/thread_pthread.h): an unnamed semaphore is only usable when +// sem_init() actually works AND a timed wait is available. Notably this is true +// on Linux but false on macOS (whose sem_init() is a non-functional stub), so +// macOS uses the mutex+condvar fallback. Free-threaded builds also use the +// mutex+condvar fallback because the semaphore backend's `locked` bookkeeping +// relies on GIL serialization. +#if !defined(Py_GIL_DISABLED) && \ + defined(_POSIX_SEMAPHORES) && (_POSIX_SEMAPHORES + 0) != -1 && \ + (defined(HAVE_SEM_TIMEDWAIT) || defined(HAVE_SEM_CLOCKWAIT)) +#define LOCK_BACKEND_SEM +#include +#else +#define LOCK_BACKEND_PTHREAD +#include +#endif + +#endif + +// +// Lock +// +// A fast mutex lock for use from mypyc-compiled code. +// +// On Python 3.14+ (all platforms), this uses CPython's PyMutex, a 1-byte atomic +// lock backed by a parking lot for contended waits. PyMutex automatically +// releases the GIL when blocking. +// +// On Python 3.13 and earlier with Windows, this uses an SRWLOCK (Slim +// Reader/Writer Lock) plus a CONDITION_VARIABLE guarding a `locked` flag. The SRWLOCK only +// protects the flag and is never held across the user's critical section, so +// release() may be called from a thread other than the acquirer (matching +// threading.Lock semantics). This mirrors CPython's Windows lock (NRMUTEX in +// Python/thread_nt.h) and is the Windows twin of the POSIX pthread+condvar +// backend below. +// +// On Python 3.13 and earlier with POSIX systems, there are two backends, both +// of which allow release() from a thread other than the one that acquired the +// lock (matching threading.Lock semantics): +// +// - Where unnamed POSIX semaphores work well (e.g. Linux) and the GIL is +// enabled, this uses a sem_t initialized to 1: acquire is sem_wait, release +// is sem_post. Semaphores have no ownership concept, so cross-thread release +// is directly well-defined. +// +// - Otherwise (e.g. macOS, whose sem_init() is a non-functional stub, and +// free-threaded builds), this uses a pthread mutex + condition variable +// guarding a `locked` flag. The mutex only protects the flag and is never +// held across the user's critical section, so the OS mutex is always +// unlocked on the same thread that locked it. +// + +// ---------- Platform-specific lock state ---------- + +#if defined(LOCK_BACKEND_PYMUTEX) + +typedef struct { + PyObject_HEAD + PyMutex mutex; +} LockObject; + +#elif defined(LOCK_BACKEND_SRWLOCK) + +typedef struct { + PyObject_HEAD + // The SRWLOCK below does NOT represent the Python lock; like the pthread + // fallback's `mut`, it only guards `locked` and the condition variable, + // and is held just long enough to inspect/flip the flag -- never across + // the user's critical section. That is what allows release() from a + // thread other than the acquirer: SRWLOCK's same-thread-release rule is + // never violated, and clearing `locked` is just a guarded store. This + // mirrors CPython's Windows lock (NRMUTEX in Python/thread_nt.h). + SRWLOCK srw; + CONDITION_VARIABLE lock_released; + int locked; // 0=unlocked, 1=locked; protected by `srw` +} LockObject; + +#elif defined(LOCK_BACKEND_SEM) + +typedef struct { + PyObject_HEAD + sem_t sem; // counting semaphore, initialized to 1 + // Tracks the locked state for locked() and release-unlocked detection. + // The semaphore itself is the source of truth for mutual exclusion; this + // flag is advisory bookkeeping. It is set after a successful acquire and + // cleared before sem_post in release. + // + // This backend is only selected when the GIL is enabled, so this flag is a + // plain int relying on the GIL for serialization: it is only ever touched + // with the GIL held (the blocking sem_wait drops the GIL, but the flag + // store happens after the GIL is reacquired). This mirrors the old + // PyThread_type_lock wrapper bookkeeping used by CPython 3.12 and earlier, + // where _thread lock kept a plain `char locked` for sanity checks and + // locked(). + int locked; +} LockObject; + +#else // pthread mutex + condvar fallback + +typedef struct { + PyObject_HEAD + // The pthread mutex below does NOT represent the Python lock; it only + // guards the `locked` flag and the condition variable. The Python lock + // state is `locked` itself. This indirection (matching CPython's POSIX + // lock) is what allows release() from a thread other than the acquirer: + // `mut` is always locked and unlocked within a single call on a single + // thread, so pthread's same-thread-unlock rule is never violated. + pthread_mutex_t mut; + pthread_cond_t lock_released; + // Always accessed while holding `mut` (including the locked() reader), + // so a plain int is sufficient -- the mutex provides the ordering. + // Matches CPython's pthread_lock.locked (a plain char). + int locked; // 0=unlocked, 1=locked; protected by `mut` +} LockObject; + +#endif + +// ---------- Platform-specific init/acquire/release ---------- + +static inline int +Lock_init_internal(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + self->mutex = (PyMutex){0}; +#elif defined(LOCK_BACKEND_SRWLOCK) + InitializeSRWLock(&self->srw); + InitializeConditionVariable(&self->lock_released); + self->locked = 0; +#elif defined(LOCK_BACKEND_SEM) + if (sem_init(&self->sem, 0, 1) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + self->locked = 0; +#else + int status = pthread_mutex_init(&self->mut, NULL); + if (status != 0) { + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + status = pthread_cond_init(&self->lock_released, NULL); + if (status != 0) { + pthread_mutex_destroy(&self->mut); + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + self->locked = 0; +#endif + return 0; +} + +// Try to acquire the lock. Returns 1 (true) on success, 0 (false) if +// non-blocking and the lock is held, or -1 if interrupted by an error-raising +// signal handler. +static int +Lock_acquire_impl(LockObject *self, int blocking) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + if (!blocking) { + PyLockStatus r = _PyMutex_LockTimed(&self->mutex, 0, _Py_LOCK_DONT_DETACH); + return r == PY_LOCK_ACQUIRED; + } + if (PyMutex_LockFast(&self->mutex)) { + return 1; + } + PyLockStatus r = _PyMutex_LockTimed(&self->mutex, -1, + _PY_LOCK_DETACH | _PY_LOCK_HANDLE_SIGNALS); + if (r == PY_LOCK_INTR) { + return -1; + } + return 1; + +#elif defined(LOCK_BACKEND_SRWLOCK) + // `srw` only guards `locked` and the condition variable; it is held just + // long enough to inspect/flip the flag, never across the user's critical + // section. This is what lets a different thread call release(). + // + // Fast path: grab the lock without releasing the GIL if it is free. This is + // also the whole story in the non-blocking case. + AcquireSRWLockExclusive(&self->srw); + if (!self->locked) { + self->locked = 1; + ReleaseSRWLockExclusive(&self->srw); + return 1; + } + ReleaseSRWLockExclusive(&self->srw); + if (!blocking) { + return 0; + } + + // Slow path: wait for the lock to be released, with the GIL dropped so + // other Python threads can run (and release the lock). + // SleepConditionVariableSRW atomically releases the SRWLOCK while sleeping + // and reacquires it on wake, exactly like pthread_cond_wait. + Py_BEGIN_ALLOW_THREADS + AcquireSRWLockExclusive(&self->srw); + while (self->locked) { + SleepConditionVariableSRW(&self->lock_released, &self->srw, INFINITE, 0); + } + self->locked = 1; + ReleaseSRWLockExclusive(&self->srw); + Py_END_ALLOW_THREADS + return 1; + +#elif defined(LOCK_BACKEND_SEM) + // A semaphore has no ownership: any thread may sem_post a token that + // another thread consumed via sem_wait, so cross-thread release is + // directly well-defined. `locked` is advisory bookkeeping for locked() + // and for guarding against releasing an unheld lock. + // + // Fast path: try a non-blocking acquire first to avoid GIL release/reacquire + // overhead in the common uncontended case. This is also the whole story in + // the non-blocking case. + { + int status; + do { + status = sem_trywait(&self->sem); + } while (status == -1 && errno == EINTR); + if (status == 0) { + self->locked = 1; + return 1; + } + } + if (!blocking) { + return 0; // EAGAIN: already held + } + + // Slow path: block with the GIL dropped so other Python threads can run + // (and release the lock). If a signal interrupts sem_wait(), run pending + // Python signal handlers with the GIL held; retry unless a handler raises. + for (;;) { + int status; + int err = 0; + + Py_BEGIN_ALLOW_THREADS + status = sem_wait(&self->sem); + if (status == -1) { + err = errno; + } + Py_END_ALLOW_THREADS + + if (status == 0) { + self->locked = 1; + return 1; + } + + if (err != EINTR) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + +#else // pthread mutex + condvar fallback + // `mut` only guards `locked` and the condition variable; it is held just + // long enough to inspect/flip the flag, never across the user's critical + // section. This is what lets a different thread call release(). + // + // Fast path: grab the lock without releasing the GIL if it is free. This is + // also the whole story in the non-blocking case. + pthread_mutex_lock(&self->mut); + if (!self->locked) { + self->locked = 1; + pthread_mutex_unlock(&self->mut); + return 1; + } + pthread_mutex_unlock(&self->mut); + if (!blocking) { + return 0; + } + + // Slow path: wait for the lock to be released, with the GIL dropped so + // other Python threads can run (and release the lock). If we wake but do + // not get the lock, give pending Python signal handlers a chance to run, + // matching CPython's pthread fallback. + for (;;) { + int acquired = 0; + int interrupted = 0; + int status; + + Py_BEGIN_ALLOW_THREADS + status = pthread_mutex_lock(&self->mut); + if (status == 0) { + while (self->locked) { + status = pthread_cond_wait(&self->lock_released, &self->mut); + if (status != 0) { + break; + } + if (self->locked) { + interrupted = 1; + break; + } + } + if (status == 0 && !interrupted) { + self->locked = 1; + acquired = 1; + } + int unlock_status = pthread_mutex_unlock(&self->mut); + if (status == 0) { + status = unlock_status; + } + } + Py_END_ALLOW_THREADS + + if (status != 0) { + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + if (acquired) { + return 1; + } + if (interrupted && Py_MakePendingCalls() < 0) { + return -1; + } + } +#endif +} + +// Release the lock. Returns 0 on success, -1 if the lock was not held. +static int +Lock_release_impl(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + // threading.Lock is unowned, so release() may be called from a different + // thread than acquire(). CPython's atomic _PyMutex_TryUnlock() is not part + // of the public API and is not exported by all CPython builds. This fast + // partial-replacement path deliberately avoids a second guard mutex: + // ordinary release() of an unlocked lock raises RuntimeError, but racy + // erroneous release() calls are undefined behavior and may make + // PyMutex_Unlock() abort. + if (!PyMutex_IsLocked(&self->mutex)) { + return -1; + } + PyMutex_Unlock(&self->mutex); + return 0; + +#elif defined(LOCK_BACKEND_SRWLOCK) + AcquireSRWLockExclusive(&self->srw); + if (!self->locked) { + ReleaseSRWLockExclusive(&self->srw); + return -1; + } + self->locked = 0; + // Wake one waiter (if any). Signalling under `srw` is fine and avoids a + // lost-wakeup race. + WakeConditionVariable(&self->lock_released); + ReleaseSRWLockExclusive(&self->srw); + return 0; + +#elif defined(LOCK_BACKEND_SEM) + // Check-then-clear the flag, then post. This backend is only selected when + // the GIL is enabled, so the check and clear are serialized without an + // atomic. + if (!self->locked) { + return -1; + } + self->locked = 0; + sem_post(&self->sem); + return 0; + +#else // pthread mutex + condvar fallback + pthread_mutex_lock(&self->mut); + if (!self->locked) { + pthread_mutex_unlock(&self->mut); + return -1; + } + self->locked = 0; + // Wake one waiter (if any). Signalling under `mut` is fine and avoids a + // lost-wakeup race. + pthread_cond_signal(&self->lock_released); + pthread_mutex_unlock(&self->mut); + return 0; +#endif +} + +static inline int +Lock_is_locked(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + return PyMutex_IsLocked(&self->mutex); +#elif defined(LOCK_BACKEND_SRWLOCK) + // locked() is not expected to be on a perf-critical path, so take the + // SRWLOCK (shared) for a clean read of the guarded flag rather than + // relying on an atomic/volatile field. + AcquireSRWLockShared(&self->srw); + int result = self->locked; + ReleaseSRWLockShared(&self->srw); + return result; +#elif defined(LOCK_BACKEND_SEM) + // The flag is GIL-serialized (see the struct comment); locked() is a + // plain read, matching the old CPython _thread lock bookkeeping described + // above. + return self->locked != 0; +#else // pthread mutex + condvar fallback + // `locked` is only ever accessed under `mut`; take it for a clean read. + // locked() is not expected to be on a perf-critical path. + pthread_mutex_lock(&self->mut); + int result = self->locked; + pthread_mutex_unlock(&self->mut); + return result; +#endif +} + +// ---------- Python type methods (shared across platforms) ---------- + +static PyTypeObject LockType; + +static PyObject * +Lock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &LockType) { + PyErr_SetString(PyExc_TypeError, "Lock cannot be subclassed"); + return NULL; + } + + LockObject *self = (LockObject *)type->tp_alloc(type, 0); + if (self != NULL && Lock_init_internal(self) < 0) { + type->tp_free((PyObject *)self); + return NULL; + } + return (PyObject *)self; +} + +static int +Lock_init(LockObject *self, PyObject *args, PyObject *kwds) +{ + if (!PyArg_ParseTuple(args, "")) { + return -1; + } + + if (kwds != NULL && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "Lock() takes no keyword arguments"); + return -1; + } + + return 0; +} + +static void +Lock_dealloc(LockObject *self) +{ +#if defined(LOCK_BACKEND_SEM) + sem_destroy(&self->sem); +#elif defined(LOCK_BACKEND_PTHREAD) + // `mut` is only ever held transiently within a single call, so it is + // always unlocked here even if the Python lock is still "locked". + // Some pthread implementations require the cond to be destroyed first. + pthread_cond_destroy(&self->lock_released); + pthread_mutex_destroy(&self->mut); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject * +Lock_acquire(LockObject *self, PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) +{ + int blocking = 1; + + Py_ssize_t nkw = kwnames ? PyTuple_GET_SIZE(kwnames) : 0; + if (nargs + nkw > 1) { + PyErr_SetString(PyExc_TypeError, "acquire() takes at most 1 argument"); + return NULL; + } + + if (nargs == 1) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) + return NULL; + } else if (nkw == 1) { + PyObject *key = PyTuple_GET_ITEM(kwnames, 0); + if (PyUnicode_CompareWithASCIIString(key, "blocking") != 0) { + PyErr_Format(PyExc_TypeError, + "acquire() got an unexpected keyword argument '%U'", + key); + return NULL; + } + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) + return NULL; + } + + int result = Lock_acquire_impl(self, blocking); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject * +Lock_release(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (Lock_release_impl(self) < 0) { + PyErr_SetString(PyExc_RuntimeError, "cannot release an unlocked lock"); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +Lock_locked(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyBool_FromLong(Lock_is_locked(self)); +} + +static PyObject * +Lock_enter(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + int result = Lock_acquire_impl(self, 1); + if (result < 0) + return NULL; + return PyBool_FromLong(result); +} + +static PyObject * +Lock_exit(LockObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + return Lock_release(self, NULL); +} + +static PyMethodDef Lock_methods[] = { + {"acquire", (PyCFunction)(void(*)(void))Lock_acquire, METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("Acquire the lock, blocking or non-blocking.\n" + "Returns True if the lock was acquired, False otherwise.")}, + {"release", (PyCFunction)Lock_release, METH_NOARGS, + PyDoc_STR("Release the lock.")}, + {"locked", (PyCFunction)Lock_locked, METH_NOARGS, + PyDoc_STR("Return True if the lock is currently held.")}, + {"__enter__", (PyCFunction)Lock_enter, METH_NOARGS, + PyDoc_STR("Acquire the lock.")}, + {"__exit__", (PyCFunction)Lock_exit, METH_FASTCALL, + PyDoc_STR("Release the lock.")}, + {NULL} +}; + +static PyTypeObject LockType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "Lock", + .tp_doc = PyDoc_STR("A fast mutual exclusion lock"), + .tp_basicsize = sizeof(LockObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = Lock_new, + .tp_init = (initproc)Lock_init, + .tp_dealloc = (destructor)Lock_dealloc, + .tp_methods = Lock_methods, +}; + +static PyTypeObject * +Lock_type_internal(void) { + return &LockType; +} + +// Create a new Lock object (for use from compiled code) +static PyObject * +Lock_new_internal(void) { + LockObject *self = (LockObject *)LockType.tp_alloc(&LockType, 0); + if (self != NULL && Lock_init_internal(self) < 0) { + LockType.tp_free((PyObject *)self); + return NULL; + } + return (PyObject *)self; +} + +// Acquire the lock (blocking), for use from compiled code. +// Returns true on success, sets error and returns 2 (ERR_MAGIC) on failure. +static char +Lock_acquire_internal(PyObject *self) { + int result = Lock_acquire_impl((LockObject *)self, 1); + if (result < 0) { + return 2; + } + return (char)result; +} + +// Acquire the lock with explicit blocking arg, for use from compiled code. +// Returns true if acquired, false otherwise. Sets error and returns 2 +// (ERR_MAGIC) on failure. +static char +Lock_acquire_blocking_internal(PyObject *self, char blocking) { + int result = Lock_acquire_impl((LockObject *)self, blocking); + if (result < 0) { + return 2; + } + return (char)result; +} + +// Release the lock, for use from compiled code. +// Returns 0 (None) on success, sets error and returns 2 (ERR_MAGIC) on failure. +static char +Lock_release_internal(PyObject *self) { + if (Lock_release_impl((LockObject *)self) < 0) { + PyErr_SetString(PyExc_RuntimeError, "cannot release an unlocked lock"); + return 2; + } + return 0; +} + +// Check if the lock is held, for use from compiled code. +static char +Lock_locked_internal(PyObject *self) { + return (char)Lock_is_locked((LockObject *)self); +} + +static PyMethodDef librt_threading_module_methods[] = { + {NULL, NULL, 0, NULL} +}; + +static int +threading_abi_version(void) { + return LIBRT_THREADING_ABI_VERSION; +} + +static int +threading_api_version(void) { + return LIBRT_THREADING_API_VERSION; +} + +static int +librt_threading_module_exec(PyObject *m) +{ + if (PyType_Ready(&LockType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Lock", (PyObject *)&LockType) < 0) { + return -1; + } + + // Export mypyc internal C API via capsule + static void *threading_api[LIBRT_THREADING_API_LEN] = { + (void *)threading_abi_version, + (void *)threading_api_version, + (void *)Lock_type_internal, + (void *)Lock_new_internal, + (void *)Lock_acquire_internal, + (void *)Lock_release_internal, + (void *)Lock_locked_internal, + (void *)Lock_acquire_blocking_internal, + }; + PyObject *c_api_object = PyCapsule_New((void *)threading_api, "librt.threading._C_API", NULL); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot librt_threading_module_slots[] = { + {Py_mod_exec, librt_threading_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef librt_threading_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "threading", + .m_doc = "Fast threading primitives optimized for mypyc", + .m_size = 0, + .m_methods = librt_threading_module_methods, + .m_slots = librt_threading_module_slots, +}; + +PyMODINIT_FUNC +PyInit_threading(void) +{ + return PyModuleDef_Init(&librt_threading_module); +} diff --git a/mypyc/lib-rt/threading/librt_threading.h b/mypyc/lib-rt/threading/librt_threading.h new file mode 100644 index 0000000000000..a22284f3fdbcd --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading.h @@ -0,0 +1,10 @@ +#ifndef LIBRT_THREADING_H +#define LIBRT_THREADING_H + +#include + +#define LIBRT_THREADING_ABI_VERSION 1 +#define LIBRT_THREADING_API_VERSION 1 +#define LIBRT_THREADING_API_LEN 8 + +#endif // LIBRT_THREADING_H diff --git a/mypyc/lib-rt/threading/librt_threading_api.c b/mypyc/lib-rt/threading/librt_threading_api.c new file mode 100644 index 0000000000000..156975d243638 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading_api.c @@ -0,0 +1,43 @@ +#include "librt_threading_api.h" + +void *LibRTThreading_API[LIBRT_THREADING_API_LEN] = {0}; + +int +import_librt_threading(void) +{ + PyObject *mod = PyImport_ImportModule("librt.threading"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + void **capsule = (void **)PyCapsule_Import("librt.threading._C_API", 0); + if (capsule == NULL) + return -1; + + // Only after version validation succeeds can we safely copy the full table. + int (*abi_version)(void) = (int (*)(void))capsule[0]; + int (*api_version)(void) = (int (*)(void))capsule[1]; + if (abi_version() != LIBRT_THREADING_ABI_VERSION) { + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.threading, expected %d, found %d", + LIBRT_THREADING_ABI_VERSION, + abi_version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + if (api_version() < LIBRT_THREADING_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.threading, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_THREADING_API_VERSION, + api_version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + // Provider API version is >= our expected version, which (by the API + // compatibility contract) means it has at least LIBRT_THREADING_API_LEN + // entries, so this copy is safe. + memcpy(LibRTThreading_API, capsule, sizeof(LibRTThreading_API)); + return 0; +} diff --git a/mypyc/lib-rt/threading/librt_threading_api.h b/mypyc/lib-rt/threading/librt_threading_api.h new file mode 100644 index 0000000000000..acb3414d79561 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading_api.h @@ -0,0 +1,20 @@ +#ifndef LIBRT_THREADING_API_H +#define LIBRT_THREADING_API_H + +#include "librt_threading.h" + +int +import_librt_threading(void); + +extern void *LibRTThreading_API[LIBRT_THREADING_API_LEN]; + +#define LibRTThreading_ABIVersion (*(int (*)(void)) LibRTThreading_API[0]) +#define LibRTThreading_APIVersion (*(int (*)(void)) LibRTThreading_API[1]) +#define LibRTThreading_Lock_type_internal (*(PyTypeObject* (*)(void)) LibRTThreading_API[2]) +#define LibRTThreading_Lock_new_internal (*(PyObject* (*)(void)) LibRTThreading_API[3]) +#define LibRTThreading_Lock_acquire_internal (*(char (*)(PyObject *self)) LibRTThreading_API[4]) +#define LibRTThreading_Lock_release_internal (*(char (*)(PyObject *self)) LibRTThreading_API[5]) +#define LibRTThreading_Lock_locked_internal (*(char (*)(PyObject *self)) LibRTThreading_API[6]) +#define LibRTThreading_Lock_acquire_blocking_internal (*(char (*)(PyObject *self, char blocking)) LibRTThreading_API[7]) + +#endif // LIBRT_THREADING_API_H diff --git a/mypyc/primitives/librt_threading_ops.py b/mypyc/primitives/librt_threading_ops.py new file mode 100644 index 0000000000000..c060742dc2050 --- /dev/null +++ b/mypyc/primitives/librt_threading_ops.py @@ -0,0 +1,54 @@ +from mypyc.ir.deps import LIBRT_THREADING +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import bool_rprimitive, lock_rprimitive, none_rprimitive +from mypyc.primitives.registry import function_op, method_op + +# Lock() +function_op( + name="librt.threading.Lock", + arg_types=[], + return_type=lock_rprimitive, + c_function_name="LibRTThreading_Lock_new_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.acquire() -- blocking acquire, returns True unless it raises +lock_acquire_op = method_op( + name="acquire", + arg_types=[lock_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_acquire_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.acquire(blocking) -- acquire with explicit blocking argument +method_op( + name="acquire", + arg_types=[lock_rprimitive, bool_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_acquire_blocking_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.release() +lock_release_op = method_op( + name="release", + arg_types=[lock_rprimitive], + return_type=none_rprimitive, + c_function_name="LibRTThreading_Lock_release_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.locked() +method_op( + name="locked", + arg_types=[lock_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_locked_internal", + error_kind=ERR_NEVER, + dependencies=[LIBRT_THREADING], +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 22422987b4277..060af0f88020c 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -411,6 +411,7 @@ def load_global_op(name: str, type: RType, src: str) -> LoadAddressDescription: import mypyc.primitives.int_ops import mypyc.primitives.librt_random_ops import mypyc.primitives.librt_strings_ops +import mypyc.primitives.librt_threading_ops import mypyc.primitives.librt_time_ops import mypyc.primitives.librt_vecs_ops import mypyc.primitives.list_ops diff --git a/mypyc/test-data/irbuild-threading.test b/mypyc/test-data/irbuild-threading.test new file mode 100644 index 0000000000000..652ccedcb9d75 --- /dev/null +++ b/mypyc/test-data/irbuild-threading.test @@ -0,0 +1,115 @@ +[case testLockBasics] +from librt.threading import Lock + +def lock_create() -> Lock: + return Lock() + +def lock_acquire(lk: Lock) -> bool: + return lk.acquire() + +def lock_release(lk: Lock) -> None: + lk.release() + +def lock_locked(lk: Lock) -> bool: + return lk.locked() +[out] +def lock_create(): + r0 :: librt.threading.Lock +L0: + r0 = LibRTThreading_Lock_new_internal() + return r0 +def lock_acquire(lk): + lk :: librt.threading.Lock + r0 :: bool +L0: + r0 = LibRTThreading_Lock_acquire_internal(lk) + return r0 +def lock_release(lk): + lk :: librt.threading.Lock + r0 :: None +L0: + r0 = LibRTThreading_Lock_release_internal(lk) + return 1 +def lock_locked(lk): + lk :: librt.threading.Lock + r0 :: bool +L0: + r0 = LibRTThreading_Lock_locked_internal(lk) + return r0 + +[case testLockAcquireRelease] +from librt.threading import Lock + +def acquire_release() -> None: + lk = Lock() + lk.acquire() + lk.release() +[out] +def acquire_release(): + r0, lk :: librt.threading.Lock + r1 :: bool + r2 :: None +L0: + r0 = LibRTThreading_Lock_new_internal() + lk = r0 + r1 = LibRTThreading_Lock_acquire_internal(lk) + r2 = LibRTThreading_Lock_release_internal(lk) + return 1 + +[case testLockAcquireBlocking] +from librt.threading import Lock + +def lock_acquire_blocking(lk: Lock, b: bool) -> bool: + return lk.acquire(b) +[out] +def lock_acquire_blocking(lk, b): + lk :: librt.threading.Lock + b, r0 :: bool +L0: + r0 = LibRTThreading_Lock_acquire_blocking_internal(lk, b) + return r0 + +[case testLockWith] +from librt.threading import Lock + +def with_lock(lk: Lock) -> None: + with lk: + a: list[int] = [] +[out] +def with_lock(lk): + lk :: librt.threading.Lock + r0 :: bool + r1, a :: list + r2, r3, r4 :: tuple[object, object, object] + r5 :: None + r6 :: bit +L0: + r0 = LibRTThreading_Lock_acquire_internal(lk) +L1: + r1 = PyList_New(0) + a = r1 +L2: +L3: + r2 = :: tuple[object, object, object] + r3 = r2 + goto L5 +L4: (handler for L1) + r4 = CPy_CatchError() + r3 = r4 +L5: + r5 = LibRTThreading_Lock_release_internal(lk) + if is_error(r3) goto L7 else goto L6 +L6: + CPy_Reraise() + unreachable +L7: + goto L11 +L8: (handler for L5, L6) + if is_error(r3) goto L10 else goto L9 +L9: + CPy_RestoreExcInfo(r3) +L10: + r6 = CPy_KeepPropagating() + unreachable +L11: + return 1 diff --git a/mypyc/test-data/run-threading.test b/mypyc/test-data/run-threading.test new file mode 100644 index 0000000000000..8af8755b76c44 --- /dev/null +++ b/mypyc/test-data/run-threading.test @@ -0,0 +1,156 @@ +# Test cases for librt.threading (compile and run) + +[case testLockBasics_librt] +from typing import Any +from testutil import assertRaises +from librt.threading import Lock + +class BadBool: + def __bool__(self) -> bool: + raise RuntimeError("bad bool") + +def test_lock_basic() -> None: + lock = Lock() + assert not lock.locked() + assert lock.acquire() + assert lock.locked() + lock.release() + assert not lock.locked() + +def test_lock_context_manager() -> None: + lock = Lock() + with lock as acquired: + assert acquired is True + assert lock.locked() + assert not lock.locked() + +def test_lock_non_blocking() -> None: + lock = Lock() + assert lock.acquire() + assert not lock.acquire(False) + lock.release() + assert lock.acquire(False) + lock.release() + +def test_contention() -> None: + import threading + lock = Lock() + counter = [0] + n_threads = 4 + n_increments = 10000 + + def worker() -> None: + for _ in range(n_increments): + lock.acquire() + counter[0] += 1 + lock.release() + + threads = [threading.Thread(target=worker) for _ in range(n_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + assert counter[0] == n_threads * n_increments + +def test_cross_thread_release() -> None: + # threading.Lock is unowned: a lock acquired on one thread may be + # released from another, after which another thread can acquire it. + import threading + lock = Lock() + assert lock.acquire() + + started = threading.Event() + released = threading.Event() + + def releaser() -> None: + started.set() + # Hand the lock off from a different thread than the acquirer. + lock.release() + released.set() + + t = threading.Thread(target=releaser) + t.start() + started.wait() + # This may block until releaser releases the lock on the other thread. + assert lock.acquire() + t.join() + assert released.is_set() + lock.release() + assert not lock.locked() + +def test_context_manager_exception() -> None: + lock = Lock() + try: + with lock: + assert lock.locked() + raise ValueError("test") + except ValueError: + pass + assert not lock.locked() + +def test_acquire_blocking_true() -> None: + lock = Lock() + assert lock.acquire(True) + assert lock.locked() + lock.release() + +def test_lock_constructor_errors() -> None: + lock_type: Any = Lock + make_type: Any = type + with assertRaises(TypeError): + lock_type(1) + with assertRaises(TypeError): + lock_type(foo=1) + with assertRaises(TypeError): + make_type("LockSubclass", (lock_type,), {}) + +def test_lock_acquire_argument_errors() -> None: + lock: Any = Lock() + with assertRaises(TypeError): + lock.acquire(True, False) + with assertRaises(TypeError): + lock.acquire(foo=True) + +def test_lock_acquire_blocking_truthiness() -> None: + lock: Any = Lock() + assert lock.acquire(blocking=True) + assert lock.locked() + assert not lock.acquire(blocking=False) + lock.release() + + assert lock.acquire(None) + assert lock.locked() + assert not lock.acquire(None) + lock.release() + + assert lock.acquire(1) + lock.release() + assert lock.acquire(0) + lock.release() + +def test_lock_acquire_blocking_bool_error() -> None: + lock: Any = Lock() + with assertRaises(RuntimeError, "bad bool"): + lock.acquire(BadBool()) + with assertRaises(RuntimeError, "bad bool"): + lock.acquire(blocking=BadBool()) + +def test_lock_exit_manual_call() -> None: + lock: Any = Lock() + lock.acquire() + assert lock.__exit__(None, None, None) is None + assert not lock.locked() + + lock.acquire() + assert lock.__exit__() is None + assert not lock.locked() + +def test_release_unlocked() -> None: + lock = Lock() + with assertRaises(RuntimeError): + lock.release() + # Also after acquire + release + lock.acquire() + lock.release() + with assertRaises(RuntimeError): + lock.release() diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index b11e425e51d3d..50ffc9004743e 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -61,6 +61,7 @@ "irbuild-librt-strings.test", "irbuild-librt-random.test", "irbuild-base64.test", + "irbuild-threading.test", "irbuild-time.test", "irbuild-match.test", ] diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 9004a28ebf598..868f83654b47c 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -82,6 +82,7 @@ "run-base64.test", "run-librt-time.test", "run-librt-random.test", + "run-threading.test", "run-match.test", "run-vecs-i64-interp.test", "run-vecs-misc-interp.test", From e986558a7ce0c32a5115137e9a6cfa24984459ee Mon Sep 17 00:00:00 2001 From: esarp <11684270+esarp@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:40:02 -0500 Subject: [PATCH 27/50] [Chore] Update changelog for 2.2 (#21691) --- CHANGELOG.md | 277 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a2c9191b71a1..02739ef6591fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,283 @@ ## Next Release +## Mypy 2.2 + +We've just uploaded mypy 2.2.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support for Closed TypedDicts (PEP 728) + +Mypy now supports closed TypedDicts as specified in PEP 728. A closed TypedDict cannot have extra +keys beyond those explicitly defined. This allows the type checker to determine that certain +operations are safe when they otherwise wouldn't be due to the potential presence of unknown keys. + +You can use the `closed` keyword argument with `TypedDict`: + +```python +HasName = TypedDict("HasName", {"name": str}) +HasOnlyName = TypedDict("HasOnlyName", {"name": str}, closed=True) +Movie = TypedDict("Movie", {"name": str, "year": int}) + +movie: Movie = {"name": "Nimona", "year": 2023} +has_name: HasName = movie # OK: HasName is open (default) +has_only_name: HasOnlyName = movie # Error: HasOnlyName is closed and Movie has extra "year" key +``` + +Closed TypedDicts enable more precise type checking because the type checker knows exactly which +keys are present. This is particularly useful when working with TypedDict unions or when you want +to ensure that a TypedDict conforms to an exact shape. + +The `closed` keyword also enables safe type narrowing with `in` checks: + +```python +Book = TypedDict('Book', {'book': str}, closed=True) +DVD = TypedDict('DVD', {'dvd': str}, closed=True) +type Inventory = Book | DVD + +def print_type(inventory: Inventory) -> None: + if "book" in inventory: + # Type is narrowed to Book here - safe because DVD is closed + print(inventory["book"]) + else: + # Type is narrowed to DVD here + print(inventory["dvd"]) +``` + +The `closed` keyword is also supported in class-based syntax: + +```python +class HasOnlyName(TypedDict, closed=True): + name: str +``` + +Note that closed TypedDicts are structural types, so a closed TypedDict is assignable to an open +TypedDict with the same keys, but not vice versa. + +Contributed by Alice (PR [21382](https://github.com/python/mypy/pull/21382)). + +### Complete Support for Type Variable Defaults (PEP 696) + +Mypy now has complete support for type variable defaults as specified in PEP 696. This allows you to +specify default values for type parameters in generic classes, functions, and type aliases. + +Traditional syntax (Python 3.11 and earlier): + +```python +T = TypeVar("T", default=int) # This means that if no type is specified T = int + +@dataclass +class Box(Generic[T]): + value: T | None = None + +reveal_type(Box()) # type is Box[int] +reveal_type(Box(value="Hello World!")) # type is Box[str] +``` + +New syntax (Python 3.12+): + +```python +class Box[T = int]: + def __init__(self, value: T) -> None: + self.value = value + +reveal_type(Box()) # type is Box[int] +reveal_type(Box(value="Hello World!")) # type is Box[str] +``` + +Type variable defaults work with all forms of generics, including classes, functions, and type aliases. +This release completes the implementation by fixing various edge cases involving recursive defaults, +dependencies between type variables, and interactions with variadic generics. + +Contributed by Ivan Levkivskyi (PRs [21491](https://github.com/python/mypy/pull/21491), +[21526](https://github.com/python/mypy/pull/21526), [21544](https://github.com/python/mypy/pull/21544)). + +### Respect Explicit Return Type of `__new__()` + +Mypy now respects explicitly annotated return types in `__new__()` methods. Previously, mypy would +always assume that `__new__()` returns an instance of the current class, ignoring explicit annotations. + +With this change, if you explicitly annotate a return type that differs from the implicit type, mypy +will use the explicit annotation: + +```python +class Factory: + def __new__(cls) -> Product: + return Product() + +reveal_type(Factory()) # type is Product, not Factory +``` + +Note that mypy still gives an error at the definition site if the explicit annotation is not a +subtype of the current class, since this is technically not type-safe. + +For backwards compatibility, there are two exceptions: +- If the return type is `Any`, mypy will still use the current class as the return type. +- If the explicit return type comes from a superclass and is a supertype of the implicit return type, + mypy will use the implicit (more specific) type: + +```python +class A: + def __new__(cls) -> A: ... + +reveal_type(A()) # type is A + +class B: + def __new__(cls) -> B: + return cls() + +class C(B): ... +reveal_type(C()) # type is C +``` + +This fixes several long-standing issues where explicit `__new__()` return types were ignored. + +Contributed by Ivan Levkivskyi (PR [21441](https://github.com/python/mypy/pull/21441)). + +### TypeForm Support No Longer Experimental + +Support for `TypeForm` is no longer experimental. `TypeForm` (introduced in Python 3.14) allows you +to annotate parameters that accept type expressions, providing better type checking for functions +that work with types as values. + +```python +from typing import TypeForm + +def make_list(tp: TypeForm[T]) -> list[T]: + ... + +# Correctly typed as list[int] +int_list = make_list(int) +``` + +`TypeForm` support was previously reverted from mypy 2.1 due to a performance regression, but this +has now been mitigated. + +Contributed by Ivan Levkivskyi and Jelle Zijlstra (PRs [21262](https://github.com/python/mypy/pull/21262), [21591](https://github.com/python/mypy/pull/21591), +[21459](https://github.com/python/mypy/pull/21459)). + +### Experimental WASM Wheel for Python 3.14 + +Mypy now ships an experimental WebAssembly (WASM) wheel for Python 3.14. This allows mypy to run +in WASM environments such as Pyodide and browser-based Python implementations. + +The WASM wheel is considered experimental and may have limitations compared to native builds. Please +report any issues you encounter when using mypy in WASM environments. + +Contributed by Ivan Levkivskyi (PR [21671](https://github.com/python/mypy/pull/21671)). + +### Mypyc Free-threading Improvements + +- Make function wrappers thread-safe on free-threaded builds (Jukka Lehtosalo, PR [21620](https://github.com/python/mypy/pull/21620)) +- Make list remove and index thread-safe on free-threaded builds (Jukka Lehtosalo, PR [21614](https://github.com/python/mypy/pull/21614)) +- Fix dict iteration memory safety on free-threaded builds (Jukka Lehtosalo, PR [21617](https://github.com/python/mypy/pull/21617)) +- Make some dict primitives thread-safe on free-threading builds (Jukka Lehtosalo, PR [21616](https://github.com/python/mypy/pull/21616)) +- Fix free-threading race condition in argument parsing (Jukka Lehtosalo, PR [21613](https://github.com/python/mypy/pull/21613)) +- Document free threading and other doc updates (Jukka Lehtosalo, PR [21494](https://github.com/python/mypy/pull/21494)) + +### `librt.strings` Updates + +- Add `librt.strings.toupper` and `librt.strings.tolower` codepoint primitives (Vaggelis Danias, PR [21553](https://github.com/python/mypy/pull/21553)) +- Add `librt.strings.isidentifier` codepoint primitive (Vaggelis Danias, PR [21522](https://github.com/python/mypy/pull/21522)) +- Add `librt.strings.isalpha` codepoint primitive (Vaggelis Danias, PR [21521](https://github.com/python/mypy/pull/21521)) +- Add `librt.strings.isalnum` codepoint primitive (Vaggelis Danias, PR [21509](https://github.com/python/mypy/pull/21509)) +- Add `librt.strings.isdigit` codepoint primitive (Vaggelis Danias, PR [21504](https://github.com/python/mypy/pull/21504)) +- Add `librt.strings.isspace` char primitive (Vaggelis Danias, PR [21462](https://github.com/python/mypy/pull/21462)) + +### Mypyc Improvements + +- Fix name lookup when class var and module var have the same name (Jukka Lehtosalo, PR [21594](https://github.com/python/mypy/pull/21594)) +- Report file and line number on uncaught exceptions (Jukka Lehtosalo, PR [21584](https://github.com/python/mypy/pull/21584)) +- Use `other` arg instead of `self` for RHS type (Ryan Heard, PR [21569](https://github.com/python/mypy/pull/21569)) +- Use `method_sig` to get the method signature (Ryan Heard, PR [21567](https://github.com/python/mypy/pull/21567)) +- Preserve inherited attribute defaults under `separate=True` (Jo, PR [21547](https://github.com/python/mypy/pull/21547)) +- Fix missing cross-group header deps in incremental builds (Jo, PR [21490](https://github.com/python/mypy/pull/21490)) +- Fix cross-group call to inherited `__mypyc_defaults_setup` (Jo, PR [21481](https://github.com/python/mypy/pull/21481)) +- Fix non-deterministic class struct layout under `separate=True` (Vaggelis Danias, PR [21530](https://github.com/python/mypy/pull/21530)) +- Specialize `s[i] == 'x'` to a codepoint int compare (Vaggelis Danias, PR [21579](https://github.com/python/mypy/pull/21579)) +- Fix reference leak in mypyc bytes concatenation (Colinxu2020, PR [21469](https://github.com/python/mypy/pull/21469)) + +### Fixes to Crashes + +- Fix crash on invalid recursive variadic alias (Ivan Levkivskyi, PR [21572](https://github.com/python/mypy/pull/21572)) +- Fix crashes on variadic unpacking in synthetic types (Ivan Levkivskyi, PR [21555](https://github.com/python/mypy/pull/21555)) +- Fix crash on unhandled meet variadic tuple vs instance (Ivan Levkivskyi, PR [21558](https://github.com/python/mypy/pull/21558)) +- Fix crash on deferred generic class nested in function (Ivan Levkivskyi, PR [21557](https://github.com/python/mypy/pull/21557)) +- Fix crash in new-style type alias with variadic unpack (Ivan Levkivskyi, PR [21551](https://github.com/python/mypy/pull/21551)) +- Fix various crashes on recursive type variable defaults (Ivan Levkivskyi, PR [21491](https://github.com/python/mypy/pull/21491)) +- Fix crash for empty `Annotated` type application (Rayan Salhab, PR [21503](https://github.com/python/mypy/pull/21503)) +- Fix crash on `Unpack` used without arguments in class bases (Sai Asish Y, PR [21470](https://github.com/python/mypy/pull/21470)) + +### Performance Improvements + +- Memoize the options snapshot (Kevin Kannammalil, PR [21354](https://github.com/python/mypy/pull/21354)) +- Don't include `not_ready_deps` tracking as relating to mypy internals (Kevin Kannammalil, PR [21389](https://github.com/python/mypy/pull/21389)) +- Speed up transitive dependency hash for singleton SCCs (Kevin Kannammalil, PR [21390](https://github.com/python/mypy/pull/21390)) +- Optimize typeform checks (Jelle Zijlstra, PR [21459](https://github.com/python/mypy/pull/21459)) + +### Improvements to the Native Parser + +- Support `--shadow-file` with `--native-parser` (Jukka Lehtosalo, PR [21623](https://github.com/python/mypy/pull/21623)) +- Add Python version checks to native parser (Kevin Kannammalil, PR [21539](https://github.com/python/mypy/pull/21539)) +- Allow nativeparse to parse source code directly (bzoracler, PR [21260](https://github.com/python/mypy/pull/21260)) + +### Other Notable Fixes and Improvements + +- Add function definition notes for `too many positional arguments` errors (Kevin Kannammalil, PR [21410](https://github.com/python/mypy/pull/21410)) +- Fix the exportjson tool (.ff cache to .json conversion) (Jukka Lehtosalo, PR [21628](https://github.com/python/mypy/pull/21628)) +- Support floats in JSON in fixed-format cache (Ivan Levkivskyi, PR [21603](https://github.com/python/mypy/pull/21603)) +- Update `TypedDictType.__init__` signature to preserve backward compat (Jukka Lehtosalo, PR [21590](https://github.com/python/mypy/pull/21590)) +- Fix constructor calls for union-bounded `TypeVar`s (Jingchen Ye, PR [21571](https://github.com/python/mypy/pull/21571)) +- Fix `TypedDict` indexing with literal keys in comprehensions (Jingchen Ye, PR [21556](https://github.com/python/mypy/pull/21556)) +- Correctly handle empty tuple index when unpacked (Ivan Levkivskyi, PR [21545](https://github.com/python/mypy/pull/21545)) +- Support protocol checks for self-types in tuple types (Ivan Levkivskyi, PR [21535](https://github.com/python/mypy/pull/21535)) +- Fix edge cases in variadic tuple subclasses (Ivan Levkivskyi, PR [21518](https://github.com/python/mypy/pull/21518)) +- Special-case constructor for tuple types (Ivan Levkivskyi, PR [21502](https://github.com/python/mypy/pull/21502)) +- Fix false positive "Expected TypedDict key to be string literal" for `Union[TypedDict, dict[K, V]]` (Zakir Jiwani, PR [21511](https://github.com/python/mypy/pull/21511)) +- Use explicit `Never` for type inference (Ivan Levkivskyi, PR [21497](https://github.com/python/mypy/pull/21497)) +- Narrow membership in statically known containers (Shantanu, PR [21461](https://github.com/python/mypy/pull/21461)) +- Improve negative narrowing for membership checks on tuples (Shantanu, PR [21456](https://github.com/python/mypy/pull/21456)) +- Analyze `TypedDict` decorators (Pranav Manglik, PR [21267](https://github.com/python/mypy/pull/21267)) +- Start testing Python 3.15 (Marc Mueller, PR [21439](https://github.com/python/mypy/pull/21439)) +- Improved handling of `NamedTuple`, `TypedDict`, `Enum`, and regular classes nested in functions (Ivan Levkivskyi, PR [21478](https://github.com/python/mypy/pull/21478)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=616424285beccaa76f90e87e1e922b1dc68710ca+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Adam Turner +- alicederyn +- bzoracler +- Colinxu2020 +- georgesittas +- Ivan Levkivskyi +- Jelle Zijlstra +- Jingchen Ye +- Jukka Lehtosalo +- Kevin Kannammalil +- lphuc2250gma +- Marc Mueller +- Pranav Manglik +- Rayan Salhab +- Ryan Heard +- Sai Asish Y +- Shantanu +- sobolevn +- Vaggelis Danias +- Victor Letichevsky +- Zakir Jiwani + +I'd also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 2.1 We’ve just uploaded mypy 2.1.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From b6d93f5a26abcb8554d9938b162ce3eda74b4ea8 Mon Sep 17 00:00:00 2001 From: ygale Date: Tue, 7 Jul 2026 22:54:01 +0300 Subject: [PATCH 28/50] Fix regression in dataclass narrowing for Python >= 3.13 (#21675) Fixes #21635 On Python >= 3.13, `@dataclass` synthesizes a `__replace__(self, ...) -> Self` method to support `copy.replace()`. When mypy tries to narrow a `type[A]` expression via `issubclass(cls, M)` (or `isinstance`) against a second, unrelated dataclass `M`, it builds an ad-hoc `` type to check whether that narrowing is sound (`intersect_instances` in `checker.py`). Building that ad-hoc type runs `check_multiple_inheritance`, which sees `A`'s synthesized `__replace__` returning `A` and `M`'s returning `M`, and flags them as incompatible. That's a false positive: a real subclass of both (e.g. `class C(M, A)`, itself decorated with `@dataclass`) gets its *own* freshly synthesized, mutually compatible `__replace__`. --- mypy/checker.py | 2 +- test-data/unit/check-dataclasses.test | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index d13b927b28f2f..212ddd65d9122 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3133,7 +3133,7 @@ class C(B, A[int]): ... # this is unsafe because... x: A[int] = C() x.foo # ...runtime type is (str) -> None, while static type is (int) -> None """ - if name in ("__init__", "__new__", "__init_subclass__"): + if name in {"__init__", "__new__", "__init_subclass__", "__replace__"}: # __init__ and friends can be incompatible -- it's a special case. return first = base1.names[name] diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 54b3afadc8b32..5ac0318421167 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2609,6 +2609,27 @@ class Y(X): [builtins fixtures/tuple.pyi] +[case testDunderReplaceDoesNotBlockPlainIssubclassNarrowing] +# https://github.com/python/mypy/issues/21635 +# flags: --python-version 3.13 +from dataclasses import dataclass + +@dataclass +class A: ... +@dataclass +class M: ... +@dataclass +class B(A): ... +@dataclass +class C(M, A): ... + +cls: type[A] = C +if issubclass(cls, M): + reveal_type(cls) # N: Revealed type is "type[__main__.]" + n: int = 'foo' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + + [case testFrozenWithFinal] from dataclasses import dataclass from typing import Final From f5163c011078ef66753cdf706b7b2dd14da401ab Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:52:31 -0700 Subject: [PATCH 29/50] Fix variance inference issues caused by dataclass replace (#21694) Fixes #17623. We already special case `__init__` and `__new__`, so adding `__replace__` here feels alright Along with #21675 should hopefully get rid of this surprising edge when moving to Python 3.13 I didn't add tests due to fixture issues --- mypy/subtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 259bb3791deaf..2d97dd534d9f5 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2278,7 +2278,7 @@ def infer_variance(info: TypeInfo, i: int) -> bool: self_type = fill_typevars(info) for member in all_non_object_members(info): # __mypy-replace is an implementation detail of the dataclass plugin - if member in ("__init__", "__new__", "__mypy-replace"): + if member in {"__init__", "__new__", "__replace__", "__mypy-replace"}: continue if isinstance(self_type, TupleType): From 89deb7db255fb414aab70d3fb77b4f2209f2b326 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 11:20:53 +0100 Subject: [PATCH 30/50] [mypyc] Document recent additions to `librt.strings`, such as `ispace` (#21696) Document `is*` functions, `toupper` and `tolower`. --- mypyc/doc/librt_strings.rst | 62 ++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/mypyc/doc/librt_strings.rst b/mypyc/doc/librt_strings.rst index aece015f4f752..6bb4beef6f25e 100644 --- a/mypyc/doc/librt_strings.rst +++ b/mypyc/doc/librt_strings.rst @@ -15,7 +15,7 @@ Thread safety ``BytesWriter`` and ``StringWriter`` objects are unsafe to access from another thread if they are concurrently modified (on free-threaded Python builds). They are optimized for maximal performance, and they aren't fully synchronized. Read-only access from multiple -threads is safe. +threads is safe, as always. BytesWriter ^^^^^^^^^^^ @@ -101,6 +101,9 @@ StringWriter Functions --------- +Reading and writing binary data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + The ``write_*`` and ``read_*`` functions allow interpreting bytes as packed binary data. They can be used as (much) more efficient but lower-level alternatives to the stdlib :mod:`struct` module in compiled code. @@ -210,3 +213,60 @@ This example writes two binary values and reads them afterwards:: Read a 64-bit floating-point value starting at the given index as a big-endian binary value (8 bytes). + +Code point classification +^^^^^^^^^^^^^^^^^^^^^^^^^ + +These functions classify a single Unicode code point, passed as an ``i32`` integer. They are +faster alternatives to calling the corresponding :py:class:`str` methods on a one-character +string in compiled code. A code point is often obtained via ``ord(s[i])``, which is a +fast operation in compiled code when ``s`` has type :py:class:`str`. + +Each function agrees with the matching :py:class:`str` method applied to the one-character +string ``chr(c)``. Out-of-range inputs (negative values, or values past the maximum Unicode +code point ``0x10FFFF``) return ``False``. + +.. function:: isspace(c: i32, /) -> bool + + Return whether the code point is whitespace. Equivalent to ``chr(c).isspace()``. + +.. function:: isalpha(c: i32, /) -> bool + + Return whether the code point is alphabetic. Equivalent to ``chr(c).isalpha()``. + +.. function:: isdigit(c: i32, /) -> bool + + Return whether the code point is a digit. Equivalent to ``chr(c).isdigit()``. + +.. function:: isalnum(c: i32, /) -> bool + + Return whether the code point is alphanumeric. Equivalent to ``chr(c).isalnum()``. + +.. function:: isidentifier(c: i32, /) -> bool + + Return whether the code point is valid as the first character of a Python identifier. + Equivalent to ``chr(c).isidentifier()``. + +Code point case conversion +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These functions convert the case of a single Unicode code point, passed as an ``i32`` integer, +and return the converted code point as an ``i32``. They are faster alternatives to +:py:meth:`str.upper` / :py:meth:`str.lower` on a one-character string in compiled code. + +For the rare code points whose Unicode uppercase or lowercase form has multiple code points +(e.g. U+00DF ``ß`` has the upper case form ``"SS"``, and U+FB01 ``fi`` maps to ``"FI"``), the +input is returned unchanged, so the signature can stay ``i32 -> i32``. Use :py:meth:`str.upper` +/ :py:meth:`str.lower` when full Unicode case conversion matters, or implement the logic to +handle the special cases explicitly. Out-of-range inputs (negative values, or values past the +maximum Unicode code point ``0x10FFFF``) are returned unchanged. + +.. function:: toupper(c: i32, /) -> i32 + + Return the uppercase of the code point, or the input unchanged if the uppercase does not + consist of exactly one code point. + +.. function:: tolower(c: i32, /) -> i32 + + Return the lowercase of the code point, or the input unchanged if the lowercase does not + consist of exactly one code point. From 732802c8b6a3859b8d2160404e6410690c214409 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 13:44:37 +0100 Subject: [PATCH 31/50] [mypyc] Document librt.threading (#21697) --- mypyc/doc/index.rst | 1 + mypyc/doc/librt.rst | 2 ++ mypyc/doc/librt_threading.rst | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 mypyc/doc/librt_threading.rst diff --git a/mypyc/doc/index.rst b/mypyc/doc/index.rst index aacf275de9885..3f74595dd47fb 100644 --- a/mypyc/doc/index.rst +++ b/mypyc/doc/index.rst @@ -35,6 +35,7 @@ generate fast code. librt_base64 librt_random librt_strings + librt_threading librt_time librt_vecs diff --git a/mypyc/doc/librt.rst b/mypyc/doc/librt.rst index 23206f8cfe806..681efe47e7603 100644 --- a/mypyc/doc/librt.rst +++ b/mypyc/doc/librt.rst @@ -30,6 +30,8 @@ Follow submodule links in the table to a detailed description of each submodule. - Pseudorandom number generation * - :doc:`librt.strings ` - String and bytes utilities + * - :doc:`librt.threading ` + - Threading primitives * - :doc:`librt.time ` - Time utilities * - :doc:`librt.vecs ` diff --git a/mypyc/doc/librt_threading.rst b/mypyc/doc/librt_threading.rst new file mode 100644 index 0000000000000..c6cb5c9e84e72 --- /dev/null +++ b/mypyc/doc/librt_threading.rst @@ -0,0 +1,54 @@ +.. _librt-threading: + +librt.threading +=============== + +The ``librt.threading`` module is part of the ``librt`` package on PyPI, and it includes +threading primitives. + +Classes +------- + +Lock +^^^^ + +.. class:: Lock + + A fast mutual exclusion lock. This can be used as a faster replacement for + :py:class:`threading.Lock` in compiled code. + + Like :py:class:`threading.Lock`, a ``Lock`` is *unowned*: it may be released by a thread + other than the one that acquired it, and it doesn't support reentrant (recursive) locking. + A newly created lock is unlocked. + + ``Lock`` can be used as a context manager. The lock is acquired (blocking) on entry and + released on exit, including when the body raises an exception:: + + def example(lock: Lock) -> None: + with lock: + ... # Critical section; the lock is held here. + + ``Lock`` cannot be subclassed. ``Lock`` cannot be used with :py:class:`threading.Condition`. + + .. method:: acquire(blocking: bool = True) -> bool + + Acquire the lock. + + When *blocking* is true (the default), block (if needed) until the lock is available, + acquire it, and return ``True``. When *blocking* is false, acquire the lock only if it + can be done without blocking: return ``True`` if the lock could be acquired, or + ``False`` otherwise (it was already locked by some thread). + + Unlike :py:meth:`threading.Lock.acquire`, there is no *timeout* argument. + + .. method:: release() -> None + + Release the lock, allowing another thread (if any) that is blocked on :meth:`acquire` + to proceed. Since the lock is unowned, it may be released from a thread other than the + one that acquired it. + + Raise :py:exc:`RuntimeError` if the lock is not currently held. + + .. method:: locked() -> bool + + Return ``True`` if the lock is currently held (by any thread). From 7a45d255818e972b293825d701393732b971c4b3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 Jul 2026 14:27:46 +0100 Subject: [PATCH 32/50] Bump librt to 0.13.0 (#21698) This includes `librt.threading.Lock`. --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d2991309e391e..87663f2adde40 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -5,5 +5,5 @@ typing_extensions>=4.14.0; python_version>='3.15' mypy_extensions>=1.0.0 pathspec>=1.0.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.12.0; platform_python_implementation != 'PyPy' +librt>=0.13.0; platform_python_implementation != 'PyPy' ast-serialize>=0.6.0,<1.0.0 diff --git a/pyproject.toml b/pyproject.toml index f166a58086821..aa9432c06e48b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.12.0; platform_python_implementation != 'PyPy'", + "librt>=0.13.0; platform_python_implementation != 'PyPy'", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -58,7 +58,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.12.0; platform_python_implementation != 'PyPy'", + "librt>=0.13.0; platform_python_implementation != 'PyPy'", "ast-serialize>=0.6.0,<1.0.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index e79319fdbeece..73d5966040c4a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ identify==2.6.19 # via pre-commit iniconfig==2.3.0 # via pytest -librt==0.12.0 ; platform_python_implementation != "PyPy" +librt==0.13.0 ; platform_python_implementation != "PyPy" # via -r mypy-requirements.txt lxml==6.1.0 ; python_version < "3.15" # via -r test-requirements.in From e68ecefd1884f7e53e8ac45f0373e60c717e4753 Mon Sep 17 00:00:00 2001 From: Jingchen Ye <11172084+97littleleaf11@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:43:16 +0800 Subject: [PATCH 33/50] Infer Coroutine for unannotated async defs (#21651) Fixes #12776 Add an extra check for unannotated async defs when checking function type. `async def foo()` is now treated like `async def foo() -> Any` --- mypy/checker.py | 15 +++++++++++++- mypy/checker_shared.py | 6 ++++++ mypy/checkexpr.py | 3 +-- mypy/checkmember.py | 7 +++---- test-data/unit/check-async-await.test | 28 +++++++++++++++++++++++++++ test-data/unit/check-flags.test | 2 +- 6 files changed, 53 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 212ddd65d9122..5190e25844029 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -8129,7 +8129,20 @@ def iterable_item_type(self, it: ProperType, context: Context) -> Type: return self.analyze_iterable_item_type_without_expression(it, context)[1] def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type("builtins.function")) + typ = function_type(func, self.named_type("builtins.function")) + if ( + isinstance(func, FuncItem) + and func.is_coroutine + and not func.is_async_generator + and func.type is None + and isinstance(typ, CallableType) + ): + any_type = AnyType(TypeOfAny.special_form) + ret_type = self.named_generic_type( + "typing.Coroutine", [any_type, any_type, typ.ret_type] + ) + return typ.copy_modified(ret_type=ret_type) + return typ def push_type_map(self, type_map: TypeMap, *, from_assignment: bool = True) -> None: if is_unreachable_map(type_map): diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 2b25d3e73a6fe..c1c4cee748b32 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -16,6 +16,7 @@ ArgKind, Context, Expression, + FuncBase, FuncItem, LambdaExpr, MypyFile, @@ -28,6 +29,7 @@ from mypy.plugin import CheckerPluginInterface, Plugin from mypy.types import ( CallableType, + FunctionLike, Instance, LiteralValue, Overloaded, @@ -149,6 +151,10 @@ def expr_checker(self) -> ExpressionCheckerSharedApi: def named_type(self, name: str) -> Instance: raise NotImplementedError + @abstractmethod + def function_type(self, func: FuncBase) -> FunctionLike: + raise NotImplementedError + @abstractmethod def lookup_typeinfo(self, fullname: str) -> TypeInfo: raise NotImplementedError diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 44855f49afaf9..172d44555b946 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -150,7 +150,6 @@ false_only, fixup_partial_type, freeze_all_type_vars, - function_type, get_all_type_vars, get_type_vars, is_literal_type_like, @@ -415,7 +414,7 @@ def analyze_static_reference( if isinstance(node, (Var, Decorator, OverloadedFuncDef)): return node.type or AnyType(TypeOfAny.special_form) elif isinstance(node, FuncDef): - return function_type(node, self.named_type("builtins.function")) + return self.chk.function_type(node) elif isinstance(node, TypeInfo): # Reference to a type object. if node.typeddict_type: diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e75a8ed7a5b03..3ba99d8e8c6b2 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -47,7 +47,6 @@ bind_self, erase_to_bound, freeze_all_type_vars, - function_type, get_all_type_vars, make_simplified_union, supported_self_type, @@ -360,7 +359,7 @@ def analyze_instance_member_access( if mx.is_lvalue and not mx.suppress_errors: mx.msg.cant_assign_to_method(mx.context) if not isinstance(method, OverloadedFuncDef): - signature = function_type(method, mx.named_type("builtins.function")) + signature = mx.chk.function_type(method) else: if method.type is None: # Overloads may be not ready if they are decorated. Handle this in same @@ -1325,7 +1324,7 @@ def analyze_class_attribute_access( return AnyType(TypeOfAny.from_error) else: assert isinstance(node.node, SYMBOL_FUNCBASE_TYPES) - typ = function_type(node.node, mx.named_type("builtins.function")) + typ = mx.chk.function_type(node.node) # Note: if we are accessing class method on class object, the cls argument is bound. # Annotated and/or explicit class methods go through other code paths above, for # unannotated implicit class methods we do this here. @@ -1490,7 +1489,7 @@ def analyze_decorator_or_funcbase_access( """ if isinstance(defn, Decorator): return analyze_var(name, defn.var, itype, mx) - typ = function_type(defn, mx.chk.named_type("builtins.function")) + typ = mx.chk.function_type(defn) if isinstance(defn, (FuncDef, OverloadedFuncDef)) and defn.is_trivial_self: return bind_self_fast(typ, mx.self_type) typ = check_self_arg(typ, mx.self_type, defn.is_class, mx.context, name, mx.msg) diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index e887b4c575528..cd40f2cf29a65 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -839,6 +839,34 @@ def bar() -> None: [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] +[case testUntypedAsyncFunctionAndMethodReturnCoroutine] +# flags: --show-error-codes +async def foo(): + pass + +class C: + async def method(self): + pass + +def f(c: C) -> None: + a = foo() + b = c.method() + reveal_type(foo) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, Any]" + reveal_type(a) # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? \ + # N: Revealed type is "typing.Coroutine[Any, Any, Any]" + foo() # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? + reveal_type(c.method) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, Any]" + reveal_type(b) # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? \ + # N: Revealed type is "typing.Coroutine[Any, Any, Any]" + c.method() # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + [case testAsyncForOutsideCoroutine] async def g(): yield 0 diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index dd4687181ca41..acdee2a10430a 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -59,7 +59,7 @@ async def f(): # E: Function is missing a return type annotation \ # N: Use "-> None" if function does not return a value pass [builtins fixtures/async_await.pyi] -[typing fixtures/typing-medium.pyi] +[typing fixtures/typing-async.pyi] [case testAsyncUnannotatedArgument] # flags: --disallow-untyped-defs From 77ccc7a4d0e15aa0b7f905e348e1c66e37c5d646 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:48:33 -0700 Subject: [PATCH 34/50] Fix custom equality handling for membership narrowing in static containers (#21706) This was an issue in the new feature #21461 In narrow_type_by_identity_equality we check for custom equality before coercing to literals, and this was done the wrong way round here Fixes #21703 --- mypy/checker.py | 10 +++++++--- test-data/unit/check-narrowing.test | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5190e25844029..3a98505cc87ec 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6858,9 +6858,13 @@ def comparison_type_narrowing_helper(self, node: ComparisonExpr) -> tuple[TypeMa for known_item in container_item_types: # Match the should_coerce_literals logic from narrow_type_by_identity_equality p_known_item = get_proper_type(known_item) - if is_literal_type_like(p_known_item) or ( - isinstance(p_known_item, Instance) and p_known_item.type.is_enum - ): + if ( + is_literal_type_like(p_known_item) + or ( + isinstance(p_known_item, Instance) + and p_known_item.type.is_enum + ) + ) and not has_custom_eq_checks(p_known_item): known_item = coerce_to_literal(known_item) if_map, else_map = self.narrow_type_by_identity_equality( "==", diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 7bab7baa6cebd..29f4cd47929d9 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -3335,6 +3335,24 @@ def narrow_dict(x: Literal['a', 'b', 'c'], t: dict[Literal['a', 'b'], int]): [builtins fixtures/primitives.pyi] +[case testNarrowCustomEqEnumInLiteralContainer] +# flags: --strict-equality --warn-unreachable +# https://github.com/python/mypy/issues/21703 +from enum import Enum + +class E(Enum): + foo = 1 + bar = 2 + + def __eq__(self, other: object) -> bool: + return True + +def f(x: int) -> None: + if x in [E.foo, E.bar]: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + + [case testNarrowingLiteralInLiteralContainer] # flags: --strict-equality --warn-unreachable from typing import Literal From af2bc0f3cc7f2f129f0c11294158d0c292692c3d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:46:12 -0700 Subject: [PATCH 35/50] Sync typeshed (#21707) Source commit: https://github.com/python/typeshed/commit/f76037a1eb3923c67a8bc0e302ee9c016ffb3431 --- mypy/typeshed/stdlib/builtins.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 7b659e5476d59..8497f9e2e1037 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1277,7 +1277,6 @@ if sys.version_info >= (3, 15): cls: type[frozendict[str, _VT]], iterable: Iterable[tuple[str, _VT]], /, **kwargs: _VT ) -> frozendict[str, _VT]: ... - def __init__(self) -> None: ... def copy(self) -> frozendict[_KT, _VT]: ... @overload From cbcb51add3094ec91b29cdd4c624943bf251b63f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 8 Jul 2026 23:55:18 -0700 Subject: [PATCH 36/50] Narrow for frozendict membership check (#21709) --- mypy/checker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/checker.py b/mypy/checker.py index 3a98505cc87ec..612d2face5b8a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -8892,6 +8892,7 @@ def builtin_item_type(tp: Type) -> Type | None: "builtins.list", "builtins.tuple", "builtins.dict", + "builtins.frozendict", "builtins.set", "builtins.frozenset", "_collections_abc.dict_keys", From b5be217392b9b2771d1764066b9d600bf93ce7a8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Jul 2026 15:35:40 +0100 Subject: [PATCH 37/50] [mypyc] Update free threading Python compatibility docs (#21711) We've made a bunch of compatibility improvements on free threaded builds recently. --- mypyc/doc/differences_from_python.rst | 44 +++++++++++++++------------ mypyc/doc/librt_threading.rst | 2 ++ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index 414476723125b..a139fab3aa2fb 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -265,25 +265,31 @@ also be modified. Free threading -------------- -Mypyc has basic support for free threading, but it doesn't provide the -same memory safety guarantees as Python in compiled modules, since -in current Python versions this would cause an unacceptable performance -impact. - -The exact details of the memory safety in the presence of data races -are likely to evolve in the future. Currently, compiled code must -ensure that proper synchronization is used to prevent data races. In -particular, these operations require explicit synchronization, such as -via ``threading.Lock``, if there is a possibility of data races -(the list is not exhaustive): - -* Reads or writes of non-final instance data attributes of native - classes. -* List item access or iteration using a ``list`` static type (using - ``Sequence`` or ``MutableSequence`` as the type ensure correct - implicit synchronization). -* Dict item access using a static ``dict`` type (using ``Mapping`` - or ``MutableMapping`` ensures correct implicit synchronization). +Mypyc supports free threading, but it doesn't provide the exact +memory safety guarantees as Python in compiled modules under +free threading when there are race conditions. + +Additionally, optimized primitive operations in compiled code may have +different atomicity properties compared to CPython. Use explicit +synchronization if code depends on operations being atomic. This is +already the recommended approach for normal Python code. + +Currently, compiled code must ensure that proper synchronization is +used to prevent data races involving non-final attributes in native +classes, unless the attribute has a value type such as ``bool``, +``float`` or ``i64``. You can use explicit +synchronization, such as via +:ref:`librt.threading.Lock ` (or +:py:class:`threading.Lock`, which is less efficient than +``librt.threading.Lock``) if there is a possibility of such a data +race. + +.. note:: + + We are working on improving memory safety in free-threading + builds of Python, and hope to make all normal Python features + memory safe, while providing more efficient but less safe + opt-in, non-standard features. As libraries often won't be able to control the concurrent access by user code, we recommend that modules document that multi-threaded diff --git a/mypyc/doc/librt_threading.rst b/mypyc/doc/librt_threading.rst index c6cb5c9e84e72..0af795bc0daab 100644 --- a/mypyc/doc/librt_threading.rst +++ b/mypyc/doc/librt_threading.rst @@ -9,6 +9,8 @@ threading primitives. Classes ------- +.. _librt-threading-lock: + Lock ^^^^ From 24c237d85b48f618e655ffff1dc0f19089d9b599 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Jul 2026 17:29:52 +0100 Subject: [PATCH 38/50] [mypyc] Improve documentation of Final (#21713) The semantics of Final were changed recently: mention that final instance attributes can't be rebound. Also add more detail and a new example. --- mypyc/doc/differences_from_python.rst | 50 +++++++++++++++++++++------ 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index a139fab3aa2fb..c65c330edbef3 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -237,28 +237,56 @@ nested function calls, typically due to out-of-control recursion. Final values ------------ -Compiled code replaces a reference to an attribute declared ``Final`` with -the value of the attribute computed at compile time. This is an example of -:ref:`early binding `. Example:: +Mypy treats variables and attributes defined as ``Final`` specially. +For instance attributes of native classes, mypyc prevents reassignment +of these attributes. + +Mypyc replaces references to a variable or attribute declared ``Final`` +with the value of the attribute computed at compile time, when it can +be determined during compilation. Example:: MAX: Final = 100 def limit_to_max(x: int) -> int: - if x > MAX: - return MAX - return x + if x > MAX: + return MAX + return x The two references to ``MAX`` don't involve any module namespace lookups, and are equivalent to this code:: def limit_to_max(x: int) -> int: - if x > 100: - return 100 - return x + if x > 100: + return 100 + return x When run as interpreted, the first example will execute slower due to -the extra namespace lookups. In interpreted code final attributes can -also be modified. +the extra namespace lookups. + +For a final class attribute or global variable whose value can't be +determined during compilation, mypyc defines a hidden native variable +that stores the value when the initialization assignment is evaluated. +Redefining the attribute using ``setattr`` or direct namespace access +has no effect for mypyc-compiled code (in the same compilation unit). + +Final global variables and class attributes are faster to read in +compiled code, since they don't have to be looked up from a namespace +dictionary. Thus using a ``Final`` object that holds mutable state in an +attribute tends to be faster than a plain global variable in compiled code:: + + class Counter: + def __init__(self) -> None: + self.value = 0 + + x: Final = Counter() + y = 0 + + def inc() -> None: + # Faster: no Python namespace lookup + x.value += 1 + # Slower: access through globals() namespace dictionary + global y + y += 1 .. _free-threading: From 3d75cdb09f0928fa8b83e5ef03572ed878ac8d09 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Jul 2026 19:24:21 +0100 Subject: [PATCH 39/50] [mypyc] Borrow final attributes more aggressively (#21702) Final instance attributes of native classes can't be rebound after initialization, so we can borrow them more aggressively than regular attributes, as long as the base object is kept alive for the duration of the borrowing. This allows skipping some incref/decref operations. Many workloads spend a significant fraction of CPU on incref/decref, and these operations are quite a bit more expensive on free-threaded builds, so the potential performance impact is significant especially on free-threaded builds. For example, the attribute `x` can only be safely borrowed if it's Final, since otherwise `foo` could assign to the attribute and free the old value: ```py def func(o: C) -> None: foo(o.x) ``` There are some subtleties in the implementation. Here are the main things that required extra care: * We don't allow borrowed values to escape from conditionally executed code paths (e.g. conditional expressions, comprehensions). * If a local variable can be modified with an assignment expression, we restrict borrowing based on that variable. * We can only borrow for a longer duration if the attribute value doesn't depend on a subexpression with a smaller borrow scope. * Lambda expressions generate a complete separate expression and borrowing scope. I used coding agent assist, especially for tests, but created the implementation is short, individually reviewed increments. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- mypyc/common.py | 8 + mypyc/ir/ops.py | 5 +- mypyc/irbuild/builder.py | 173 ++++++- mypyc/irbuild/expression.py | 65 ++- mypyc/irbuild/for_helpers.py | 11 +- mypyc/irbuild/ll_builder.py | 116 ++++- mypyc/irbuild/statement.py | 23 +- mypyc/irbuild/vec.py | 4 +- mypyc/test-data/irbuild-final.test | 252 ++++++++++ mypyc/test-data/refcount.test | 714 ++++++++++++++++++++++++++++ mypyc/test-data/run-async.test | 93 ++++ mypyc/test-data/run-classes.test | 73 +++ mypyc/test-data/run-generators.test | 43 ++ mypyc/test/test_irbuild.py | 1 + 15 files changed, 1536 insertions(+), 47 deletions(-) create mode 100644 mypyc/test-data/irbuild-final.test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 361290e55a1d1..548b568621a70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: codespell args: - - --ignore-words-list=HAX,Nam,ccompiler,ot,statics,whet,zar + - --ignore-words-list=HAX,Nam,ccompiler,keep-alives,ot,statics,whet,zar exclude: ^(mypy/test/|mypy/typeshed/|mypyc/test-data/|test-data/).+$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 diff --git a/mypyc/common.py b/mypyc/common.py index 382d640a84083..fa34647c5c729 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -68,6 +68,14 @@ BITMAP_TYPE: Final = "uint32_t" BITMAP_BITS: Final = 32 +# Constant for keeping a (often borrowed) op alive for a short time (a few subexpressions, +# no arbitrary computation or memory allocations), until flush_keep_alives() is called. +KEEP_ALIVE_SHORT_LIVED: Final = 0 +# Keep value alive longer, up to until the current top-level expression has been fully +# evaluated. This allows keeping alive values across function calls and arbitrary +# computation. Note that some expressions (e.g. lambdas), restrict the scope of borrowing. +KEEP_ALIVE_WHOLE_EXPRESSION: Final = 1 + # Runtime C library files that are always included (some ops may bring # extra dependencies via mypyc.ir.deps.SourceDep or mypyc.ir.deps.HeaderDep) RUNTIME_C_FILES: Final = [ diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 4bc7671b82082..14e12559be1eb 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -31,7 +31,7 @@ class to enable the new behavior. Sometimes adding a new abstract from mypy_extensions import trait -from mypyc.common import PROPSET_PREFIX +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, PROPSET_PREFIX from mypyc.ir.deps import Dependency from mypyc.ir.rtypes import ( RArray, @@ -898,6 +898,7 @@ def __init__( *, borrow: bool = False, allow_error_value: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> None: super().__init__(line) self.obj = obj @@ -912,6 +913,8 @@ def __init__( elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted + # How long a borrowed result of this op stays valid (a KEEP_ALIVE_* constant). + self.borrow_scope = borrow_scope def sources(self) -> list[Value]: return [self.obj] diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 56a3f944fe77f..d8ae71e33bf59 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -20,12 +20,17 @@ TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, ArgKind, + AssignmentExpr, + AwaitExpr, CallExpr, Decorator, + DictionaryComprehension, Expression, FuncDef, + GeneratorExpr, IndexExpr, IntExpr, + LambdaExpr, Lvalue, MemberExpr, MypyFile, @@ -41,7 +46,10 @@ TypeInfo, TypeParam, Var, + YieldExpr, + YieldFromExpr, ) +from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, DeletedType, @@ -63,6 +71,8 @@ EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -80,6 +90,7 @@ BasicBlock, Branch, Call, + Cast, ComparisonOp, GetAttr, InitStatic, @@ -284,6 +295,20 @@ def __init__( self.imports: dict[str, None] = {} self.can_borrow = False + self.expression_depth = 0 + # Symbols (local vars) reassigned via a walrus expression within the current + # top-level expression. Used to avoid borrowing an attribute over the whole + # expression when the borrow root could be rebound (and thus freed) partway. + self.reassigned_in_expr: set[SymbolNode] = set() + # Whether the current top-level expression contains a suspension point + # (await, yield or yield from). A whole-expression borrow can't span such a + # point, since the borrowed value (and its root) live in registers that are + # not spilled into the generator environment across the suspend. + self.expr_has_suspend = False + # Saved expression state for enclosing functions (see enter()/leave()). + self.expression_depth_stack: list[int] = [] + self.reassigned_in_expr_stack: list[set[SymbolNode]] = [] + self.expr_has_suspend_stack: list[bool] = [] # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -315,6 +340,10 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V """ with self.catch_errors(node.line): if isinstance(node, Expression): + self.expression_depth += 1 + if self.expression_depth == 1: + self.reassigned_in_expr = find_walrus_targets(node) + self.expr_has_suspend = expr_has_suspend(node) old_can_borrow = self.can_borrow self.can_borrow = can_borrow try: @@ -329,6 +358,11 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V self.can_borrow = old_can_borrow if not can_borrow: self.flush_keep_alives(node.line) + self.expression_depth -= 1 + if self.expression_depth == 0: + self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) + self.reassigned_in_expr = set() + self.expr_has_suspend = False return res else: try: @@ -337,8 +371,8 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V pass return None - def flush_keep_alives(self, line: int) -> None: - self.builder.flush_keep_alives(line) + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + self.builder.flush_keep_alives(line, scope=scope) # Pass through methods for the most common low-level builder ops, for convenience. @@ -1192,7 +1226,9 @@ def is_synthetic_type(self, typ: TypeInfo) -> bool: return typ.is_named_tuple or typ.is_newtype or typ.typeddict_type is not None def get_final_ref(self, expr: MemberExpr) -> tuple[str, Var, bool] | None: - """Check if `expr` is a final attribute. + """Check if `expr` is a final class or module attribute. + + Return False for instance attributes. This needs to be done differently for class and module attributes to correctly determine fully qualified name. Return a tuple that consists of @@ -1341,6 +1377,15 @@ def enter(self, fn_info: FuncInfo | str = "", *, ret_type: RType = none_rprimiti self.fn_info = fn_info self.fn_infos.append(self.fn_info) self.ret_types.append(ret_type) + # A function body is its own top-level expression context, even when the + # function (e.g. a lambda) is being generated in the middle of an outer + # expression. Save the outer expression state and start fresh. + self.expression_depth_stack.append(self.expression_depth) + self.reassigned_in_expr_stack.append(self.reassigned_in_expr) + self.expr_has_suspend_stack.append(self.expr_has_suspend) + self.expression_depth = 0 + self.reassigned_in_expr = set() + self.expr_has_suspend = False if fn_info.is_generator: self.nonlocal_control.append(GeneratorNonlocalControl()) else: @@ -1354,6 +1399,9 @@ def leave(self) -> tuple[list[Register], list[RuntimeArg], list[BasicBlock], RTy ret_type = self.ret_types.pop() fn_info = self.fn_infos.pop() self.nonlocal_control.pop() + self.expression_depth = self.expression_depth_stack.pop() + self.reassigned_in_expr = self.reassigned_in_expr_stack.pop() + self.expr_has_suspend = self.expr_has_suspend_stack.pop() self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] return builder.args, runtime_args, builder.blocks, ret_type, fn_info @@ -1389,6 +1437,27 @@ def enter_scope(self, fn_info: FuncInfo) -> Iterator[None]: self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] + @contextmanager + def enter_borrow_scope(self, line: int) -> Iterator[None]: + """Enter new borrow scope from which borrows can't leak to outer expressions. + + This is a borrow region (see LowLevelIRBuilder.borrow_region) that also + resets the per-expression borrowing heuristic state, since the body forms + its own top-level expression context (e.g. a comprehension iteration or a + lambda body). + """ + old_expression_depth = self.expression_depth + old_reassigned_in_expr = self.reassigned_in_expr + old_expr_has_suspend = self.expr_has_suspend + self.expression_depth = 0 + try: + with self.builder.borrow_region(line): + yield + finally: + self.expression_depth = old_expression_depth + self.reassigned_in_expr = old_reassigned_in_expr + self.expr_has_suspend = old_expr_has_suspend + @contextmanager def enter_method( self, @@ -1581,6 +1650,32 @@ def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: return expr.name in ir.final_attributes return False + def root_is_reassigned(self, v: Value) -> bool: + """Is the root local variable a borrow chain 'v' reads from reassigned this expression? + + A whole-expression borrow of an attribute keeps the borrow root alive only + via the register holding it. If that register belongs to a local variable + that is rebound (via a walrus assignment) during the same top-level + expression, the old value may be freed while the borrow is still live. + """ + if not self.reassigned_in_expr: + return False + # Peel borrowed links back to the root value the chain reads from. + while True: + if isinstance(v, GetAttr) and v.is_borrowed: + v = v.obj + elif isinstance(v, Cast) and v.is_borrowed: + v = v.src + else: + break + if not isinstance(v, Register): + return False + for symbol in self.reassigned_in_expr: + target = self.symtables[-1].get(symbol) + if isinstance(target, AssignmentTargetRegister) and target.register is v: + return True + return False + def mark_block_unreachable(self) -> None: """Mark statements in the innermost block being processed as unreachable. @@ -1695,6 +1790,78 @@ def get_call_target_fullname(ref: RefExpr) -> str: return ref.fullname +class WalrusTargetCollector(TraverserVisitor): + """Collect the symbols assigned to by walrus expressions in a subtree.""" + + def __init__(self) -> None: + self.targets: set[SymbolNode] = set() + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + if o.target.node is not None: + self.targets.add(o.target.node) + super().visit_assignment_expr(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own expression context, so don't descend into it. + pass + + +def find_walrus_targets(expr: Expression) -> set[SymbolNode]: + """Return the symbols reassigned via a walrus expression within 'expr'. + + Walrus (':=') is the only way to rebind a variable in the middle of evaluating + an expression, so this is the complete set of in-expression reassignments. + """ + collector = WalrusTargetCollector() + expr.accept(collector) + return collector.targets + + +class SuspendDetector(TraverserVisitor): + """Detect await/yield/yield from expressions in a subtree.""" + + def __init__(self) -> None: + self.found = False + + def visit_await_expr(self, o: AwaitExpr) -> None: + self.found = True + + def visit_yield_expr(self, o: YieldExpr) -> None: + self.found = True + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + self.found = True + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + # An 'async for' clause suspends via an implicit await on __anext__ that + # isn't represented as an AwaitExpr node in the AST (list/set comprehensions + # delegate to a GeneratorExpr, so they are covered here too). + if any(o.is_async): + self.found = True + super().visit_generator_expr(o) + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + if any(o.is_async): + self.found = True + super().visit_dictionary_comprehension(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own function (and suspension) context. + pass + + +def expr_has_suspend(expr: Expression) -> bool: + """Does evaluating 'expr' involve a suspension point (await/yield/yield from)? + + A whole-expression borrow can't safely span a suspension point, since the + borrowed value and its borrow root are held in registers that aren't spilled + into the generator environment across the suspend. + """ + detector = SuspendDetector() + expr.accept(detector) + return detector.found + + def create_type_params( builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int ) -> list[Value]: diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index b99e8161c08c5..91e1e55d8da26 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -58,7 +58,12 @@ TypeType, get_proper_type, ) -from mypyc.common import IS_FREE_THREADED, MAX_SHORT_INT +from mypyc.common import ( + IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, + MAX_SHORT_INT, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.ir.ops import ( @@ -66,11 +71,14 @@ Assign, BasicBlock, CallC, + Cast, ComparisonOp, + GetAttr, Integer, LoadAddress, LoadLiteral, PrimitiveDescription, + PrimitiveOp, RaiseStandardError, Register, TupleGet, @@ -252,7 +260,7 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if expr.fullname in ("typing.TYPE_CHECKING", "typing_extensions.TYPE_CHECKING"): return builder.false(expr.line) - # First check if this is maybe a final attribute. + # First check if this is maybe a final class/module attribute. final = builder.get_final_ref(expr) if final is not None: fullname, final_var, native = final @@ -305,8 +313,43 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) - borrow = can_borrow and builder.can_borrow - return builder.builder.get_attr(obj, expr.name, rtype, expr.line, borrow=borrow) + is_final = builder.is_final_native_attr_ref(expr) + scope = KEEP_ALIVE_SHORT_LIVED + if ( + is_final + and builder.expression_depth > 1 + and value_borrow_scope(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION + # Don't borrow across the whole expression if the borrow root can be + # rebound via a walrus assignment + and not builder.root_is_reassigned(obj) + # Don't borrow across a suspension point (await/yield/yield from), since + # the borrow would not survive the suspend (registers aren't spilled). + and not builder.expr_has_suspend + ): + scope = KEEP_ALIVE_WHOLE_EXPRESSION + borrow = (can_borrow and builder.can_borrow) or scope == KEEP_ALIVE_WHOLE_EXPRESSION + return builder.builder.get_attr( + obj, expr.name, rtype, expr.line, borrow=borrow, borrow_scope=scope + ) + + +def value_borrow_scope(builder: IRBuilder, v: Value) -> int: + """Compute how long an existing borrowed value can safely be kept alive. + + Returns a KEEP_ALIVE_* constant (or a large value if 'v' is not borrowed and + thus has no borrowing constraint). + """ + if isinstance(v, GetAttr) and v.is_borrowed: + return min(v.borrow_scope, value_borrow_scope(builder, v.obj)) + elif isinstance(v, Cast) and v.is_borrowed: + # A cast just propagates the value (when successful), so the scope is + # determined by the source. + return value_borrow_scope(builder, v.src) + elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: + # Values borrowed from a C function (e.g. a borrowed list/vec item) may be + # invalidated by arbitrary computation, so they are only short-lived. + return KEEP_ALIVE_SHORT_LIVED + return 999 def check_instance_attribute_access_through_class( @@ -875,15 +918,17 @@ def transform_conditional_expr(builder: IRBuilder, expr: ConditionalExpr) -> Val target = Register(expr_type) builder.activate_block(if_body) - true_value = builder.accept(expr.if_expr) - true_value = builder.coerce(true_value, expr_type, expr.line) - builder.add(Assign(target, true_value, expr.line)) + with builder.builder.borrow_region(expr.line): + true_value = builder.accept(expr.if_expr) + true_value = builder.coerce(true_value, expr_type, expr.line) + builder.add(Assign(target, true_value, expr.line)) builder.goto(next_block) builder.activate_block(else_body) - false_value = builder.accept(expr.else_expr) - false_value = builder.coerce(false_value, expr_type, expr.line) - builder.add(Assign(target, false_value, expr.line)) + with builder.builder.borrow_region(expr.line): + false_value = builder.accept(expr.else_expr) + false_value = builder.coerce(false_value, expr_type, expr.line) + builder.add(Assign(target, false_value, expr.line)) builder.goto(next_block) builder.activate_block(next_block) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index fc539a1fc090b..5a36dcb80ff7d 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -293,7 +293,8 @@ def sequence_from_generator_preallocate_helper( target_op = empty_op_llbuilder(length, line) def set_item(item_index: Value) -> None: - e = builder.accept(gen.left_expr) + with builder.enter_borrow_scope(line): + e = builder.accept(gen.left_expr) set_item_op(target_op, item_index, e, line) for_loop_helper_with_index( @@ -446,9 +447,10 @@ def loop_contents( remaining_loop_params: the parameters for any further nested loops; if it's empty we'll instead evaluate the "gen_inner_stmts" function """ - # Check conditions, in order, short circuiting them. + # Check conditions, in order, short-circuiting them. for cond in conds: - cond_val = builder.accept(cond) + with builder.enter_borrow_scope(line): + cond_val = builder.accept(cond) cont_block, rest_block = BasicBlock(), BasicBlock() # If the condition is true we'll skip the continue. builder.add_bool_branch(cond_val, rest_block, cont_block) @@ -462,7 +464,8 @@ def loop_contents( else: # We finally reached the actual body of the generator. # Generate the IR for the inner loop body. - gen_inner_stmts() + with builder.enter_borrow_scope(line): + gen_inner_stmts() handle_loop(loop_params) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c0ad1cf1f8264..aafdb73fed84e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -7,7 +7,8 @@ from __future__ import annotations import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from typing import Final, TypeGuard, cast from mypy.argmap import map_actuals_to_formals @@ -19,6 +20,7 @@ FAST_ISINSTANCE_MAX_SUBCLASSES, FAST_PREFIX, IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -250,6 +252,15 @@ BOOL_BINARY_OPS: Final = {"&", "&=", "|", "|=", "^", "^=", "==", "!=", "<", "<=", ">", ">="} +class PendingKeepAlive: + __slots__ = ("value", "scope", "seq") + + def __init__(self, value: Value, scope: int, seq: int) -> None: + self.value = value + self.scope = scope + self.seq = seq + + class LowLevelIRBuilder: """A "low-level" IR builder class. @@ -276,7 +287,10 @@ def __init__(self, errors: Errors | None, options: CompilerOptions) -> None: self.error_handlers: list[BasicBlock | None] = [None] # Values that we need to keep alive as long as we have borrowed # temporaries. Use flush_keep_alives() to mark the end of the live range. - self.keep_alives: list[Value] = [] + # The second value is the scope/duration of keep alive (KEEP_ALIVE_* constant). + # Different values must be kept alive for different durations. + self.keep_alives: list[PendingKeepAlive] = [] + self.next_keep_alive_seq = 0 def set_module(self, module_name: str, module_path: str) -> None: """Set the name and path of the current module.""" @@ -367,10 +381,55 @@ def self(self) -> Register: """ return self.args[0] - def flush_keep_alives(self, line: int) -> None: - if self.keep_alives: - self.add(KeepAlive(self.keep_alives.copy(), line)) - self.keep_alives = [] + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + if any(entry.scope == scope for entry in self.keep_alives): + self.add( + KeepAlive( + [entry.value for entry in self.keep_alives if entry.scope == scope], line + ) + ) + self.keep_alives = [entry for entry in self.keep_alives if entry.scope != scope] + + def keep_alive_checkpoint(self) -> int: + return self.next_keep_alive_seq + + def flush_keep_alives_since(self, line: int, checkpoint: int) -> None: + """Flush keep-alives added after 'checkpoint' without touching earlier ones.""" + new_keep_alives = [entry for entry in self.keep_alives if entry.seq >= checkpoint] + if new_keep_alives: + self.add(KeepAlive([entry.value for entry in new_keep_alives], line)) + self.keep_alives = [entry for entry in self.keep_alives if entry.seq < checkpoint] + + @contextmanager + def borrow_region(self, line: int) -> Iterator[None]: + """Confine borrows created in this region: flush them when leaving it. + + Keep-alives added inside the region (regardless of their duration scope) + are flushed on exit, so borrows can't leak past a lexical boundary such as + a conditional branch, an 'and'/'or' branch or a comprehension iteration. + """ + checkpoint = self.keep_alive_checkpoint() + yield + self.flush_keep_alives_since(line, checkpoint) + + def add_keep_alive(self, value: Value, scope: int) -> None: + self.keep_alives.append(PendingKeepAlive(value, scope, self.next_keep_alive_seq)) + self.next_keep_alive_seq += 1 + + def keep_borrow_source_alive(self, value: Value, borrow_scope: int) -> None: + """Ensure the source of borrowed value will be alive for given borrow scope. + + A borrow keeps reading from 'value', so 'value' must stay valid for the + borrow's whole scope. When 'value' is itself a borrowed cast it holds no + reference of its own (and coerce() only kept its source alive short-term), + so we follow the cast to the underlying owned value and keep *that* alive + for the requested scope too. Otherwise, a longer-lived borrow through a cast + (e.g. a Final attribute read via cast(T, make())) could read freed memory. + """ + self.add_keep_alive(value, borrow_scope) + while isinstance(value, Cast) and value.is_borrowed: + value = value.src + self.add_keep_alive(value, borrow_scope) def debug_print(self, toprint: str | Value) -> None: if isinstance(toprint, str): @@ -400,7 +459,7 @@ def unbox_or_cast( return self.add(Unbox(src, target_type, line)) else: if can_borrow: - self.keep_alives.append(src) + self.add_keep_alive(src, KEEP_ALIVE_SHORT_LIVED) return self.add(Cast(src, target_type, line, borrow=can_borrow, unchecked=unchecked)) def coerce( @@ -805,19 +864,32 @@ def coerce_nullable(self, src: Value, target_type: RType, line: int) -> Value: # Attribute access def get_attr( - self, obj: Value, attr: str, result_type: RType, line: int, *, borrow: bool = False + self, + obj: Value, + attr: str, + result_type: RType, + line: int, + *, + borrow: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> Value: - """Get a native or Python attribute of an object.""" + """Get a native or Python attribute of an object. + + If the result is borrowed, borrow_scope controls how long it stays valid + (a KEEP_ALIVE_* constant). The caller is responsible for choosing a scope + that is actually safe (e.g. downgrading to short-lived if the borrow root + may be rebound during the borrow's live range). + """ if ( isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr) ): - op = GetAttr(obj, attr, line, borrow=borrow) + op = GetAttr(obj, attr, line, borrow=borrow, borrow_scope=borrow_scope) # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.keep_alives.append(obj) + self.keep_borrow_source_alive(obj, borrow_scope) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) @@ -2112,18 +2184,20 @@ def shortcircuit_helper( # it is the right side if the left is false. true_body, false_body = (right_body, left_body) if op == "and" else (left_body, right_body) - left_value = left() - self.add_bool_branch(left_value, true_body, false_body) + with self.borrow_region(line): + left_value = left() + self.add_bool_branch(left_value, true_body, false_body) - self.activate_block(left_body) - left_coerced = self.coerce(left_value, expr_type, line) - self.add(Assign(target, left_coerced, line)) + self.activate_block(left_body) + left_coerced = self.coerce(left_value, expr_type, line) + self.add(Assign(target, left_coerced, line)) self.goto(next_block) self.activate_block(right_body) - right_value = right() - right_coerced = self.coerce(right_value, expr_type, line) - self.add(Assign(target, right_coerced, line)) + with self.borrow_region(line): + right_value = right() + right_coerced = self.coerce(right_value, expr_type, line) + self.add(Assign(target, right_coerced, line)) self.goto(next_block) self.activate_block(next_block) @@ -2281,7 +2355,7 @@ def call_c( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE @@ -2402,7 +2476,7 @@ def primitive_op( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index e90aa089305f2..21f47b190c302 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,7 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import TEMP_ATTR_NAME +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -86,7 +86,13 @@ object_rprimitive, ) from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional -from mypyc.irbuild.builder import IRBuilder, create_type_params, int_borrow_friendly_op +from mypyc.irbuild.builder import ( + IRBuilder, + create_type_params, + expr_has_suspend, + find_walrus_targets, + int_borrow_friendly_op, +) from mypyc.irbuild.for_helpers import for_loop_helper from mypyc.irbuild.generator import add_raise_exception_blocks_to_generator_class from mypyc.irbuild.nonlocalcontrol import ( @@ -161,10 +167,17 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: if isinstance(stmt.expr, StrExpr): # Docstring. Ignore return - # ExpressionStmts do not need to be coerced like other Expressions, so we shouldn't - # call builder.accept here. + # ExpressionStmts do not need to be coerced like other Expressions, so + # we shouldn't call builder.accept here. + builder.expression_depth += 1 + builder.reassigned_in_expr = find_walrus_targets(stmt.expr) + builder.expr_has_suspend = expr_has_suspend(stmt.expr) stmt.expr.accept(builder.visitor) - builder.flush_keep_alives(stmt.line) + builder.expression_depth -= 1 + builder.reassigned_in_expr = set() + builder.expr_has_suspend = False + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 38615ebe16026..8dffc12a22130 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final -from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE +from mypyc.common import IS_32_BIT_PLATFORM, KEEP_ALIVE_SHORT_LIVED, PLATFORM_SIZE from mypyc.ir.ops import ( ERR_MAGIC, Assign, @@ -351,7 +351,7 @@ def vec_get_item_unsafe_lower( vtype = base.type item_addr = vec_item_ptr(builder, base, index) result = vec_load_mem_item(builder, item_addr, vtype.item_type, can_borrow=can_borrow) - builder.keep_alives.append(base) + builder.add_keep_alive(base, KEEP_ALIVE_SHORT_LIVED) return result diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test new file mode 100644 index 0000000000000..0ede988a43206 --- /dev/null +++ b/mypyc/test-data/irbuild-final.test @@ -0,0 +1,252 @@ +[case testFinalAttrBorrowed] +from typing import Final + +class C: + def __init__(self, x: str) -> None: + self.f: Final = x + self.x = x + +def f(s: str) -> None: pass + +def calls(c: C) -> None: + f(c.f) + f(c.x) + c.f.startswith("a") + +def ret(c: C) -> str: + return c.f +[out] +def C.__init__(self, x): + self :: __main__.C + x :: str +L0: + self.f = x + self.x = x + return 1 +def f(s): + s :: str +L0: + return 1 +def calls(c): + c :: __main__.C + r0 :: str + r1 :: None + r2 :: str + r3 :: None + r4, r5 :: str + r6 :: i32 + r7 :: bool +L0: + r0 = borrow c.f + r1 = f(r0) + keep_alive c + r2 = c.x + r3 = f(r2) + r4 = borrow c.f + r5 = 'a' + r6 = CPyStr_Startswith(r4, r5) + r7 = truncate r6: i32 to builtins.bool + keep_alive c + return 1 +def ret(c): + c :: __main__.C + r0 :: str +L0: + r0 = c.f + return r0 + +[case testInheritedFinalAttrBorrowed] +from typing import Final + +class Base: + x: str + + def __init__(self, s: str) -> None: + self.s: Final = s + +class Deriv(Base): + def __init__(self, n: int) -> None: + self.n: Final = n + +def f(b: Base) -> str: + return b.s + "a" + +def g(d: Deriv) -> str: + return d.s * d.n + +def h(d: Deriv) -> str: + return d.x + "a" +[out] +def Base.__init__(self, s): + self :: __main__.Base + s :: str +L0: + self.s = s + return 1 +def Deriv.__init__(self, n): + self :: __main__.Deriv + n :: int +L0: + self.n = n + return 1 +def f(b): + b :: __main__.Base + r0, r1, r2 :: str +L0: + r0 = borrow b.s + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + keep_alive b + return r2 +def g(d): + d :: __main__.Deriv + r0 :: str + r1 :: int + r2 :: str +L0: + r0 = borrow d.s + r1 = borrow d.n + r2 = CPyStr_Multiply(r0, r1) + keep_alive d, d + return r2 +def h(d): + d :: __main__.Deriv + r0, r1, r2 :: str +L0: + r0 = d.x + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + return r2 + +[case testFinalAttrExpressionKinds] +from typing import Final + +class C: + def __init__(self, s: str, n: int, l: list[str]) -> None: + self.f: Final = s + self.n: Final = n + self.l: Final = l + self.s = s + +def f() -> C: + return C("", 0, []) + +def assign(c: C) -> None: + a = c.f + c.s = c.f + +def indexing(c: C) -> str: + return c.l[c.n] + +def call_eq(c: C) -> bool: + return f().f == c.f +[out] +def C.__init__(self, s, n, l): + self :: __main__.C + s :: str + n :: int + l :: list +L0: + self.f = s + self.n = n + self.l = l + self.s = s + return 1 +def f(): + r0 :: str + r1 :: list + r2 :: __main__.C +L0: + r0 = '' + r1 = PyList_New(0) + r2 = C(r0, 0, r1) + return r2 +def assign(c): + c :: __main__.C + r0, a, r1 :: str + r2 :: bool +L0: + r0 = c.f + a = r0 + r1 = c.f + c.s = r1; r2 = is_error + return 1 +def indexing(c): + c :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = borrow c.l + r1 = borrow c.n + r2 = CPyList_GetItem(r0, r1) + r3 = cast(str, r2) + keep_alive c, c + return r3 +def call_eq(c): + c, r0 :: __main__.C + r1, r2 :: str + r3 :: bool +L0: + r0 = f() + r1 = borrow r0.f + r2 = borrow c.f + r3 = CPyStr_Equal(r1, r2) + keep_alive r0, c + return r3 + +[case testFinalAttrExpressionKinds2_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f(c: C) -> str: + return c.d.f + "a" + +def g(a: list[D], n: int) -> str: + return a[n].f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + self.f = s + return 1 +def f(c): + c :: __main__.C + r0 :: __main__.D + r1, r2, r3 :: str +L0: + r0 = borrow c.d + r1 = borrow r0.f + r2 = 'a' + r3 = PyUnicode_Concat(r1, r2) + keep_alive c, r0 + return r3 +def g(a, n): + a :: list + n :: int + r0 :: object + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CPyList_GetItemBorrow(a, n) + r1 = borrow cast(__main__.D, r0) + r2 = r1.f + keep_alive a, n, r0 + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + return r4 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 7c7134dcfbfaa..bedfabac791d4 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -2366,3 +2366,717 @@ L0: r3 = CPyBytes_Concat(r2, c) dec_ref r2 return r3 + +[case testBorrowedFinalAttribute] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, b: bool) -> str: + a = c.s if b else "x" + return a + +def g(c: C) -> C: + return C(c.s) +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, b): + c :: __main__.C + b :: bool + r0, r1, r2, a :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = borrow c.s + inc_ref r0 + r1 = r0 + goto L3 +L2: + r2 = 'x' + inc_ref r2 + r1 = r2 +L3: + a = r1 + return a +def g(c): + c :: __main__.C + r0 :: str + r1 :: __main__.C +L0: + r0 = borrow c.s + r1 = C(r0) + return r1 + +[case testBorrowedFinalAttribute2_withgil] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: int) -> str: + a = [c] + return a[n].s + "a" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: int + r0 :: list + r1 :: ptr + a :: list + r2 :: object + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = PyList_New(1) + r1 = list_items r0 + inc_ref c + buf_init_item r1, 0, c + a = r0 + r2 = CPyList_GetItemBorrow(a, n) + r3 = borrow cast(__main__.C, r2) + r4 = r3.s + dec_ref a + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 + +[case testBorrowedFinalAttribute3_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d = d + +class CC: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f() -> str: + c = C(D("x")) + return c.d.f + "a" + +def g(d: D) -> str: + c = CC(d) + return c.d.f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def CC.__init__(self, d): + self :: __main__.CC + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.f = s + return 1 +def f(): + r0 :: str + r1 :: __main__.D + r2, c :: __main__.C + r3 :: __main__.D + r4, r5, r6 :: str +L0: + r0 = 'x' + r1 = D(r0) + r2 = C(r1) + dec_ref r1 + c = r2 + r3 = borrow c.d + r4 = r3.f + dec_ref c + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 +def g(d): + d :: __main__.D + r0, c :: __main__.CC + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CC(d) + c = r0 + r1 = borrow c.d + r2 = borrow r1.f + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + dec_ref c + return r4 + +[case testBorrowedFinalAttribute4] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + + def method(self) -> int: + return self.d.foo() + +class D: + def foo(self) -> int: + return 1 +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def C.method(self): + self :: __main__.C + r0 :: __main__.D + r1 :: int +L0: + r0 = borrow self.d + r1 = r0.foo() + return r1 +def D.foo(self): + self :: __main__.D +L0: + return 2 + +[case testBorrowedFinalAttribute5_64bit] +from typing import Final +from librt.vecs import vec +from mypy_extensions import i64 + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: i64) -> str: + a = vec[C]([c]) + return a[n].s + "a" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: i64 + r0 :: object + r1 :: ptr + r2 :: vec[__main__.C] + r3, r4 :: ptr + a :: vec[__main__.C] + r5 :: native_int + r6 :: bit + r7 :: i64 + r8 :: bit + r9 :: bool + r10 :: i64 + r11 :: __main__.C + r12, r13, r14 :: str +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = VecTApi.alloc(1, 1, r1) + r3 = r2.items + inc_ref c + set_mem r3, c :: __main__.C* + r4 = r3 + 8 + a = r2 + r5 = a.len + r6 = n < r5 :: unsigned + if r6 goto L4 else goto L1 :: bool +L1: + r7 = n + r5 + r8 = r7 < r5 :: unsigned + if r8 goto L3 else goto L6 :: bool +L2: + r9 = raise IndexError + unreachable +L3: + r10 = r7 + goto L5 +L4: + r10 = n +L5: + r11 = vec_get_item_unsafe_borrow[__main__.C] a, r10 + r12 = r11.s + dec_ref a + r13 = 'a' + r14 = PyUnicode_Concat(r12, r13) + dec_ref r12 + return r14 +L6: + dec_ref a + goto L2 + +[case testBorrowedFinalAttributeReassignViaWalrus] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, d: C) -> str: + return c.s + (c := d).s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, d): + c, d :: __main__.C + r0, r1, r2 :: str +L0: + r0 = c.s + inc_ref d + c = d + dec_ref c + r1 = borrow d.s + r2 = PyUnicode_Concat(r0, r1) + dec_ref r0 + return r2 + +[case testBorrowedFinalAttributeConditionalMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(b: bool) -> str: + return (C("x").s if b else "y") + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(b): + b :: bool + r0 :: str + r1 :: __main__.C + r2, r3, r4, r5, r6 :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = 'x' + r1 = C(r0) + r2 = borrow r1.s + inc_ref r2 + r3 = r2 + dec_ref r1 + goto L3 +L2: + r4 = 'y' + inc_ref r4 + r3 = r4 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r3, r5) + dec_ref r3 + return r6 + +[case testBorrowedFinalAttributeAndMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s and C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L2 else goto L1 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeOrMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s or C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L1 else goto L2 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeChainedComparisonMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> bool: + return s == "x" == C("x").s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s, r0 :: str + r1, r2 :: bool + r3 :: str + r4 :: __main__.C + r5 :: str + r6 :: bool +L0: + r0 = 'x' + r1 = CPyStr_EqualLiteral(s, r0, 1) + if r1 goto L2 else goto L1 :: bool +L1: + r2 = r1 + goto L3 +L2: + r3 = 'x' + r4 = C(r3) + r5 = borrow r4.s + r6 = CPyStr_EqualLiteral(r5, r0, 1) + r2 = r6 + dec_ref r4 +L3: + return r2 + +[case testBorrowedFinalInListComprehension_withgil] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(a: list[C]) -> list[str]: + return [x.s for x in a] +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(a): + a :: list + r0 :: native_int + r1 :: list + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x :: __main__.C + r7 :: str + r8 :: native_int +L0: + r0 = var_object_size a + r1 = PyList_New(r0) + r2 = 0 +L1: + r3 = var_object_size a + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = list_get_item_unsafe a, r2 + r6 = cast(__main__.C, r5) + x = r6 + r7 = x.s + dec_ref x + CPyList_SetItemUnsafe(r1, r2, r7) +L3: + r8 = r2 + 1 + r2 = r8 + goto L1 +L4: + return r1 + +[case testBorrowedFinalInSetComprehension_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.s: Final = s + +def use(s: str) -> bool: + return True + +def f(a: list[C]) -> set[C]: + return {x for x in a if use(x.d.s)} +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.s = s + return 1 +def use(s): + s :: str +L0: + return 1 +def f(a): + a :: list + r0 :: set + r1, r2 :: native_int + r3 :: bit + r4 :: object + r5, x :: __main__.C + r6 :: __main__.D + r7 :: str + r8 :: bool + r9 :: i32 + r10 :: bit + r11 :: native_int +L0: + r0 = PySet_New(0) + r1 = 0 +L1: + r2 = var_object_size a + r3 = r1 < r2 :: signed + if r3 goto L2 else goto L5 :: bool +L2: + r4 = list_get_item_unsafe a, r1 + r5 = cast(__main__.C, r4) + x = r5 + r6 = borrow x.d + r7 = borrow r6.s + r8 = use(r7) + if r8 goto L3 else goto L6 :: bool +L3: + r9 = PySet_Add(r0, x) + dec_ref x + r10 = r9 >= 0 :: signed +L4: + r11 = r1 + 1 + r1 = r11 + goto L1 +L5: + return r0 +L6: + dec_ref x + goto L4 + +[case testBorrowedFinalAttributeThroughCast] +from typing import Final, cast + +class C: + def __init__(self, x: object) -> None: + self.x: Final = x + +def make() -> object: + return C(1) + +def g(a: object) -> object: + return a + +def h() -> object: + return g(cast(C, make()).x) +[out] +def C.__init__(self, x): + self :: __main__.C + x :: object +L0: + inc_ref x + self.x = x + return 1 +def make(): + r0 :: object + r1 :: __main__.C +L0: + r0 = object 1 + r1 = C(r0) + return r1 +def g(a): + a :: object +L0: + inc_ref a + return a +def h(): + r0 :: object + r1 :: __main__.C + r2, r3 :: object +L0: + r0 = make() + r1 = borrow cast(__main__.C, r0) + r2 = borrow r1.x + r3 = g(r2) + dec_ref r0 + return r3 + +[case testBorrowedFinalAttributeCastChain] +from typing import Final, cast + +class Inner: + def __init__(self, x: object) -> None: + self.x: Final = x + +class Outer: + def __init__(self, inner: object) -> None: + self.inner: Final = inner + +def g(a: object) -> object: + return a + +def make() -> object: + return Outer(Inner(1)) + +def h() -> object: + # cast( Inner, ).x + o = cast(Outer, make()) + return g(cast(Inner, o.inner).x) +[out] +def Inner.__init__(self, x): + self :: __main__.Inner + x :: object +L0: + inc_ref x + self.x = x + return 1 +def Outer.__init__(self, inner): + self :: __main__.Outer + inner :: object +L0: + inc_ref inner + self.inner = inner + return 1 +def g(a): + a :: object +L0: + inc_ref a + return a +def make(): + r0 :: object + r1 :: __main__.Inner + r2 :: __main__.Outer +L0: + r0 = object 1 + r1 = Inner(r0) + r2 = Outer(r1) + dec_ref r1 + return r2 +def h(): + r0 :: object + r1, o :: __main__.Outer + r2 :: object + r3 :: __main__.Inner + r4, r5 :: object +L0: + r0 = make() + r1 = cast(__main__.Outer, r0) + o = r1 + r2 = borrow o.inner + r3 = borrow cast(__main__.Inner, r2) + r4 = borrow r3.x + r5 = g(r4) + dec_ref o + return r5 diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index cf9b54368c277..a7a08e7ced5f2 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1974,3 +1974,96 @@ async def async_iter(vals: list[int]) -> AsyncIterator[int]: yield v [typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwait] +from typing import Final +from testutil import async_val + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the await, which is a + # suspension point in the middle of the expression. + return c.s + await async_val("hello") + +[file driver.py] +from native import f, C +from testutil import run_generator + +for i in range(50): + c = C("value" + str(i)) + yields, val = run_generator(f(c), inputs=["world"]) + assert yields == ("hello",), yields + assert val == "value" + str(i) + "world", repr(val) + +[case testBorrowedFinalAttrAcrossAsyncComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +class AsyncRange: + def __init__(self, n: int) -> None: + self.n = n + self.i = 0 + + def __aiter__(self) -> "AsyncRange": + return self + + async def __anext__(self) -> str: + if self.i >= self.n: + raise StopAsyncIteration + i = self.i + self.i += 1 + # Suspend in the middle of iteration. + await asyncio.sleep(0) + return "item" + str(i) + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the async comprehension, + # which suspends via an implicit await on __anext__ that isn't represented as + # an AwaitExpr node in the AST. + return c.s + "".join([x async for x in AsyncRange(3)]) + +async def test_async_comprehension_borrow() -> None: + for i in range(50): + c = C("value" + str(i)) + assert await f(c) == "value" + str(i) + "item0item1item2" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwaitAfterComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def sink() -> str: + await asyncio.sleep(0) + return "z" + +async def f(c: C, xs: list[int]) -> str: + # A (sync) comprehension enters a nested borrow scope earlier in the same + # top-level expression. It must not clear the enclosing expression's suspend + # flag, or c.s (a final attr) gets borrowed across the await below, which + # produces invalid C (the borrow does not survive the suspension point). + return str([x for x in xs]) + (c.s + await sink()) + +async def test_borrow_final_attr_across_await_after_comprehension() -> None: + for i in range(50): + c = C("v" + str(i)) + assert await f(c, [1, 2, 3]) == "[1, 2, 3]" + "v" + str(i) + "z" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 73927bb037a71..4e8b67a0061a8 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -6104,3 +6104,76 @@ base = sys.getrefcount(BIG) o.v = BIG o.v = BIG assert sys.getrefcount(BIG) == base, "reassignment leaked refs" + +[case testBorrowedFinalAttributeInLambdaAndNestedFunction] +from typing import Final, Callable + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def make_lambda() -> Callable[[], int]: + # The lambda body creates a C, borrows its final attribute, then calls + # spend() (which allocates heavily) before reading the borrow. + return lambda: use(C(Box(111)).b, spend(Box(0))) + +def make_nested() -> Callable[[], int]: + def inner() -> int: + return use(C(Box(111)).b, spend(Box(0))) + return inner + +def test_borrowed_final_attribute_in_lambda() -> None: + f = make_lambda() + for _ in range(1000): + assert f() == 111 + +def test_borrowed_final_attribute_in_nested_function() -> None: + g = make_nested() + for _ in range(1000): + assert g() == 111 + +[case testBorrowedFinalAttributeInComprehension] +from typing import Final + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def comp(ns: list[int]) -> list[int]: + # Each iteration allocates a fresh C, borrows its final attribute .b, then + # calls spend() (which allocates heavily) before reading the borrow. The + # borrow container must be kept alive for the current iteration only; if it + # is scoped to the whole (comprehension) expression, the refcount pass + # dec_refs an uninitialized temporary on the loop back-edge and leaks/uses + # freed memory. + return [use(C(Box(111)).b, spend(Box(i))) for i in ns] + +def test_borrowed_final_attribute_in_comprehension() -> None: + for _ in range(1000): + assert comp([1, 2, 3, 4, 5]) == [112, 113, 114, 115, 116] diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index cf1dac7c57333..a23af2ed6eea7 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -944,3 +944,46 @@ from typing import Optional, Union def test_compiledGeneratorEmptyTuple() -> None: jobs: Generator[Optional[str], None, None] = (_ for _ in ()) assert list(jobs) == [] + +[case testBorrowedFinalAttrAcrossYield] +from typing import Final, Generator + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C) -> Generator[str, str, None]: + # c.s is a final attr; it must not be borrowed across the yield, which is + # a suspension point in the middle of the expression. + x = c.s + (yield "first") + yield x + +def test_borrow_across_yield() -> None: + for i in range(50): + c = C("value" + str(i)) + g = f(c) + assert next(g) == "first" + assert g.send("world") == "value" + str(i) + "world" + +def yield_from_gen(c: C) -> Generator[str, str, str]: + r = yield "inner" + return c.s + r + +def g2(c: C) -> Generator[str, str, str]: + # Borrow a final attr across a yield from expression. + x = c.s + (yield from yield_from_gen(c)) + return x + +def test_borrow_across_yield_from() -> None: + for i in range(50): + c = C("v" + str(i)) + gen = g2(c) + assert next(gen) == "inner" + try: + gen.send("!") + except StopIteration as e: + assert e.value == "v" + str(i) + "v" + str(i) + "!", e.value + else: + assert False + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 50ffc9004743e..6608e6db8e7b4 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -36,6 +36,7 @@ "irbuild-statements.test", "irbuild-nested.test", "irbuild-classes.test", + "irbuild-final.test", "irbuild-optional.test", "irbuild-any.test", "irbuild-generics.test", From 0faa413ebf7c924a864ef5dabd70303d898e7766 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 10 Jul 2026 05:47:43 +0530 Subject: [PATCH 40/50] Use `PYODIDE` environment variable for Emscripten cross-compilation detection (#21714) I noticed that y'all were carrying this workaround ever since #14888. A better way to detect this is through [the `PYODIDE` environment variable, which we've documented over the years](https://pyodide-build.readthedocs.io/en/latest/how-to/migrate.html#detecting-pyodide-at-build-time), and is what most projects use now. The MACHDEP generality would mostly apply to non-Pyodide-specific WASM build targets, which are growing in relevance but are a bit less mature as Pyodide at this time. --- mypy/options.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index 2f03cd3eab5b7..92c9ea3b1701d 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -1,9 +1,9 @@ from __future__ import annotations +import os import pprint import re import sys -import sysconfig from collections.abc import Callable from re import Pattern from typing import Any, Final @@ -112,11 +112,11 @@ def __init__(self) -> None: # then mypy does not search for PEP 561 packages. self.python_executable: str | None = sys.executable - # When cross compiling to emscripten, we need to rely on MACHDEP because - # sys.platform is the host build platform, not emscripten. - MACHDEP = sysconfig.get_config_var("MACHDEP") - if MACHDEP == "emscripten": - self.platform = MACHDEP + # When cross compiling to emscripten, sys.platform is the host build + # platform, not emscripten. Pyodide sets the PYODIDE environment variable + # at build time, so we use it to detect the emscripten target. + if "PYODIDE" in os.environ: + self.platform = "emscripten" else: self.platform = sys.platform From a9f62a3cf98a58a7a2607b7c81695802b39f5edc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 Jul 2026 12:29:39 +0100 Subject: [PATCH 41/50] [mypyc] Make attribute access memory safe on free-threaded builds (#21705) This fixes memory unsafety when there is a race condition related to attribute get/set operations on a free-threaded build, for simple reference-based attributes only (`PyObject *`). This includes attributes with primitive types like `list`, `set` and `str` and `dict`, and native class types. The main idea is to use delayed decref for the old attribute value when assigning to an attribute. There are detailed comments explaining the approach in detail. This causes a ~5% slowdown in self check when using 3.14t. Currently it looks like a significant perf cost is unavoidable if we want to fix memory unsafety, but we can further optimize mypy to reduce the impact by introducing final attributes, for example. Remaining work includes fixing attributes with fixed-length tuple types and tagged integer types (in follow-up PRs). I used coding agent assist heavily. Work on mypyc/mypyc#1203. --- mypyc/codegen/emitclass.py | 53 ++++++++++- mypyc/codegen/emitfunc.py | 83 ++++++++++++++--- mypyc/ir/class_ir.py | 16 ++++ mypyc/ir/rtypes.py | 13 +++ mypyc/irbuild/builder.py | 12 +-- mypyc/lib-rt/pythonsupport.c | 21 +++++ mypyc/lib-rt/pythonsupport.h | 134 +++++++++++++++++++++++++++ mypyc/test-data/irbuild-classes.test | 40 ++++++++ mypyc/test-data/run-classes.test | 52 +++++++++++ 9 files changed, 404 insertions(+), 20 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 9baaec06a4611..a1127b15ad9d1 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -29,6 +29,7 @@ BITMAP_BITS, BITMAP_TYPE, CPYFUNCTION_NAME, + IS_FREE_THREADED, MYPYC_DEFAULTS_SETUP, NATIVE_PREFIX, PREFIX, @@ -43,7 +44,7 @@ FuncIR, get_text_signature, ) -from mypyc.ir.rtypes import RTuple, RType, object_rprimitive +from mypyc.ir.rtypes import RTuple, RType, is_simple_refcounted_pointer, object_rprimitive from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type @@ -1165,6 +1166,32 @@ def generate_getter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("{") attr_expr = f"self->{attr_field}" + if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + # In free-threaded builds, load the attribute and take a new reference + # atomically to avoid a use-after-free race with a concurrent setter. + # CPy_GetAttrRef returns NULL if the attribute is undefined (NULL field), + # which is exactly the error/undefined value for a 'PyObject *' field. + # + # Final attributes are never rebound (no setter), so there is no concurrent + # writer to race with: a plain load + incref is safe. Use the cheaper + # CPy_GetAttrRefFinal, which skips the try-incref and _Py_NewRefWithLock + # slow path entirely (an unconditional Py_INCREF needs no maybe-weakref). + # This getter is generated per defining class, so a direct membership test + # matches the read-only getset table above (no need to walk the MRO). + if attr in cl.final_attributes: + getattr_ref = f"CPy_GetAttrRefFinal((PyObject **)&{attr_expr})" + else: + getattr_ref = f"CPy_GetAttrRef((PyObject **)&{attr_expr})" + emitter.emit_line(f"PyObject *retval = {getattr_ref};") + emitter.emit_line("if (unlikely(retval == NULL)) {") + emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") + emitter.emit_line(f' "attribute {repr(attr)} of {repr(cl.name)} undefined");') + emitter.emit_line("return NULL;") + emitter.emit_line("}") + emitter.emit_line("return retval;") + emitter.emit_line("}") + return + # HACK: Don't consider refcounted values as always defined, since it's possible to # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted # values is benign. @@ -1202,6 +1229,30 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("return -1;") emitter.emit_line("}") + if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + # In free-threaded builds, publish the new value atomically via + # CPy_SetAttrRef so a concurrent reader (see CPy_GetAttrRef) never sees a + # torn pointer or a freed old value. CPy_SetAttrRef steals its value and + # reclaims the old one, so we cast/type-check the incoming value, take a + # new reference (the setter only borrows 'value'), then hand it over. + # A NULL value deletes the attribute (reclaims the old value, stores NULL). + if deletable: + emitter.emit_line("if (value != NULL) {") + if is_same_type(rtype, object_rprimitive): + emitter.emit_line("PyObject *tmp = value;") + else: + emitter.emit_cast("value", "tmp", rtype, declare_dest=True) + emitter.emit_lines("if (!tmp)", " return -1;") + emitter.emit_inc_ref("tmp", rtype) + emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&self->{attr_field}, tmp);") + if deletable: + emitter.emit_line("} else {") + emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&self->{attr_field}, NULL);") + emitter.emit_line("}") + emitter.emit_line("return 0;") + emitter.emit_line("}") + return + # HACK: Don't consider refcounted values as always defined, since it's possible to # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted # values is benign. diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index dcb606f6ab51b..4c3c17d047c8a 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -12,7 +12,13 @@ TracebackAndGotoHandler, c_array_initializer, ) -from mypyc.common import GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, NATIVE_PREFIX, REG_PREFIX +from mypyc.common import ( + GENERATOR_ATTRIBUTE_PREFIX, + HAVE_IMMORTAL, + IS_FREE_THREADED, + NATIVE_PREFIX, + REG_PREFIX, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( @@ -83,6 +89,7 @@ is_int_rprimitive, is_none_rprimitive, is_pointer_rprimitive, + is_simple_refcounted_pointer, is_tagged, ) @@ -394,6 +401,35 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st cast = f"({decl_cl.struct_name(self.emitter.names)} *)" return f"({cast}{obj})->{self.emitter.attr(op.attr)}" + def emit_load_attr_take_ref( + self, dest: str, obj: str, op: GetAttr, cl: ClassIR, attr_rtype: RType, attr_expr: str + ) -> bool: + """Emit the load of a native attribute into 'dest', taking a new reference. + + On free-threaded builds, reading a single reference-counted 'PyObject *' field + and taking a new reference must be done atomically to avoid a use-after-free + race with a concurrent setter. CPy_GetAttrRef performs the load and incref + atomically and returns a new reference (or NULL if undefined), so callers must + NOT emit a separate inc_ref. Return True in that case so the caller can skip it. + + Final attributes are never rebound (no setter), so there is no concurrent writer + and no use-after-free window; an owned read uses the cheaper CPy_GetAttrRefFinal + (a plain load + incref). Borrowed reads keep the plain load: they are only emitted + for attributes safe to borrow on free-threaded builds (Final and vec attrs -- see + transform_member_expr in irbuild), whose values live as long as their container. + The default (GIL) build always takes the plain-load path and increfs separately. + """ + use_get_attr_ref = ( + IS_FREE_THREADED and is_simple_refcounted_pointer(attr_rtype) and not op.is_borrowed + ) + if use_get_attr_ref and cl.is_final_attr(op.attr): + self.emitter.emit_line(f"{dest} = CPy_GetAttrRefFinal((PyObject **)&{attr_expr});") + elif use_get_attr_ref: + self.emitter.emit_line(f"{dest} = CPy_GetAttrRef((PyObject **)&{attr_expr});") + else: + self.emitter.emit_line(f"{dest} = {attr_expr};") + return use_get_attr_ref + def visit_get_attr(self, op: GetAttr) -> None: if op.allow_error_value: self.get_attr_with_allow_error_value(op) @@ -427,7 +463,9 @@ def visit_get_attr(self, op: GetAttr) -> None: else: # Otherwise, use direct or offset struct access. attr_expr = self.get_attr_expr(obj, op, decl_cl) - self.emitter.emit_line(f"{dest} = {attr_expr};") + use_get_attr_ref = self.emit_load_attr_take_ref( + dest, obj, op, cl, attr_rtype, attr_expr + ) always_defined = cl.is_always_defined(op.attr) merged_branch = None if not always_defined: @@ -458,7 +496,7 @@ def visit_get_attr(self, op: GetAttr) -> None: ) ) - if attr_rtype.is_refcounted and not op.is_borrowed: + if attr_rtype.is_refcounted and not op.is_borrowed and not use_get_attr_ref: if not merged_branch and not always_defined: self.emitter.emit_line("} else {") self.emitter.emit_inc_ref(dest, attr_rtype) @@ -480,16 +518,20 @@ def get_attr_with_allow_error_value(self, op: GetAttr) -> None: cl = rtype.class_ir attr_rtype, decl_cl = cl.attr_details(op.attr) - # Direct struct access without NULL check attr_expr = self.get_attr_expr(obj, op, decl_cl) - self.emitter.emit_line(f"{dest} = {attr_expr};") - - # Only emit inc_ref if not NULL - if attr_rtype.is_refcounted and not op.is_borrowed: - check = self.error_value_check(op, "!=") - self.emitter.emit_line(f"if ({check}) {{") - self.emitter.emit_inc_ref(dest, attr_rtype) - self.emitter.emit_line("}") + # On free-threaded builds this takes a new reference atomically (see + # emit_load_attr_take_ref). CPy_GetAttrRef returns NULL when the field is + # undefined, which is precisely the error value here, so the "NULL without + # AttributeError" behavior is preserved (this op has error_kind ERR_NEVER). + # is_simple_refcounted_pointer excludes tuples/vecs, so error_value_check's + # special cases only matter on the plain-load path (where no ref was taken). + if not self.emit_load_attr_take_ref(dest, obj, op, cl, attr_rtype, attr_expr): + # Only emit inc_ref if not NULL + if attr_rtype.is_refcounted and not op.is_borrowed: + check = self.error_value_check(op, "!=") + self.emitter.emit_line(f"if ({check}) {{") + self.emitter.emit_inc_ref(dest, attr_rtype) + self.emitter.emit_line("}") def next_branch(self) -> Branch | None: if self.op_index + 1 < len(self.ops): @@ -529,6 +571,23 @@ def visit_set_attr(self, op: SetAttr) -> None: op.attr, ) ) + elif IS_FREE_THREADED and is_simple_refcounted_pointer(attr_rtype): + # In free-threaded builds, publishing a single reference-counted + # 'PyObject *' field must be atomic so a concurrent reader (see + # CPy_GetAttrRef) never observes a torn pointer or a freed value. + # Both helpers steal the reference to src. + attr_expr = self.get_attr_expr(obj, op, decl_cl) + if op.is_init: + # The attribute is known to be previously undefined (NULL), so + # there is no old value to reclaim; a relaxed store suffices + # (self's later publication provides the release barrier -- see + # CPy_InitAttrRef). + self.emitter.emit_line(f"CPy_InitAttrRef((PyObject **)&{attr_expr}, {src});") + else: + # Atomically swap in the new value and reclaim the old one. + self.emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&{attr_expr}, {src});") + if op.error_kind == ERR_FALSE: + self.emitter.emit_line(f"{dest} = 1;") else: # ...and struct access for normal attributes. attr_expr = self.get_attr_expr(obj, op, decl_cl) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 028df5898d920..6ea1eb072999d 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -269,6 +269,22 @@ def attr_details(self, name: str) -> tuple[RType, ClassIR]: def attr_type(self, name: str) -> RType: return self.attr_details(name)[0] + def is_final_attr(self, name: str) -> bool: + """Is the (possibly inherited) attribute Final, i.e. never rebound? + + A Final attribute is read-only at runtime (it has no setter) and is assigned + exactly once during construction, so it can never be reassigned afterwards. + This makes it safe to borrow on free-threaded builds (no concurrent store can + invalidate a borrowed reference) and lets reads skip the concurrent-writer + guard. Returns False for properties and for attributes this class doesn't have. + """ + for ir in self.mro: + if name in ir.attributes: + return name in ir.final_attributes + if name in ir.property_types: + return False + return False + def method_decl(self, name: str) -> FuncDecl: for ir in self.mro: if name in ir.method_decls: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 1a5515d5621a8..afe3d3e9df39b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -569,6 +569,19 @@ def is_native_rprimitive(rtype: RType) -> bool: return isinstance(rtype, RPrimitive) and rtype.name in KNOWN_NATIVE_TYPES +def is_simple_refcounted_pointer(rtype: RType) -> bool: + """Is rtype represented at runtime as a single, reference-counted 'PyObject *'? + + This covers 'object', 'str', containers, instances of native classes and + optional/union types -- everything whose C representation is exactly one + 'PyObject *' field that owns a reference. It excludes unboxed types (tagged + 'int', fixed-width ints, floats, bools), inline tuples ('RTuple'), vectors + ('RVec') and C structs ('RStruct'), which need different treatment for + free-threaded memory safety. + """ + return rtype.is_refcounted and not rtype.is_unboxed and not isinstance(rtype, RStruct) + + def is_tagged(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive or rtype is short_int_rprimitive diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index d8ae71e33bf59..0b598a00889f5 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1642,13 +1642,11 @@ def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: builds, since no concurrent store can invalidate the borrowed reference. """ obj_rtype = self.node_type(expr.expr) - if not (isinstance(obj_rtype, RInstance) and obj_rtype.class_ir.is_ext_class): - return False - # Find the class that defines the attribute and check whether it's Final there. - for ir in obj_rtype.class_ir.mro: - if expr.name in ir.attributes: - return expr.name in ir.final_attributes - return False + return ( + isinstance(obj_rtype, RInstance) + and obj_rtype.class_ir.is_ext_class + and obj_rtype.class_ir.is_final_attr(expr.name) + ) def root_is_reassigned(self, v: Value) -> bool: """Is the root local variable a borrow chain 'v' reads from reassigned this expression? diff --git a/mypyc/lib-rt/pythonsupport.c b/mypyc/lib-rt/pythonsupport.c index 0a99f0ae2e29b..a8f5a4f4ad4ea 100644 --- a/mypyc/lib-rt/pythonsupport.c +++ b/mypyc/lib-rt/pythonsupport.c @@ -5,6 +5,27 @@ #include "pythonsupport.h" +#ifdef Py_GIL_DISABLED +// Cold slow path of CPy_GetAttrRef (declared in pythonsupport.h). Reached only +// when the inline fast-path try-incref fails: the value is owned by another +// thread, so taking a reference requires an atomic shared-refcount operation. +// Kept out-of-line so the inline fast path stays small. +// +// First try the lock-free shared-refcount CAS. If the value has not had +// maybe-weakref set yet (for example, it was published by CPy_InitAttrRef), force +// a cross-thread reference via _Py_NewRefWithLock, which cannot fail and sets +// maybe-weakref so subsequent reads take the fast path. The value was already +// observed in the field by CPy_GetAttrRef; CPy_SetAttrRef's QSBR-delayed decref +// keeps any replaced value alive long enough for this reader. +CPy_NOINLINE +PyObject *CPy_GetAttrRefSlow(PyObject *v) { + if (_Py_TryIncRefShared(v)) { + return v; + } + return _Py_NewRefWithLock(v); // sets maybe-weakref; cannot fail +} +#endif + ///////////////////////////////////////// // Adapted from bltinmodule.c in Python 3.7.0 PyObject* diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 35f1e78df3915..33c5a596d1747 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -23,6 +23,10 @@ #include "internal/pycore_setobject.h" // _PySet_Update #endif +#ifdef Py_GIL_DISABLED +#include "internal/pycore_object.h" // _Py_TryIncrefFast, _Py_TryIncRefShared +#endif + #if CPY_3_12_FEATURES #include "internal/pycore_frame.h" #endif @@ -34,6 +38,136 @@ extern "C" { } // why isn't emacs smart enough to not indent this #endif +#ifdef Py_GIL_DISABLED +// Read a native attribute that is a single reference-counted 'PyObject *' field, +// returning a new reference (or NULL if the field is NULL/undefined). +// +// On free-threaded builds a plain load followed by an incref races with a +// concurrent setter that may decref the old value to zero and free it before the +// incref runs (use-after-free). CPy_SetAttrRef avoids that by reclaiming old +// values through QSBR-delayed decref, so a value observed in the field remains +// safe to touch while this reader is running. +// +// Only the hot case is inlined here: an incref of a value owned by this thread or +// immortal, via '_Py_TryIncrefFast' (no CAS, no loop). Everything colder -- the +// cross-thread shared-refcount CAS and the _Py_NewRefWithLock fallback -- lives +// out-of-line in 'CPy_GetAttrRefSlow'. Splitting it this way keeps each call +// site's fast path small enough to inline, which is measurably faster than +// letting the compiler auto-out-line the whole helper (that merges every read +// site's branch history into one shared copy and mispredicts). It is only used in +// free-threaded builds; the default (GIL) build keeps the plain load + incref +// generated inline by mypyc. +PyObject *CPy_GetAttrRefSlow(PyObject *v); + +static inline PyObject *CPy_GetAttrRef(PyObject **field) { + PyObject *v = (PyObject *)_Py_atomic_load_ptr_acquire(field); + if (v == NULL) { + return NULL; + } + if (_Py_TryIncrefFast(v)) { + return v; + } + return CPy_GetAttrRefSlow(v); +} + +// Read a native attribute that is a single reference-counted 'PyObject *' field +// AND is Final (assigned once during construction, never rebound -- mypyc emits no +// setter for it), returning a new reference (or NULL if undefined). +// +// A Final attribute has no concurrent writer after 'self' is published, so the +// use-after-free race that CPy_GetAttrRef guards against cannot happen: the field +// holds a strong reference for the object's whole lifetime, and any thread reading +// it necessarily holds 'self', which keeps the value alive. So the try-incref + +// _Py_NewRefWithLock fallback are unnecessary here -- a plain load + Py_INCREF is +// safe. A cross-thread Py_INCREF is an unconditional +// atomic add on ob_ref_shared, so (unlike CPy_GetAttrRef's try-incref) it needs no +// maybe-weakref and has no slow path. The load is relaxed rather than acquire: the +// reader reached 'self' through a synchronization edge (self's own publication) +// that already ordered the construction stores before it, exactly as with +// CPy_InitAttrRef's relaxed store. Relaxed keeps it TSan-clean at zero cost (plain +// mov/ldr). +static inline PyObject *CPy_GetAttrRefFinal(PyObject **field) { + PyObject *v = (PyObject *)_Py_atomic_load_ptr_relaxed(field); + if (v != NULL) { + Py_INCREF(v); + } + return v; +} + +// Reclaim the previous value of a native attribute after it has been replaced. +// +// CPy_GetAttrRef reads the field optimistically without holding any lock the +// writer also takes, so a reader can load the old pointer and then try to take a +// reference after this store. The old value must therefore stay alive until every +// thread has passed a quiescent point, which is exactly what a QSBR-deferred +// decref guarantees. So all mortal old values are +// reclaimed via _PyObject_XDecRefDelayed, matching CPython's own replace-a-slot +// paths (e.g. _PyObject_SetDict / _PyObject_SetManagedDict). +// +// We deliberately do NOT take a "local refcount > 1, owned by this thread" fast +// path: dropping the field's reference is not the only decref of the object, so a +// non-freeing local decrement here does not prevent an unrelated reference holder +// from driving the object to zero (a plain, non-deferred Py_DECREF -> _Py_Dealloc) +// while an in-flight reader still holds the stale pointer -- a use-after-free that +// only QSBR deferral closes. Immortal objects are never freed, so skipping their +// decref entirely is safe and avoids queuing a no-op onto the delayed-free list. +static inline void CPy_DecRefAttrOld(PyObject *op) { + if (op == NULL) { + return; + } + if (_Py_IsImmortal(op)) { + return; + } + _PyObject_XDecRefDelayed(op); +} + +// Set a native attribute that is a single reference-counted 'PyObject *' field, +// stealing the reference to 'value' (which may be NULL to delete the attribute) +// and safely reclaiming the previous value. +// +// Memory safety does NOT depend on SetMaybeWeakref here, so (unlike an earlier +// version) we do not call it. Two things keep this safe: +// - The old value is reclaimed via CPy_DecRefAttrOld, a QSBR-deferred decref, so +// it cannot be freed while an in-flight CPy_GetAttrRef still holds the stale +// pointer. +// - A concurrent cross-thread reader whose inline fast-path try-incref fails on +// an unflagged 'value' still cannot fail and never blocks on this writer: +// CPy_GetAttrRef falls into CPy_GetAttrRefSlow, which is fully lock-free. It +// retries with a lock-free shared-refcount CAS (_Py_TryIncRefShared) and, only +// if that also fails, forces a reference via _Py_NewRefWithLock, which cannot +// fail and lazily sets maybe-weakref so later cross-thread reads take the CAS. +// This mirrors CPy_InitAttrRef, which already omits SetMaybeWeakref for the same +// reason. Setting the flag here would only be a possible performance tuning knob +// (it would let that first cross-thread reader succeed on the cheaper +// _Py_TryIncRefShared CAS instead of falling through to _Py_NewRefWithLock); it is +// not needed for correctness. The atomic exchange publishes the new pointer and +// hands back the old one without a writer/writer race. +static inline void CPy_SetAttrRef(PyObject **field, PyObject *value) { + PyObject *old = (PyObject *)_Py_atomic_exchange_ptr(field, value); + CPy_DecRefAttrOld(old); +} + +// Initialize a native attribute that is known to be previously undefined (NULL), +// stealing the reference to 'value'. +// +// Initializer stores only happen while 'self' is still thread-local (the +// attribute-definedness analysis marks a SetAttr as an initializer only before +// 'self' can leak -- see mypyc/analysis/attrdefined.py). So there is no old value +// to reclaim, no competing writer, and the field store is not itself the +// publication point: 'self' is published later (when it escapes __init__ or is +// returned), and that publication carries the release barrier making all the +// construction stores visible. A relaxed store therefore suffices. +// +// Unlike CPy_SetAttrRef, this deliberately does NOT call SetMaybeWeakref (its CAS +// is pure overhead here, ~+2.6ns per fresh store, and construction-heavy code +// pays it on every attribute of every new object). The cost is moved off this hot +// path onto CPy_GetAttrRef's cold slow path, which sets maybe-weakref lazily on +// the first cross-thread read that needs it. +static inline void CPy_InitAttrRef(PyObject **field, PyObject *value) { + _Py_atomic_store_ptr_relaxed(field, value); +} +#endif + PyObject* update_bases(PyObject *bases); int init_subclass(PyTypeObject *type, PyObject *kwds); diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 66caf0772ec40..8eaa63e4583b5 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1276,6 +1276,46 @@ L0: r1 = r0.x return r1 +[case testCanBorrowFinalAttribute_nogil] +from typing import Final + +# On free-threaded builds native attribute reads are not borrowed (a concurrent +# store could free the old value), EXCEPT Final attributes, which can't be rebound. +# Here the intermediate 'd.c' is borrowed because 'c' is Final; contrast with +# testCannotBorrowAttribute_nogil, where the non-Final 'c' is read owned. The +# borrow decision uses the same ClassIR.is_final_attr predicate as codegen. +def f(d: D) -> int: + return d.c.xf + +class C: + def __init__(self, xf: int) -> None: + self.xf: Final = xf +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.xf + keep_alive d + return r1 +def C.__init__(self, xf): + self :: __main__.C + xf :: int +L0: + self.xf = xf + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + self.c = c + return 1 + [case testNoBorrowOverPropertyAccess] class C: d: D diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 4e8b67a0061a8..56ad3673e2896 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2865,6 +2865,58 @@ def test_rebind_inherited_via_setattr() -> None: assert d.x == 1 assert d.y == 2 +[case testFinalRefcountedAttributeRead] +# Exercises reading Final reference-counted ('PyObject *') attributes, which on +# free-threaded builds use a faster read path (CPy_GetAttrRefFinal) that skips the +# concurrent-writer guard, since a Final attribute is never rebound. +from typing import Final + +class C: + def __init__(self, s: str, items: list[int]) -> None: + self.s: Final = s + self.items: Final = items + + def read_s(self) -> str: + return self.s + + def read_items(self) -> list[int]: + return self.items + + def chained_len(self) -> int: + # borrowed intermediate read of a Final attr, then a call on it + return len(self.items) + +class D(C): + def __init__(self, s: str, items: list[int], t: str) -> None: + super().__init__(s, items) + self.t: Final = t + + def read_inherited(self) -> str: + return self.s + +def test_read_final_object_attrs() -> None: + c = C("hello", [1, 2, 3]) + # Read via getter (property access) and via native method bodies. + assert c.s == "hello" + assert c.read_s() == "hello" + assert c.items == [1, 2, 3] + assert c.read_items() == [1, 2, 3] + assert c.chained_len() == 3 + # Reading repeatedly must not corrupt the refcount / free the value. + for _ in range(1000): + assert c.read_s() == "hello" + assert c.read_items() == [1, 2, 3] + assert c.s == "hello" + assert c.read_items() is c.items + +def test_read_inherited_final_attr() -> None: + d = D("a", [4, 5], "b") + assert d.s == "a" + assert d.t == "b" + assert d.read_s() == "a" + assert d.read_inherited() == "a" + assert d.read_items() == [4, 5] + [case testClassDerivedFromIntEnum] from enum import IntEnum, auto From 2c2154672040c52e481f423854d104e6cf172585 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 13 Jul 2026 09:07:04 +0100 Subject: [PATCH 42/50] [mypyc] Update documentation of race conditions under free threading (#21726) Mypyc now gives additional thread safety guarantees. --- mypyc/doc/differences_from_python.rst | 48 ++++++++++----------------- mypyc/doc/librt_vecs.rst | 16 ++++++--- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index c65c330edbef3..7e6cf37154d3e 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -293,40 +293,26 @@ attribute tends to be faster than a plain global variable in compiled code:: Free threading -------------- -Mypyc supports free threading, but it doesn't provide the exact -memory safety guarantees as Python in compiled modules under -free threading when there are race conditions. - -Additionally, optimized primitive operations in compiled code may have -different atomicity properties compared to CPython. Use explicit -synchronization if code depends on operations being atomic. This is -already the recommended approach for normal Python code. - -Currently, compiled code must ensure that proper synchronization is -used to prevent data races involving non-final attributes in native -classes, unless the attribute has a value type such as ``bool``, -``float`` or ``i64``. You can use explicit -synchronization, such as via -:ref:`librt.threading.Lock ` (or -:py:class:`threading.Lock`, which is less efficient than -``librt.threading.Lock``) if there is a possibility of such a data -race. +Mypyc supports free threading. However, optimized primitive operations in +compiled code may have different atomicity properties compared to CPython. +Use explicit synchronization if code depends on operations being atomic and +race conditions are possible. This is already the recommended approach for +normal Python code. You can often use :ref:`librt.threading.Lock ` +(or :py:class:`threading.Lock`, which is less efficient than +``librt.threading.Lock``) to fix data races. + +Since mypyc 2.3, the vast majority of operations are memory safe even if +there are race conditions (unlike earlier mypyc releases). This includes +list operations and access to native instance attributes (except for +a few less common use cases that will be fixed in future releases). .. note:: - We are working on improving memory safety in free-threading - builds of Python, and hope to make all normal Python features - memory safe, while providing more efficient but less safe - opt-in, non-standard features. - -As libraries often won't be able to control the concurrent access by -user code, we recommend that modules document that multi-threaded -access is only supported via public interfaces that ensure correct -synchronization. Marking attributes as internal using an underscore -attribute prefix is another possibility, but this is not enforced at -runtime. Another option is to document that multithreaded access is -not supported, or that particular objects should not be used from -multiple threads concurrently. + Operations that aren't safe under race conditions in interpreted CPython + are not expected to be memory safe in compiled code either. + Some :ref:`librt ` features are heavily optimized for performance and + don't guarantee memory safety when there are race conditions + (notably the :ref:`vec ` type). It's always safe to perform read-only operations concurrently. Using objects with final attributes and tuple objects can help prevent diff --git a/mypyc/doc/librt_vecs.rst b/mypyc/doc/librt_vecs.rst index dad3be621ff4c..a7f5bc3ffbd96 100644 --- a/mypyc/doc/librt_vecs.rst +++ b/mypyc/doc/librt_vecs.rst @@ -1,3 +1,5 @@ +.. _librt-vecs: + librt.vecs ========== @@ -198,15 +200,21 @@ with no unnecessary temporary objects. Thread safety ------------- +The ``vec`` type is heavily optimized for performance, and this means that ``vec`` +gives fewer thread safety guarantees than built-in types such as ``list``. +``vec`` is a lower-level type where implicit synchronization would have a very +significant performance cost. However, since vec lengths are immutable, some race +conditions that lists can be susceptible to are not possible with vecs. + In free-threaded Python builds, it's unsafe to write or modify an item if other threads might be concurrently accessing *the same item*. For example, writing ``v[4]`` is not safe to do if another thread might be reading ``v[4]``. Similarly, two threads concurrently calling ``append`` or ``remove`` on the same vec object is not safe. -This is different from list objects, since vec is a lower-level type where implicit -synchronization would have a significant performance cost. However, since vec lengths -are immutable, some race conditions that lists can be susceptible to are not possible -with vecs. +Similarly, assigning to an instance attribute of a native class with a ``vec`` type +is unsafe if another thread might be accessing the same attribute concurrently +(in a free-threaded Python build only). Using ``Final`` attributes is a way +to prevent this race condition, since ``Final`` attributes cannot be rebound. Implementation details ---------------------- From 4d8ad2ab5e86c99581b73775f2c00b9b8265b589 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 13 Jul 2026 10:49:03 +0100 Subject: [PATCH 43/50] Update changelog for 2.3 release (#21728) Release tracking issue: #21717 --- CHANGELOG.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02739ef6591fc..efe63effbfb4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,122 @@ ## Next Release +## Mypy 2.3 + +We've just uploaded mypy 2.3.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### The Upcoming Switch to the New Native Parser + +We are planning to enable the new native parser (`--native-parser`) by +default soon. We recommend that you test the native parser in your projects and report +any issues in the [mypy issue tracker](https://github.com/python/mypy/issues). + +### Mypyc Free-threading Memory Safety + +Free-threaded Python builds that don't have the GIL require additional synchronization +primitives or lock-free algorithms to ensure memory safety when there are race conditions +(for example, when a thread reads a list item while another thread writes the same list +item concurrently). This release greatly improves memory safety of free threading. + +List operations are now memory-safe on free threaded Python builds, even in the presence of +race conditions. This has some performance cost. For list-heavy workloads, using +`librt.vecs.vec` instead of list is often significantly faster, but note that `vec` is not +(and likely won't be) fully memory safe, and the user is expected to avoid race conditions. +The newly introduced `librt.threading.Lock` helps with this. Using variable-length tuples +can also be more efficient than lists, since tuples are immutable and don't require +expensive synchronization to ensure memory safety. + +Instance attribute access is also (mostly) memory safe now on free-threaded builds in +the presence of race conditions. We are planning to fix the remaining unsafe cases in a +future release. + +Full list of changes: + +- Make attribute access memory safe on free-threaded builds (Jukka Lehtosalo, PR [21705](https://github.com/python/mypy/pull/21705)) +- Fix unsafe borrowing of instance attributes with free-threading (Jukka Lehtosalo, PR [21688](https://github.com/python/mypy/pull/21688)) +- Make list get/set item more memory safe on free-threaded builds (Jukka Lehtosalo, PR [21683](https://github.com/python/mypy/pull/21683)) +- Don't borrow list items on free-threaded builds (Jukka Lehtosalo, PR [21679](https://github.com/python/mypy/pull/21679)) +- Make multiple assignment from list memory-safe on free-threaded builds (Jukka Lehtosalo, PR [21684](https://github.com/python/mypy/pull/21684)) +- Make `for` loop over list memory-safe on free-threaded builds (Jukka Lehtosalo, PR [21686](https://github.com/python/mypy/pull/21686)) +- Fix memory safety of `list.count` on free-threaded builds (Jukka Lehtosalo, PR [21680](https://github.com/python/mypy/pull/21680)) +- Make `vec` creation from list memory safe on free-threaded builds (Jukka Lehtosalo, PR [21681](https://github.com/python/mypy/pull/21681)) + +### librt.threading: Fast Native Lock Type + +Mypyc now supports `librt.threading.Lock`, which is a lock type optimized for use +in compiled code. It can be 2x to 4x faster than `threading.Lock`. + +This feature was contributed by Jukka Lehtosalo (PR [21690](https://github.com/python/mypy/pull/21690), PR [21697](https://github.com/python/mypy/pull/21697)). + +### Mypyc: Read-only Final Instance Attributes + +Instance attributes of native classes declared as `Final` are now read-only at runtime. +This enables additional optimizations, and it's now recommended to use `Final` for +all performance-sensitive attributes when feasible. + +Related changes: + +- Make instance attribute read-only at runtime if `Final` (Jukka Lehtosalo, PR [21666](https://github.com/python/mypy/pull/21666)) +- Borrow final attributes more aggressively (Jukka Lehtosalo, PR [21702](https://github.com/python/mypy/pull/21702)) +- Improve documentation of `Final` in mypyc (Jukka Lehtosalo, PR [21713](https://github.com/python/mypy/pull/21713)) + +### Mypyc Documentation Updates + +- Update documentation of race conditions under free threading (Jukka Lehtosalo, PR [21726](https://github.com/python/mypy/pull/21726)) +- Update mypyc free threading Python compatibility docs (Jukka Lehtosalo, PR [21711](https://github.com/python/mypy/pull/21711)) +- Document recent additions to `librt.strings`, such as `ispace` (Jukka Lehtosalo, PR [21696](https://github.com/python/mypy/pull/21696)) + +### Miscellaneous Mypyc Improvements + +- Fix reference leak when setting unboxed refcounted attributes (Tom Bannink, PR [21657](https://github.com/python/mypy/pull/21657)) +- Fix function wrapper memory leak (Piotr Sawicki, PR [21654](https://github.com/python/mypy/pull/21654)) +- Fix handling of invalid codepoint values in `librt.strings` (Jukka Lehtosalo, PR [21634](https://github.com/python/mypy/pull/21634)) +- Fix non-deterministic ordering of spilled registers (Jukka Lehtosalo, PR [21632](https://github.com/python/mypy/pull/21632)) +- Fix non-deterministic compiler output due to frozensets (Jukka Lehtosalo, PR [21631](https://github.com/python/mypy/pull/21631)) + +### Changes to Messages + +- Fix error code of note about unbound type variable (Jukka Lehtosalo, PR [21668](https://github.com/python/mypy/pull/21668)) + +### Other Notable Fixes and Improvements + +- Use `PYODIDE` environment variable for Emscripten cross-compilation detection (Agriya Khetarpal, PR [21714](https://github.com/python/mypy/pull/21714)) +- Narrow for frozendict membership check (Shantanu, PR [21709](https://github.com/python/mypy/pull/21709)) +- Fix custom equality handling for membership narrowing in static containers (Shantanu, PR [21706](https://github.com/python/mypy/pull/21706)) +- Infer `Coroutine` for unannotated async functions (Jingchen Ye, PR [21651](https://github.com/python/mypy/pull/21651)) +- Fix variance inference issues caused by dataclass replace (Shantanu, PR [21694](https://github.com/python/mypy/pull/21694)) +- Fix regression in dataclass narrowing for Python >= 3.13 (ygale, PR [21675](https://github.com/python/mypy/pull/21675)) +- Fix star import dependencies in mypy daemon (Jukka Lehtosalo, PR [21673](https://github.com/python/mypy/pull/21673)) +- Fix skipped imports considered stale (Piotr Sawicki, PR [21639](https://github.com/python/mypy/pull/21639)) +- Support `.ff` files with `--cache-map` (Jukka Lehtosalo, PR [21633](https://github.com/python/mypy/pull/21633)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=f76037a1eb3923c67a8bc0e302ee9c016ffb3431+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Agriya Khetarpal +- Ethan Sarp +- Ivan Levkivskyi +- Jingchen Ye +- Jukka Lehtosalo +- Piotr Sawicki +- Shantanu +- Tom Bannink +- Viktor Szépe +- ygale + +I'd also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 2.2 We've just uploaded mypy 2.2.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 8aabf8435357eaffceca7237f371e293b8168e54 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 13 Jul 2026 10:50:09 +0100 Subject: [PATCH 44/50] Drop +dev from version --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 13a4418314c25..dd5800e0daced 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "2.3.0+dev" +__version__ = "2.3.0" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From a3857467da126d28b55724e8bb682019df9a503e Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 14 Aug 2026 17:18:58 -0700 Subject: [PATCH 45/50] Bump version to 2.3.1+dev --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index dd5800e0daced..c0df0f0d598be 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "2.3.0" +__version__ = "2.3.1+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 6dfa06dda6e34912279e498d35a43ba6dc30bfee Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 10 Aug 2026 06:22:37 -0700 Subject: [PATCH 46/50] Fix crash when unpacking return value from overload (#21830) Fixes #21824 Fixes #19920 Closes #19921 Note that ilevkivskyi has a suggestion to change overload inference here: https://github.com/python/mypy/issues/19920#issuecomment-3341557826 While that is right, it is a little separate, and I think it is okay to remove the crash given that it now affects numpy and has been reported in other contexts too. We just keep the originally inferred tuple --- mypy/checker.py | 6 ++++-- test-data/unit/check-tuples.test | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 612d2face5b8a..122a7512b44c4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4408,8 +4408,10 @@ def check_multi_assignment_from_tuple( # inferred return type for an overloaded function # to be ambiguous. return - assert isinstance(reinferred_rvalue_type, TupleType) - rvalue_type = reinferred_rvalue_type + if isinstance(reinferred_rvalue_type, TupleType): + # This branch will usually be taken, but in some cases context can + # e.g. select a different overload + rvalue_type = reinferred_rvalue_type left_rv_types, star_rv_types, right_rv_types = self.split_around_star( rvalue_type.items, star_index, len(lvalues) diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index bfbd2e631f5d8..79c6f630f4d98 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -426,6 +426,42 @@ class A: pass class B: pass [builtins fixtures/tuple.pyi] +[case testMultipleAssignmentWithOverloadReinferredAsHomogeneousTuple] +from typing import Any, TypeVar, overload + +T = TypeVar("T") + +@overload +def f(value: T) -> tuple[T, int]: ... +@overload +def f(*values: object) -> tuple[Any, ...]: ... +def f(*values: object, **kwargs: object) -> object: ... + +def g() -> None: + first: str + first, second = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(second) # N: Revealed type is "builtins.int" + + last: str + _, last = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithOverloadReinferredAsNonTuple] +from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def f(value: T) -> tuple[T]: ... +@overload +def f(*values: object) -> int: ... +def f(*values: object, **kwargs: object) -> object: ... + +def g() -> None: + value: str + (value,) = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + [case testMultipleAssignmentWithSquareBracketTuples] # flags: --no-strict-optional from typing import Tuple From 14f5df93ed8d1be4f4cc9c447eb2e6e619362e05 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 17 Jul 2026 18:17:02 +0200 Subject: [PATCH 47/50] [mypyc] Clear coroutine env on coroutine completion (#21734) The `env_class` object associated with a mypyc coroutine is not immediately cleared when the coroutine completes. This can significantly increase memory usage, since the env class may hold references to captured locals and values spilled across suspension points. The objects are eventually collectible by the GC but the collection might be delayed in cases where a nested coroutine is awaited, eg. ```python async def allocate(size: int) -> None: payload = bytearray(size) async def nested() -> int: return payload[-1] assert await nested() == 0 ``` With nesting mypyc creates env classes for both `allocate` and `nested` that may reference each other and form a cycle. To fix this, clear the `env_class` immediately after the coroutine completes. This matches behavior of cpython, which clears frames of completed coroutines immediately in the eval loop. --- mypyc/codegen/emitclass.py | 29 +++-- mypyc/ir/class_ir.py | 25 +++- mypyc/irbuild/builder.py | 3 + mypyc/irbuild/env_class.py | 44 ++++++- mypyc/irbuild/generator.py | 10 +- mypyc/irbuild/nonlocalcontrol.py | 27 ++++- mypyc/test-data/run-async.test | 199 +++++++++++++++++++++++++++++++ 7 files changed, 318 insertions(+), 19 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index a1127b15ad9d1..aeea3374353f9 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -252,7 +252,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: getseters_name = f"{name_prefix}_getseters" vtable_name = f"{name_prefix}_vtable" traverse_name = f"{name_prefix}_traverse" - clear_name = f"{name_prefix}_clear" + clear_name = emitter.native_function_name(cl.clear) dealloc_name = f"{name_prefix}_dealloc" methods_name = f"{name_prefix}_methods" vtable_setup_name = f"{name_prefix}_trait_vtable_setup" @@ -271,7 +271,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: fields["tp_dealloc"] = f"(destructor){name_prefix}_dealloc" if not cl.is_acyclic: fields["tp_traverse"] = f"(traverseproc){name_prefix}_traverse" - fields["tp_clear"] = f"(inquiry){name_prefix}_clear" + fields["tp_clear"] = f"(inquiry){clear_name}" # Populate .tp_finalize and generate a finalize method only if __del__ is defined for this class. del_method = next((e.method for e in cl.vtable_entries if e.name == "__del__"), None) if del_method: @@ -344,7 +344,11 @@ def emit_line() -> None: if not cl.is_acyclic: generate_traverse_for_class(cl, traverse_name, emitter) emit_line() - generate_clear_for_class(cl, clear_name, emitter) + generate_clear_for_class(cl, cl.clear, emitter) + emit_line() + generate_clear_for_class( + cl, cl.clear_on_completion, emitter, skip_attrs=cl.attrs_to_keep_alive_on_completion + ) emit_line() generate_dealloc_for_class(cl, dealloc_name, clear_name, bool(del_method), emitter) emit_line() @@ -895,12 +899,21 @@ def generate_traverse_for_class(cl: ClassIR, func_name: str, emitter: Emitter) - emitter.emit_line("}") -def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> None: - emitter.emit_line("static int") - emitter.emit_line(f"{func_name}({cl.struct_name(emitter.names)} *self)") +def generate_clear_for_class( + cl: ClassIR, func_decl: FuncDecl, emitter: Emitter, skip_attrs: set[str] | None = None +) -> None: + if skip_attrs is None: + skip_attrs = set() + emitter.emit_line("static " + native_function_header(func_decl, emitter)) emitter.emit_line("{") + emitter.emit_line( + f"{cl.struct_name(emitter.names)} *self = " + f"({cl.struct_name(emitter.names)} *)cpy_r_self;" + ) for base in reversed(cl.base_mro): for attr, rtype in base.attributes.items(): + if attr in skip_attrs: + continue emitter.emit_gc_clear(f"self->{emitter.attr(attr)}", rtype) base_args = "(PyObject *)self" if cl.builtin_base: @@ -951,7 +964,7 @@ def generate_dealloc_for_class( if not cl.is_acyclic: emitter.emit_line("PyObject_GC_UnTrack(self);") if cl.builtin_base: - emitter.emit_line(f"{clear_func_name}(self);") + emitter.emit_line(f"{clear_func_name}((PyObject *)self);") # For native subclasses of builtins such as dict, the base deallocator # is responsible for tearing down base-owned storage and freeing memory. # Re-track self if base is GC-aware to match cpython's subtype_dealloc. @@ -966,7 +979,7 @@ def generate_dealloc_for_class( emit_reuse_dealloc(cl, emitter) # The trashcan is needed to handle deep recursive deallocations emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") - emitter.emit_line(f"{clear_func_name}(self);") + emitter.emit_line(f"{clear_func_name}((PyObject *)self);") emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);") emitter.emit_line("CPy_TRASHCAN_END(self)") emitter.emit_line("done: ;") diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 6ea1eb072999d..3b54331cb2f07 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -7,7 +7,7 @@ from mypyc.common import PROPSET_PREFIX, JsonDict from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import DeserMaps, Value -from mypyc.ir.rtypes import RInstance, RType, deserialize_type, object_rprimitive +from mypyc.ir.rtypes import RInstance, RType, c_int_rprimitive, deserialize_type, object_rprimitive from mypyc.namegen import NameGenerator, exported_name # Some notes on the vtable layout: Each concrete class has a vtable @@ -143,8 +143,25 @@ def __init__( module_name, FuncSignature([RuntimeArg("type", object_rprimitive)], RInstance(self)), ) + self.clear = FuncDecl( + name + "_clear", + None, + module_name, + FuncSignature([RuntimeArg("self", RInstance(self))], c_int_rprimitive), + internal=True, + ) + self.clear_on_completion = FuncDecl( + name + "_clear_on_completion", + None, + module_name, + FuncSignature([RuntimeArg("self", RInstance(self))], c_int_rprimitive), + internal=True, + ) # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} + # Attributes that must survive generator/coroutine completion because + # escaped nested functions may still read them as closure variables. + self.attrs_to_keep_alive_on_completion: set[str] = set() # Final attributes defined in the class (not inherited) self.final_attributes: set[str] = set() # Deletable attributes @@ -412,8 +429,11 @@ def serialize(self) -> JsonDict: "_serializable": self._serializable, "builtin_base": self.builtin_base, "ctor": self.ctor.serialize(), + "clear": self.clear.serialize(), + "clear_on_completion": self.clear_on_completion.serialize(), # We serialize dicts as lists to ensure order is preserved "attributes": [(k, t.serialize()) for k, t in self.attributes.items()], + "attrs_to_keep_alive_on_completion": sorted(self.attrs_to_keep_alive_on_completion), "final_attributes": sorted(self.final_attributes), # We try to serialize a name reference, but if the decl isn't in methods # then we can't be sure that will work so we serialize the whole decl. @@ -474,7 +494,10 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir._serializable = data["_serializable"] ir.builtin_base = data["builtin_base"] ir.ctor = FuncDecl.deserialize(data["ctor"], ctx) + ir.clear = FuncDecl.deserialize(data["clear"], ctx) + ir.clear_on_completion = FuncDecl.deserialize(data["clear_on_completion"], ctx) ir.attributes = {k: deserialize_type(t, ctx) for k, t in data["attributes"]} + ir.attrs_to_keep_alive_on_completion = set(data["attrs_to_keep_alive_on_completion"]) ir.final_attributes = set(data["final_attributes"]) ir.method_decls = { k: ctx.functions[v].decl if isinstance(v, str) else FuncDecl.deserialize(v, ctx) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 0b598a00889f5..f4e3745836cf5 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1568,12 +1568,15 @@ def add_var_to_env_class( base: FuncInfo | ImplicitClass, reassign: bool = False, always_defined: bool = False, + keep_alive_on_completion: bool = False, prefix: str = "", ) -> AssignmentTarget: # First, define the variable name as an attribute of the environment class, and then # construct a target for that attribute. name = prefix + remangle_redefinition_name(var.name) self.fn_info.env_class.attributes[name] = rtype + if keep_alive_on_completion: + self.fn_info.env_class.attrs_to_keep_alive_on_completion.add(name) if always_defined: self.fn_info.env_class.attrs_with_defaults.add(name) attr_target = AssignmentTargetAttr(base.curr_env_reg, name) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 3128543d4cd2c..c8c5f7fa68593 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -17,7 +17,7 @@ def g() -> int: from __future__ import annotations -from mypy.nodes import Argument, FuncDef, SymbolNode, Var +from mypy.nodes import Argument, FuncDef, FuncItem, SymbolNode, Var from mypyc.common import ( BITMAP_BITS, ENV_ATTR_NAME, @@ -60,6 +60,8 @@ class is generated, the function environment has not yet been # If the function is nested, its environment class must contain an environment # attribute pointing to its encapsulating functions' environment class. env_class.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + if builder.fn_info.contains_nested: + env_class.attrs_to_keep_alive_on_completion.add(ENV_ATTR_NAME) env_class.mro = [env_class] builder.fn_info.env_class = env_class builder.classes.append(env_class) @@ -229,7 +231,17 @@ def add_args_to_env( rtype = builder.type_to_rtype(arg.variable.type) assert base is not None, "base cannot be None for adding nonlocal args" builder.add_var_to_env_class( - arg.variable, rtype, base, reassign=reassign, prefix=prefix + arg.variable, + rtype, + base, + reassign=reassign, + keep_alive_on_completion=( + is_free_variable(builder, arg.variable) + or is_free_variable_in_nested_func( + builder, builder.fn_info.fitem, arg.variable + ) + ), + prefix=prefix, ) @@ -256,7 +268,12 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None: if isinstance(var, Var): rtype = builder.type_to_rtype(var.type) builder.add_var_to_env_class( - var, rtype, env_for_func, reassign=False, prefix=prefix + var, + rtype, + env_for_func, + reassign=False, + keep_alive_on_completion=True, + prefix=prefix, ) if builder.fn_info.fitem in builder.encapsulating_funcs: @@ -267,10 +284,16 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None: # the same name and signature across conditional blocks # will generate different callable classes, so the callable # class that gets instantiated must be generic. + nested_prefix = prefix if nested_fn.is_generator or nested_fn.is_coroutine: - prefix = GENERATOR_ATTRIBUTE_PREFIX + nested_prefix = GENERATOR_ATTRIBUTE_PREFIX builder.add_var_to_env_class( - nested_fn, object_rprimitive, env_for_func, reassign=False, prefix=prefix + nested_fn, + object_rprimitive, + env_for_func, + reassign=False, + keep_alive_on_completion=is_free_variable(builder, nested_fn), + prefix=nested_prefix, ) @@ -308,3 +331,14 @@ def setup_func_for_recursive_call( def is_free_variable(builder: IRBuilder, symbol: SymbolNode) -> bool: fitem = builder.fn_info.fitem return fitem in builder.free_variables and symbol in builder.free_variables[fitem] + + +def is_free_variable_in_nested_func( + builder: IRBuilder, fitem: FuncItem, symbol: SymbolNode +) -> bool: + for nested in builder.encapsulating_funcs.get(fitem, []): + if symbol in builder.free_variables.get(nested, set()): + return True + if is_free_variable_in_nested_func(builder, nested, symbol): + return True + return False diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 82fc74f9b1ce5..3e7f18c91c753 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -23,6 +23,7 @@ Call, Goto, Integer, + LoadErrorValue, MethodCall, RaiseStandardError, Register, @@ -49,7 +50,7 @@ load_outer_envs, setup_func_for_recursive_call, ) -from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl +from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl, gen_generator_func_cleanup from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.primitives.exc_ops import ( error_catch_op, @@ -107,6 +108,8 @@ class that implements the function (each function gets a separate class). setup_func_for_recursive_call( builder, fitem, builder.fn_info.generator_class, prefix=GENERATOR_ATTRIBUTE_PREFIX ) + cleanup_on_error = BasicBlock() + builder.builder.push_error_handler(cleanup_on_error) create_switch_for_generator_class(builder) add_raise_exception_blocks_to_generator_class(builder, fitem.line) @@ -114,6 +117,11 @@ class that implements the function (each function gets a separate class). builder.accept(fitem.body) builder.maybe_add_implicit_return() + builder.builder.pop_error_handler() + + builder.activate_block(cleanup_on_error) + gen_generator_func_cleanup(builder, fitem.line) + builder.add(Return(builder.add(LoadErrorValue(object_rprimitive)))) populate_switch_for_generator_class(builder) diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index e0a23769770d7..009a1d6a986bc 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -8,10 +8,13 @@ from abc import abstractmethod from typing import TYPE_CHECKING +from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import ( + ERR_NEVER, NO_TRACEBACK_LINE_NO, BasicBlock, Branch, + Call, Goto, Integer, Register, @@ -90,10 +93,7 @@ class GeneratorNonlocalControl(BaseNonlocalControl): """Default nonlocal control in a generator function outside statements.""" def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: - # Assign an invalid next label number so that the next time - # __next__ is called, we jump to the case in which - # StopIteration is raised. - builder.assign(builder.fn_info.generator_class.next_label_target, Integer(-1), line) + gen_generator_func_cleanup(builder, line) # Raise a StopIteration containing a field for the value that # should be returned. Before doing so, create a new block @@ -132,6 +132,25 @@ def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: builder.add(Return(Integer(0, object_rprimitive))) +def gen_class_clear_on_completion(builder: IRBuilder, obj: Value, cl: ClassIR, line: int) -> None: + call = Call(cl.clear_on_completion, [obj], line) + call.error_kind = ERR_NEVER + builder.add(call) + + +def gen_generator_func_cleanup(builder: IRBuilder, line: int) -> None: + """Clear references held by a completed generator or coroutine.""" + cls = builder.fn_info.generator_class + + # Assign an invalid next label number so that the next time __next__ is + # called, we jump to the case in which StopIteration is raised. + builder.assign(cls.next_label_target, Integer(-1), line) + + gen_class_clear_on_completion(builder, cls.curr_env_reg, builder.fn_info.env_class, line) + if builder.fn_info.env_class is not cls.ir: + gen_class_clear_on_completion(builder, cls.self_reg, cls.ir, line) + + class CleanupNonlocalControl(NonlocalControl): """Abstract nonlocal control that runs some cleanup code.""" diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index a7a08e7ced5f2..4c9fca54f4a2f 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -407,6 +407,12 @@ import asyncio import gc import platform +from testutil import is_gil_disabled + +def immediate_deallocation_checks_are_reliable() -> bool: + # Free-threaded builds can defer deallocation, so weakref liveness is not deterministic. + return not is_gil_disabled() + async def sleep() -> None: # Non-zero value hangs on Windows. time = 0 if platform.system() == "Windows" else 0.0001 @@ -512,6 +518,106 @@ async def stolen(n: int) -> int: async def test_stolen() -> None: await assert_no_leaks(lambda: stolen(200), 16) +async def retain_arg(c: set[int]) -> None: + async def nested() -> int: + await sleep() + return len(c) + + await nested() + +async def test_completed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + + c = {1} + ref = weakref.ref(c) + coro = retain_arg(c) + c = {2} + await coro + assert ref() is None + +async def retain_arg_then_raise(c: set[int]) -> None: + async def nested() -> int: + await sleep() + return len(c) + + await nested() + raise ValueError + +async def test_failed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + + c = {1} + ref = weakref.ref(c) + coro = retain_arg_then_raise(c) + c = {2} + try: + await coro + except ValueError: + pass + assert ref() is None + +async def close_retain_arg(c: set[int]) -> None: + await sleep() + len(c) + +async def test_closed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + from typing import Any, cast + + c = {1} + ref = weakref.ref(c) + coro = cast(Any, close_retain_arg(c)) + coro.send(None) + c = {2} + coro.close() + assert ref() is None + +async def interpreted_leaf(c: set[int]) -> int: + await sleep() + return len(c) + +async def compiled_leaf(c: set[int]) -> int: + await sleep() + return len(c) + +async def compiled_awaits_interpreted(c: set[int]) -> int: + import interpreted + + return await interpreted.interpreted_leaf(c) + +async def interpreted_awaits_compiled(fn, c: set[int]) -> int: + return await fn(c) + +async def test_completed_mixed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import interpreted + import weakref + + c = {1} + ref = weakref.ref(c) + coro = compiled_awaits_interpreted(c) + c = {2} + assert await coro == 1 + assert ref() is None + + c = {3} + ref = weakref.ref(c) + coro = interpreted.interpreted_awaits_compiled(compiled_leaf, c) + c = {4} + assert await coro == 1 + assert ref() is None + [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... @@ -1998,6 +2104,99 @@ for i in range(50): assert yields == ("hello",), yields assert val == "value" + str(i) + "world", repr(val) +[case testCoroutineClosureOutlivesCompletedCoroutine] +import asyncio +from typing import Callable + +async def make_reader() -> Callable[[], int]: + payload = [7] + await asyncio.sleep(0) + + def read_payload() -> int: + return payload[0] + + return read_payload + +async def make_arg_reader(payload: list[int]) -> Callable[[], int]: + await asyncio.sleep(0) + + def read_payload() -> int: + return payload[0] + + return read_payload + +async def make_indirect_reader() -> Callable[[], int]: + def leaf() -> int: + return 17 + + def reader() -> int: + return leaf() + + await asyncio.sleep(0) + return reader + +def test_closure_returned_from_completed_coroutine_keeps_capture() -> None: + read_payload = asyncio.run(make_reader()) + assert read_payload() == 7 + + payload = [13] + read_arg_payload = asyncio.run(make_arg_reader(payload)) + payload = [0] + assert read_arg_payload() == 13 + + indirect_reader = asyncio.run(make_indirect_reader()) + assert indirect_reader() == 17 + +async def test_closure_returned_from_awaited_coroutine_keeps_capture() -> None: + read_payload = await make_reader() + assert read_payload() == 7 + + payload = [13] + read_arg_payload = await make_arg_reader(payload) + payload = [0] + assert read_arg_payload() == 13 + + indirect_reader = await make_indirect_reader() + assert indirect_reader() == 17 + +[file asyncio/__init__.pyi] +from typing import Awaitable, TypeVar + +T = TypeVar("T") + +def run(x: Awaitable[T]) -> T: ... +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[case testNestedCoroutineOutlivesCompletedCoroutine] +import asyncio +from typing import Awaitable + +async def make_nested_coroutine() -> Awaitable[int]: + payload = [11] + await asyncio.sleep(0) + + async def read_payload() -> int: + await asyncio.sleep(0) + return payload[0] + + return read_payload() + +def test_nested_coroutine_returned_from_completed_coroutine_keeps_capture() -> None: + read_payload = asyncio.run(make_nested_coroutine()) + assert asyncio.run(read_payload) == 11 + +[file asyncio/__init__.pyi] +from typing import Awaitable, TypeVar + +T = TypeVar("T") + +def run(x: Awaitable[T]) -> T: ... +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + [case testBorrowedFinalAttrAcrossAsyncComprehension] import asyncio from typing import Final From 4843e7773e7dc8fe3f1fd1319277d6d11cd6cdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:33:26 +0200 Subject: [PATCH 48/50] [mypyc] Fix `default_factory` for inherited dataclass (#21785) Closes https://github.com/mypyc/mypyc/issues/1204, a `mypyc` issue. The issue likely unblocks `isort` from being compiled with `mypyc`, which is why I took an interest it. Full disclosure is that I used AI to write this, it provided this test and code just from the linked issue but it looks sensible to me. I did not use additional prompting. Like I said, fix looks sensible: we now call `dataclass_type` for each entry in the MRO, instead of once. I am aware the contributing guidelines say new contributors are not encourage to use LLMs but I hope the size of the PR and my track record as open source maintainer gives some flexibility here. I'm very aware that reviewing AI written PRs creates maintainer burden, but I'm committed to getting this over the finish line. To that end I: Tested locally with `pytest mypyc/test/test_run.py ` and that looked okay. Also tested that without the changes the added test would fail on `master`. Also tried to run CI on my local fork (https://github.com/DanielNoord/mypy/pull/1), which succeeded. Feel free to push changes to the branch or cherry pick this into another branch. I am not necessarily interested in getting credits for the fix/PR, I'd just like to continue with my attempt to get `isort` compiled and for that I need this in a `mypy` release :) --- mypyc/irbuild/classdef.py | 4 ++-- mypyc/test-data/run-python37.test | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 4fd9a2e2cc748..61587b46161f7 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -760,7 +760,6 @@ def find_attr_initializers( if cls.builtin_base: return set(), [] - cls_type = dataclass_type(cdef) attrs_with_defaults: set[str] = set() default_assignments: list[tuple[AssignmentStmt, str]] = [] @@ -774,10 +773,11 @@ def find_attr_initializers( info_ir = builder.mapper.type_to_ir.get(info) if info_ir is None: continue + info_cls_type = dataclass_type(info.defn) for stmt in info.defn.defs.body: if not isinstance(stmt, AssignmentStmt): continue - name = default_attr_name(stmt, info_ir, cls_type) + name = default_attr_name(stmt, info_ir, info_cls_type) if name is None: continue attrs_with_defaults.add(name) diff --git a/mypyc/test-data/run-python37.test b/mypyc/test-data/run-python37.test index 61d428c17a441..1245ad516a2d9 100644 --- a/mypyc/test-data/run-python37.test +++ b/mypyc/test-data/run-python37.test @@ -74,8 +74,15 @@ class Person5: friends: Set[str] = field(default_factory=set) parents: FrozenSet[str] = frozenset() +@dataclass +class HasFactoryDefault: + x: dict[str, int] = field(default_factory=dict) + +class InheritsDataclassInit(HasFactoryDefault): + pass + [file other.py] -from native import Person1, Person1b, Person2, Person3, Person4, Person5, testBool +from native import Person1, Person1b, Person2, Person3, Person4, Person5, testBool, InheritsDataclassInit i1 = Person1(age = 5, name = 'robot') assert i1.age == 5 assert i1.name == 'robot' @@ -125,6 +132,9 @@ assert Person1.__annotations__ == {'age': int, 'name': str} assert Person2.__annotations__ == {'age': int, 'name': str} assert Person5.__annotations__ == {'weight': float, 'friends': set, 'parents': frozenset} +v = InheritsDataclassInit() +assert isinstance(v.x, dict) +assert v.x == {} [file driver.py] import sys From a39242983d3c2cb85886a1eb6d5869180672784c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:27:46 +0200 Subject: [PATCH 49/50] [mypyc] Fix crash on double yielding Iterators (#21826) Closes https://github.com/mypyc/mypyc/issues/1210 This fixes another issue I ran into while trying to use `mypyc` for `isort`. I am aware the contributing guidelines say new contributors are not encouraged to use LLMs but I hope this can get the same handling as https://github.com/python/mypy/pull/21785. I have tried to ensure this patch works as much as I can by: Testing locally with `pytest mypyc/test/test_run.py `. Tested that the test fails on `master` without the changes. Also tried to run CI on my local fork (https://github.com/DanielNoord/mypy/pull/2), which succeeded. As with the previous PR, feel free to push changes to the branch or cherry pick this into another branch. I just want to unblock `isort` :) --- mypyc/irbuild/generator.py | 6 ++++++ mypyc/test-data/run-generators.test | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 3e7f18c91c753..555baaada0e81 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -174,6 +174,12 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: builder.fn_info.env_class = generator_class_ir else: generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) + if not builder.fn_info.fitem.is_coroutine: + # After completion generators still need generator.__mypyc_env__ for subsequent + # __next__() calls to observe the terminal next-label and raise StopIteration. + # Coroutines can't be resumed after completion, so keeping the environment alive + # there would just extend local lifetimes unnecessarily. + generator_class_ir.attrs_to_keep_alive_on_completion.add(ENV_ATTR_NAME) builder.classes.append(generator_class_ir) return generator_class_ir diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index a23af2ed6eea7..f4cef93ea8f3e 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1,7 +1,7 @@ # Test cases for generators and yield (compile and run) [case testYield] -from typing import Generator, Iterable, Union, Tuple, Dict +from typing import Callable, Dict, Generator, Iterable, Iterator, Tuple, Union def yield_three_times() -> Iterable[int]: yield 1 @@ -87,6 +87,19 @@ def return_tuple() -> Generator[int, None, Tuple[int, int]]: yield 0 return 1, 2 +def call_lambda(fn: Callable[[], None]) -> None: + fn() + +def get_iterator() -> Iterator[str]: + call_lambda(lambda: None) + yield "" + +def yield_twice_via_generator_reuse() -> Iterator[str]: + identified_imports = get_iterator() + yield from identified_imports + for identified_import in identified_imports: + yield identified_import + [file driver.py] from native import ( yield_three_times, @@ -99,6 +112,7 @@ from native import ( A, return_tuple, yield_dict_methods, + yield_twice_via_generator_reuse, ) from testutil import run_generator from collections import defaultdict @@ -114,6 +128,7 @@ assert run_generator(A(0).generator()) == ((0,), None) assert run_generator(return_tuple()) == ((0,), (1, 2)) assert run_generator(yield_dict_methods({}, {}, {})) == ((), None) assert run_generator(yield_dict_methods({1: 2}, {3: 4}, {5: 6})) == ((1, 3, 4, 6), None) +assert run_generator(yield_twice_via_generator_reuse()) == (("",), None) dd = defaultdict(int, {0: 1}) assert run_generator(yield_dict_methods(dd, dd, dd)) == ((0, 0, 1, 1), None) From d642c4478e9e3acbe9233edbe17ffc569a1a778c Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 14 Aug 2026 17:44:15 -0700 Subject: [PATCH 50/50] Bump version to 2.3.1 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index c0df0f0d598be..b4d4e0d35d054 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "2.3.1+dev" +__version__ = "2.3.1" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))