diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 361290e55a1d1..548b568621a70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: codespell args: - - --ignore-words-list=HAX,Nam,ccompiler,ot,statics,whet,zar + - --ignore-words-list=HAX,Nam,ccompiler,keep-alives,ot,statics,whet,zar exclude: ^(mypy/test/|mypy/typeshed/|mypyc/test-data/|test-data/).+$ - repo: https://github.com/rhysd/actionlint rev: v1.7.7 diff --git a/CHANGELOG.md b/CHANGELOG.md index ac2cb8036dc71..efe63effbfb4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,399 @@ ## Next Release +## Mypy 2.3 + +We've just uploaded mypy 2.3.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### The Upcoming Switch to the New Native Parser + +We are planning to enable the new native parser (`--native-parser`) by +default soon. We recommend that you test the native parser in your projects and report +any issues in the [mypy issue tracker](https://github.com/python/mypy/issues). + +### Mypyc Free-threading Memory Safety + +Free-threaded Python builds that don't have the GIL require additional synchronization +primitives or lock-free algorithms to ensure memory safety when there are race conditions +(for example, when a thread reads a list item while another thread writes the same list +item concurrently). This release greatly improves memory safety of free threading. + +List operations are now memory-safe on free threaded Python builds, even in the presence of +race conditions. This has some performance cost. For list-heavy workloads, using +`librt.vecs.vec` instead of list is often significantly faster, but note that `vec` is not +(and likely won't be) fully memory safe, and the user is expected to avoid race conditions. +The newly introduced `librt.threading.Lock` helps with this. Using variable-length tuples +can also be more efficient than lists, since tuples are immutable and don't require +expensive synchronization to ensure memory safety. + +Instance attribute access is also (mostly) memory safe now on free-threaded builds in +the presence of race conditions. We are planning to fix the remaining unsafe cases in a +future release. + +Full list of changes: + +- Make attribute access memory safe on free-threaded builds (Jukka Lehtosalo, PR [21705](https://github.com/python/mypy/pull/21705)) +- Fix unsafe borrowing of instance attributes with free-threading (Jukka Lehtosalo, PR [21688](https://github.com/python/mypy/pull/21688)) +- Make list get/set item more memory safe on free-threaded builds (Jukka Lehtosalo, PR [21683](https://github.com/python/mypy/pull/21683)) +- Don't borrow list items on free-threaded builds (Jukka Lehtosalo, PR [21679](https://github.com/python/mypy/pull/21679)) +- Make multiple assignment from list memory-safe on free-threaded builds (Jukka Lehtosalo, PR [21684](https://github.com/python/mypy/pull/21684)) +- Make `for` loop over list memory-safe on free-threaded builds (Jukka Lehtosalo, PR [21686](https://github.com/python/mypy/pull/21686)) +- Fix memory safety of `list.count` on free-threaded builds (Jukka Lehtosalo, PR [21680](https://github.com/python/mypy/pull/21680)) +- Make `vec` creation from list memory safe on free-threaded builds (Jukka Lehtosalo, PR [21681](https://github.com/python/mypy/pull/21681)) + +### librt.threading: Fast Native Lock Type + +Mypyc now supports `librt.threading.Lock`, which is a lock type optimized for use +in compiled code. It can be 2x to 4x faster than `threading.Lock`. + +This feature was contributed by Jukka Lehtosalo (PR [21690](https://github.com/python/mypy/pull/21690), PR [21697](https://github.com/python/mypy/pull/21697)). + +### Mypyc: Read-only Final Instance Attributes + +Instance attributes of native classes declared as `Final` are now read-only at runtime. +This enables additional optimizations, and it's now recommended to use `Final` for +all performance-sensitive attributes when feasible. + +Related changes: + +- Make instance attribute read-only at runtime if `Final` (Jukka Lehtosalo, PR [21666](https://github.com/python/mypy/pull/21666)) +- Borrow final attributes more aggressively (Jukka Lehtosalo, PR [21702](https://github.com/python/mypy/pull/21702)) +- Improve documentation of `Final` in mypyc (Jukka Lehtosalo, PR [21713](https://github.com/python/mypy/pull/21713)) + +### Mypyc Documentation Updates + +- Update documentation of race conditions under free threading (Jukka Lehtosalo, PR [21726](https://github.com/python/mypy/pull/21726)) +- Update mypyc free threading Python compatibility docs (Jukka Lehtosalo, PR [21711](https://github.com/python/mypy/pull/21711)) +- Document recent additions to `librt.strings`, such as `ispace` (Jukka Lehtosalo, PR [21696](https://github.com/python/mypy/pull/21696)) + +### Miscellaneous Mypyc Improvements + +- Fix reference leak when setting unboxed refcounted attributes (Tom Bannink, PR [21657](https://github.com/python/mypy/pull/21657)) +- Fix function wrapper memory leak (Piotr Sawicki, PR [21654](https://github.com/python/mypy/pull/21654)) +- Fix handling of invalid codepoint values in `librt.strings` (Jukka Lehtosalo, PR [21634](https://github.com/python/mypy/pull/21634)) +- Fix non-deterministic ordering of spilled registers (Jukka Lehtosalo, PR [21632](https://github.com/python/mypy/pull/21632)) +- Fix non-deterministic compiler output due to frozensets (Jukka Lehtosalo, PR [21631](https://github.com/python/mypy/pull/21631)) + +### Changes to Messages + +- Fix error code of note about unbound type variable (Jukka Lehtosalo, PR [21668](https://github.com/python/mypy/pull/21668)) + +### Other Notable Fixes and Improvements + +- Use `PYODIDE` environment variable for Emscripten cross-compilation detection (Agriya Khetarpal, PR [21714](https://github.com/python/mypy/pull/21714)) +- Narrow for frozendict membership check (Shantanu, PR [21709](https://github.com/python/mypy/pull/21709)) +- Fix custom equality handling for membership narrowing in static containers (Shantanu, PR [21706](https://github.com/python/mypy/pull/21706)) +- Infer `Coroutine` for unannotated async functions (Jingchen Ye, PR [21651](https://github.com/python/mypy/pull/21651)) +- Fix variance inference issues caused by dataclass replace (Shantanu, PR [21694](https://github.com/python/mypy/pull/21694)) +- Fix regression in dataclass narrowing for Python >= 3.13 (ygale, PR [21675](https://github.com/python/mypy/pull/21675)) +- Fix star import dependencies in mypy daemon (Jukka Lehtosalo, PR [21673](https://github.com/python/mypy/pull/21673)) +- Fix skipped imports considered stale (Piotr Sawicki, PR [21639](https://github.com/python/mypy/pull/21639)) +- Support `.ff` files with `--cache-map` (Jukka Lehtosalo, PR [21633](https://github.com/python/mypy/pull/21633)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=f76037a1eb3923c67a8bc0e302ee9c016ffb3431+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Agriya Khetarpal +- Ethan Sarp +- Ivan Levkivskyi +- Jingchen Ye +- Jukka Lehtosalo +- Piotr Sawicki +- Shantanu +- Tom Bannink +- Viktor Szépe +- ygale + +I'd also like to thank my employer, Dropbox, for supporting mypy development. + +## Mypy 2.2 + +We've just uploaded mypy 2.2.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support for Closed TypedDicts (PEP 728) + +Mypy now supports closed TypedDicts as specified in PEP 728. A closed TypedDict cannot have extra +keys beyond those explicitly defined. This allows the type checker to determine that certain +operations are safe when they otherwise wouldn't be due to the potential presence of unknown keys. + +You can use the `closed` keyword argument with `TypedDict`: + +```python +HasName = TypedDict("HasName", {"name": str}) +HasOnlyName = TypedDict("HasOnlyName", {"name": str}, closed=True) +Movie = TypedDict("Movie", {"name": str, "year": int}) + +movie: Movie = {"name": "Nimona", "year": 2023} +has_name: HasName = movie # OK: HasName is open (default) +has_only_name: HasOnlyName = movie # Error: HasOnlyName is closed and Movie has extra "year" key +``` + +Closed TypedDicts enable more precise type checking because the type checker knows exactly which +keys are present. This is particularly useful when working with TypedDict unions or when you want +to ensure that a TypedDict conforms to an exact shape. + +The `closed` keyword also enables safe type narrowing with `in` checks: + +```python +Book = TypedDict('Book', {'book': str}, closed=True) +DVD = TypedDict('DVD', {'dvd': str}, closed=True) +type Inventory = Book | DVD + +def print_type(inventory: Inventory) -> None: + if "book" in inventory: + # Type is narrowed to Book here - safe because DVD is closed + print(inventory["book"]) + else: + # Type is narrowed to DVD here + print(inventory["dvd"]) +``` + +The `closed` keyword is also supported in class-based syntax: + +```python +class HasOnlyName(TypedDict, closed=True): + name: str +``` + +Note that closed TypedDicts are structural types, so a closed TypedDict is assignable to an open +TypedDict with the same keys, but not vice versa. + +Contributed by Alice (PR [21382](https://github.com/python/mypy/pull/21382)). + +### Complete Support for Type Variable Defaults (PEP 696) + +Mypy now has complete support for type variable defaults as specified in PEP 696. This allows you to +specify default values for type parameters in generic classes, functions, and type aliases. + +Traditional syntax (Python 3.11 and earlier): + +```python +T = TypeVar("T", default=int) # This means that if no type is specified T = int + +@dataclass +class Box(Generic[T]): + value: T | None = None + +reveal_type(Box()) # type is Box[int] +reveal_type(Box(value="Hello World!")) # type is Box[str] +``` + +New syntax (Python 3.12+): + +```python +class Box[T = int]: + def __init__(self, value: T) -> None: + self.value = value + +reveal_type(Box()) # type is Box[int] +reveal_type(Box(value="Hello World!")) # type is Box[str] +``` + +Type variable defaults work with all forms of generics, including classes, functions, and type aliases. +This release completes the implementation by fixing various edge cases involving recursive defaults, +dependencies between type variables, and interactions with variadic generics. + +Contributed by Ivan Levkivskyi (PRs [21491](https://github.com/python/mypy/pull/21491), +[21526](https://github.com/python/mypy/pull/21526), [21544](https://github.com/python/mypy/pull/21544)). + +### Respect Explicit Return Type of `__new__()` + +Mypy now respects explicitly annotated return types in `__new__()` methods. Previously, mypy would +always assume that `__new__()` returns an instance of the current class, ignoring explicit annotations. + +With this change, if you explicitly annotate a return type that differs from the implicit type, mypy +will use the explicit annotation: + +```python +class Factory: + def __new__(cls) -> Product: + return Product() + +reveal_type(Factory()) # type is Product, not Factory +``` + +Note that mypy still gives an error at the definition site if the explicit annotation is not a +subtype of the current class, since this is technically not type-safe. + +For backwards compatibility, there are two exceptions: +- If the return type is `Any`, mypy will still use the current class as the return type. +- If the explicit return type comes from a superclass and is a supertype of the implicit return type, + mypy will use the implicit (more specific) type: + +```python +class A: + def __new__(cls) -> A: ... + +reveal_type(A()) # type is A + +class B: + def __new__(cls) -> B: + return cls() + +class C(B): ... +reveal_type(C()) # type is C +``` + +This fixes several long-standing issues where explicit `__new__()` return types were ignored. + +Contributed by Ivan Levkivskyi (PR [21441](https://github.com/python/mypy/pull/21441)). + +### TypeForm Support No Longer Experimental + +Support for `TypeForm` is no longer experimental. `TypeForm` (introduced in Python 3.14) allows you +to annotate parameters that accept type expressions, providing better type checking for functions +that work with types as values. + +```python +from typing import TypeForm + +def make_list(tp: TypeForm[T]) -> list[T]: + ... + +# Correctly typed as list[int] +int_list = make_list(int) +``` + +`TypeForm` support was previously reverted from mypy 2.1 due to a performance regression, but this +has now been mitigated. + +Contributed by Ivan Levkivskyi and Jelle Zijlstra (PRs [21262](https://github.com/python/mypy/pull/21262), [21591](https://github.com/python/mypy/pull/21591), +[21459](https://github.com/python/mypy/pull/21459)). + +### Experimental WASM Wheel for Python 3.14 + +Mypy now ships an experimental WebAssembly (WASM) wheel for Python 3.14. This allows mypy to run +in WASM environments such as Pyodide and browser-based Python implementations. + +The WASM wheel is considered experimental and may have limitations compared to native builds. Please +report any issues you encounter when using mypy in WASM environments. + +Contributed by Ivan Levkivskyi (PR [21671](https://github.com/python/mypy/pull/21671)). + +### Mypyc Free-threading Improvements + +- Make function wrappers thread-safe on free-threaded builds (Jukka Lehtosalo, PR [21620](https://github.com/python/mypy/pull/21620)) +- Make list remove and index thread-safe on free-threaded builds (Jukka Lehtosalo, PR [21614](https://github.com/python/mypy/pull/21614)) +- Fix dict iteration memory safety on free-threaded builds (Jukka Lehtosalo, PR [21617](https://github.com/python/mypy/pull/21617)) +- Make some dict primitives thread-safe on free-threading builds (Jukka Lehtosalo, PR [21616](https://github.com/python/mypy/pull/21616)) +- Fix free-threading race condition in argument parsing (Jukka Lehtosalo, PR [21613](https://github.com/python/mypy/pull/21613)) +- Document free threading and other doc updates (Jukka Lehtosalo, PR [21494](https://github.com/python/mypy/pull/21494)) + +### `librt.strings` Updates + +- Add `librt.strings.toupper` and `librt.strings.tolower` codepoint primitives (Vaggelis Danias, PR [21553](https://github.com/python/mypy/pull/21553)) +- Add `librt.strings.isidentifier` codepoint primitive (Vaggelis Danias, PR [21522](https://github.com/python/mypy/pull/21522)) +- Add `librt.strings.isalpha` codepoint primitive (Vaggelis Danias, PR [21521](https://github.com/python/mypy/pull/21521)) +- Add `librt.strings.isalnum` codepoint primitive (Vaggelis Danias, PR [21509](https://github.com/python/mypy/pull/21509)) +- Add `librt.strings.isdigit` codepoint primitive (Vaggelis Danias, PR [21504](https://github.com/python/mypy/pull/21504)) +- Add `librt.strings.isspace` char primitive (Vaggelis Danias, PR [21462](https://github.com/python/mypy/pull/21462)) + +### Mypyc Improvements + +- Fix name lookup when class var and module var have the same name (Jukka Lehtosalo, PR [21594](https://github.com/python/mypy/pull/21594)) +- Report file and line number on uncaught exceptions (Jukka Lehtosalo, PR [21584](https://github.com/python/mypy/pull/21584)) +- Use `other` arg instead of `self` for RHS type (Ryan Heard, PR [21569](https://github.com/python/mypy/pull/21569)) +- Use `method_sig` to get the method signature (Ryan Heard, PR [21567](https://github.com/python/mypy/pull/21567)) +- Preserve inherited attribute defaults under `separate=True` (Jo, PR [21547](https://github.com/python/mypy/pull/21547)) +- Fix missing cross-group header deps in incremental builds (Jo, PR [21490](https://github.com/python/mypy/pull/21490)) +- Fix cross-group call to inherited `__mypyc_defaults_setup` (Jo, PR [21481](https://github.com/python/mypy/pull/21481)) +- Fix non-deterministic class struct layout under `separate=True` (Vaggelis Danias, PR [21530](https://github.com/python/mypy/pull/21530)) +- Specialize `s[i] == 'x'` to a codepoint int compare (Vaggelis Danias, PR [21579](https://github.com/python/mypy/pull/21579)) +- Fix reference leak in mypyc bytes concatenation (Colinxu2020, PR [21469](https://github.com/python/mypy/pull/21469)) + +### Fixes to Crashes + +- Fix crash on invalid recursive variadic alias (Ivan Levkivskyi, PR [21572](https://github.com/python/mypy/pull/21572)) +- Fix crashes on variadic unpacking in synthetic types (Ivan Levkivskyi, PR [21555](https://github.com/python/mypy/pull/21555)) +- Fix crash on unhandled meet variadic tuple vs instance (Ivan Levkivskyi, PR [21558](https://github.com/python/mypy/pull/21558)) +- Fix crash on deferred generic class nested in function (Ivan Levkivskyi, PR [21557](https://github.com/python/mypy/pull/21557)) +- Fix crash in new-style type alias with variadic unpack (Ivan Levkivskyi, PR [21551](https://github.com/python/mypy/pull/21551)) +- Fix various crashes on recursive type variable defaults (Ivan Levkivskyi, PR [21491](https://github.com/python/mypy/pull/21491)) +- Fix crash for empty `Annotated` type application (Rayan Salhab, PR [21503](https://github.com/python/mypy/pull/21503)) +- Fix crash on `Unpack` used without arguments in class bases (Sai Asish Y, PR [21470](https://github.com/python/mypy/pull/21470)) + +### Performance Improvements + +- Memoize the options snapshot (Kevin Kannammalil, PR [21354](https://github.com/python/mypy/pull/21354)) +- Don't include `not_ready_deps` tracking as relating to mypy internals (Kevin Kannammalil, PR [21389](https://github.com/python/mypy/pull/21389)) +- Speed up transitive dependency hash for singleton SCCs (Kevin Kannammalil, PR [21390](https://github.com/python/mypy/pull/21390)) +- Optimize typeform checks (Jelle Zijlstra, PR [21459](https://github.com/python/mypy/pull/21459)) + +### Improvements to the Native Parser + +- Support `--shadow-file` with `--native-parser` (Jukka Lehtosalo, PR [21623](https://github.com/python/mypy/pull/21623)) +- Add Python version checks to native parser (Kevin Kannammalil, PR [21539](https://github.com/python/mypy/pull/21539)) +- Allow nativeparse to parse source code directly (bzoracler, PR [21260](https://github.com/python/mypy/pull/21260)) + +### Other Notable Fixes and Improvements + +- Add function definition notes for `too many positional arguments` errors (Kevin Kannammalil, PR [21410](https://github.com/python/mypy/pull/21410)) +- Fix the exportjson tool (.ff cache to .json conversion) (Jukka Lehtosalo, PR [21628](https://github.com/python/mypy/pull/21628)) +- Support floats in JSON in fixed-format cache (Ivan Levkivskyi, PR [21603](https://github.com/python/mypy/pull/21603)) +- Update `TypedDictType.__init__` signature to preserve backward compat (Jukka Lehtosalo, PR [21590](https://github.com/python/mypy/pull/21590)) +- Fix constructor calls for union-bounded `TypeVar`s (Jingchen Ye, PR [21571](https://github.com/python/mypy/pull/21571)) +- Fix `TypedDict` indexing with literal keys in comprehensions (Jingchen Ye, PR [21556](https://github.com/python/mypy/pull/21556)) +- Correctly handle empty tuple index when unpacked (Ivan Levkivskyi, PR [21545](https://github.com/python/mypy/pull/21545)) +- Support protocol checks for self-types in tuple types (Ivan Levkivskyi, PR [21535](https://github.com/python/mypy/pull/21535)) +- Fix edge cases in variadic tuple subclasses (Ivan Levkivskyi, PR [21518](https://github.com/python/mypy/pull/21518)) +- Special-case constructor for tuple types (Ivan Levkivskyi, PR [21502](https://github.com/python/mypy/pull/21502)) +- Fix false positive "Expected TypedDict key to be string literal" for `Union[TypedDict, dict[K, V]]` (Zakir Jiwani, PR [21511](https://github.com/python/mypy/pull/21511)) +- Use explicit `Never` for type inference (Ivan Levkivskyi, PR [21497](https://github.com/python/mypy/pull/21497)) +- Narrow membership in statically known containers (Shantanu, PR [21461](https://github.com/python/mypy/pull/21461)) +- Improve negative narrowing for membership checks on tuples (Shantanu, PR [21456](https://github.com/python/mypy/pull/21456)) +- Analyze `TypedDict` decorators (Pranav Manglik, PR [21267](https://github.com/python/mypy/pull/21267)) +- Start testing Python 3.15 (Marc Mueller, PR [21439](https://github.com/python/mypy/pull/21439)) +- Improved handling of `NamedTuple`, `TypedDict`, `Enum`, and regular classes nested in functions (Ivan Levkivskyi, PR [21478](https://github.com/python/mypy/pull/21478)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=616424285beccaa76f90e87e1e922b1dc68710ca+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Adam Turner +- alicederyn +- bzoracler +- Colinxu2020 +- georgesittas +- Ivan Levkivskyi +- Jelle Zijlstra +- Jingchen Ye +- Jukka Lehtosalo +- Kevin Kannammalil +- lphuc2250gma +- Marc Mueller +- Pranav Manglik +- Rayan Salhab +- Ryan Heard +- Sai Asish Y +- Shantanu +- sobolevn +- Vaggelis Danias +- Victor Letichevsky +- Zakir Jiwani + +I'd also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 2.1 We’ve just uploaded mypy 2.1.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). @@ -449,7 +842,7 @@ Contributed by Marc Mueller (PR [20156](https://github.com/python/mypy/pull/2015 For best performance, mypy can be compiled to C extension modules using mypyc. This makes mypy 3-5x faster than when interpreted with pure Python. We now build and upload mypyc accelerated mypy wheels for `win_arm64` and `cp314t-...` to PyPI, making it easy for Windows -users on ARM and those using the free theading builds for Python 3.14 to realise this speedup +users on ARM and those using the free threading builds for Python 3.14 to realise this speedup -- just `pip install` the latest mypy. Contributed by Marc Mueller diff --git a/misc/upload-pypi.py b/misc/upload-pypi.py index 8ea86bbea584b..52092bd288f0f 100644 --- a/misc/upload-pypi.py +++ b/misc/upload-pypi.py @@ -31,16 +31,7 @@ def is_whl_or_tar(name: str) -> bool: def item_ok_for_pypi(name: str) -> bool: - if not is_whl_or_tar(name): - return False - - name = name.removesuffix(".tar.gz") - name = name.removesuffix(".whl") - - if name.endswith("wasm32"): - return False - - return True + return is_whl_or_tar(name) def get_release_for_tag(tag: str) -> dict[str, Any]: diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 04381db913521..87663f2adde40 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -5,5 +5,5 @@ typing_extensions>=4.14.0; python_version>='3.15' mypy_extensions>=1.0.0 pathspec>=1.0.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.11.0; platform_python_implementation != 'PyPy' -ast-serialize>=0.5.0,<1.0.0 +librt>=0.13.0; platform_python_implementation != 'PyPy' +ast-serialize>=0.6.0,<1.0.0 diff --git a/mypy/build.py b/mypy/build.py index 7fc9526ccb2fe..a03a6eb8972cc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -4345,7 +4345,7 @@ def load_graph( for dep in st.ancestors + dependencies + st.suppressed: ignored = dep in st.suppressed_set and dep not in entry_points if ignored and dep not in added: - manager.missing_modules[dep] = SuppressionReason.NOT_FOUND + manager.missing_modules.setdefault(dep, SuppressionReason.NOT_FOUND) # TODO: for now we skip this in the daemon as a performance optimization. # This however creates a correctness issue, see #7777 and State.is_fresh(). if not manager.use_fine_grained_cache() or manager.options.warn_unused_configs: diff --git a/mypy/checker.py b/mypy/checker.py index 33705c98e10c3..122a7512b44c4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1825,6 +1825,7 @@ def check_unbound_return_typevar(self, typ: CallableType) -> None: "Consider using the upper bound " f"{format_type(typ.ret_type.upper_bound, self.options)} instead", context=typ.ret_type, + code=TYPE_VAR, ) def check_default_params(self, item: FuncItem, body_is_trivial: bool | None = None) -> None: @@ -3132,7 +3133,7 @@ class C(B, A[int]): ... # this is unsafe because... x: A[int] = C() x.foo # ...runtime type is (str) -> None, while static type is (int) -> None """ - if name in ("__init__", "__new__", "__init_subclass__"): + if name in {"__init__", "__new__", "__init_subclass__", "__replace__"}: # __init__ and friends can be incompatible -- it's a special case. return first = base1.names[name] @@ -4407,8 +4408,10 @@ def check_multi_assignment_from_tuple( # inferred return type for an overloaded function # to be ambiguous. return - assert isinstance(reinferred_rvalue_type, TupleType) - rvalue_type = reinferred_rvalue_type + if isinstance(reinferred_rvalue_type, TupleType): + # This branch will usually be taken, but in some cases context can + # e.g. select a different overload + rvalue_type = reinferred_rvalue_type left_rv_types, star_rv_types, right_rv_types = self.split_around_star( rvalue_type.items, star_index, len(lvalues) @@ -6857,9 +6860,13 @@ def comparison_type_narrowing_helper(self, node: ComparisonExpr) -> tuple[TypeMa for known_item in container_item_types: # Match the should_coerce_literals logic from narrow_type_by_identity_equality p_known_item = get_proper_type(known_item) - if is_literal_type_like(p_known_item) or ( - isinstance(p_known_item, Instance) and p_known_item.type.is_enum - ): + if ( + is_literal_type_like(p_known_item) + or ( + isinstance(p_known_item, Instance) + and p_known_item.type.is_enum + ) + ) and not has_custom_eq_checks(p_known_item): known_item = coerce_to_literal(known_item) if_map, else_map = self.narrow_type_by_identity_equality( "==", @@ -8128,7 +8135,20 @@ def iterable_item_type(self, it: ProperType, context: Context) -> Type: return self.analyze_iterable_item_type_without_expression(it, context)[1] def function_type(self, func: FuncBase) -> FunctionLike: - return function_type(func, self.named_type("builtins.function")) + typ = function_type(func, self.named_type("builtins.function")) + if ( + isinstance(func, FuncItem) + and func.is_coroutine + and not func.is_async_generator + and func.type is None + and isinstance(typ, CallableType) + ): + any_type = AnyType(TypeOfAny.special_form) + ret_type = self.named_generic_type( + "typing.Coroutine", [any_type, any_type, typ.ret_type] + ) + return typ.copy_modified(ret_type=ret_type) + return typ def push_type_map(self, type_map: TypeMap, *, from_assignment: bool = True) -> None: if is_unreachable_map(type_map): @@ -8874,6 +8894,7 @@ def builtin_item_type(tp: Type) -> Type | None: "builtins.list", "builtins.tuple", "builtins.dict", + "builtins.frozendict", "builtins.set", "builtins.frozenset", "_collections_abc.dict_keys", diff --git a/mypy/checker_shared.py b/mypy/checker_shared.py index 2b25d3e73a6fe..c1c4cee748b32 100644 --- a/mypy/checker_shared.py +++ b/mypy/checker_shared.py @@ -16,6 +16,7 @@ ArgKind, Context, Expression, + FuncBase, FuncItem, LambdaExpr, MypyFile, @@ -28,6 +29,7 @@ from mypy.plugin import CheckerPluginInterface, Plugin from mypy.types import ( CallableType, + FunctionLike, Instance, LiteralValue, Overloaded, @@ -149,6 +151,10 @@ def expr_checker(self) -> ExpressionCheckerSharedApi: def named_type(self, name: str) -> Instance: raise NotImplementedError + @abstractmethod + def function_type(self, func: FuncBase) -> FunctionLike: + raise NotImplementedError + @abstractmethod def lookup_typeinfo(self, fullname: str) -> TypeInfo: raise NotImplementedError diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 44855f49afaf9..172d44555b946 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -150,7 +150,6 @@ false_only, fixup_partial_type, freeze_all_type_vars, - function_type, get_all_type_vars, get_type_vars, is_literal_type_like, @@ -415,7 +414,7 @@ def analyze_static_reference( if isinstance(node, (Var, Decorator, OverloadedFuncDef)): return node.type or AnyType(TypeOfAny.special_form) elif isinstance(node, FuncDef): - return function_type(node, self.named_type("builtins.function")) + return self.chk.function_type(node) elif isinstance(node, TypeInfo): # Reference to a type object. if node.typeddict_type: diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e75a8ed7a5b03..3ba99d8e8c6b2 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -47,7 +47,6 @@ bind_self, erase_to_bound, freeze_all_type_vars, - function_type, get_all_type_vars, make_simplified_union, supported_self_type, @@ -360,7 +359,7 @@ def analyze_instance_member_access( if mx.is_lvalue and not mx.suppress_errors: mx.msg.cant_assign_to_method(mx.context) if not isinstance(method, OverloadedFuncDef): - signature = function_type(method, mx.named_type("builtins.function")) + signature = mx.chk.function_type(method) else: if method.type is None: # Overloads may be not ready if they are decorated. Handle this in same @@ -1325,7 +1324,7 @@ def analyze_class_attribute_access( return AnyType(TypeOfAny.from_error) else: assert isinstance(node.node, SYMBOL_FUNCBASE_TYPES) - typ = function_type(node.node, mx.named_type("builtins.function")) + typ = mx.chk.function_type(node.node) # Note: if we are accessing class method on class object, the cls argument is bound. # Annotated and/or explicit class methods go through other code paths above, for # unannotated implicit class methods we do this here. @@ -1490,7 +1489,7 @@ def analyze_decorator_or_funcbase_access( """ if isinstance(defn, Decorator): return analyze_var(name, defn.var, itype, mx) - typ = function_type(defn, mx.chk.named_type("builtins.function")) + typ = mx.chk.function_type(defn) if isinstance(defn, (FuncDef, OverloadedFuncDef)) and defn.is_trivial_self: return bind_self_fast(typ, mx.self_type) typ = check_self_arg(typ, mx.self_type, defn.is_class, mx.context, name, mx.msg) diff --git a/mypy/inspections.py b/mypy/inspections.py index 1a869438101fe..6e2018262944c 100644 --- a/mypy/inspections.py +++ b/mypy/inspections.py @@ -297,7 +297,7 @@ def cmp_types(x: TypeInfo, y: TypeInfo) -> int: result = {} for base in sorted_bases: if not combined_attrs[base]: - # Skip bases where everytihng was filtered out. + # Skip bases where everything was filtered out. continue result[base] = combined_attrs[base] return result diff --git a/mypy/main.py b/mypy/main.py index 9227e24680ce6..0cf624a3a5d7b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1677,13 +1677,15 @@ def process_cache_map( parser.error(f"Duplicate --cache-map source {source})") if not source.endswith(".py") and not source.endswith(".pyi"): parser.error(f"Invalid --cache-map source {source} (triple[0] must be *.py[i])") - if not meta_file.endswith(".meta.json"): + if not meta_file.endswith((".meta.json", ".meta.ff")): parser.error( - "Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)" % meta_file + "Invalid --cache-map meta_file %s (triple[1] must be *.meta.json or *.meta.ff)" + % meta_file ) - if not data_file.endswith(".data.json"): + if not data_file.endswith((".data.json", ".data.ff")): parser.error( - "Invalid --cache-map data_file %s (triple[2] must be *.data.json)" % data_file + "Invalid --cache-map data_file %s (triple[2] must be *.data.json or *.data.ff)" + % data_file ) options.cache_map[source] = (meta_file, data_file) diff --git a/mypy/options.py b/mypy/options.py index 2f03cd3eab5b7..92c9ea3b1701d 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -1,9 +1,9 @@ from __future__ import annotations +import os import pprint import re import sys -import sysconfig from collections.abc import Callable from re import Pattern from typing import Any, Final @@ -112,11 +112,11 @@ def __init__(self) -> None: # then mypy does not search for PEP 561 packages. self.python_executable: str | None = sys.executable - # When cross compiling to emscripten, we need to rely on MACHDEP because - # sys.platform is the host build platform, not emscripten. - MACHDEP = sysconfig.get_config_var("MACHDEP") - if MACHDEP == "emscripten": - self.platform = MACHDEP + # When cross compiling to emscripten, sys.platform is the host build + # platform, not emscripten. Pyodide sets the PYODIDE environment variable + # at build time, so we use it to detect the emscripten target. + if "PYODIDE" in os.environ: + self.platform = "emscripten" else: self.platform = sys.platform diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 5b39b11a623cc..790fe618c95c7 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -914,7 +914,7 @@ def _infer_dataclass_attr_init_type( # Perform a simple-minded inference from the signature of __set__, if present. # We can't use mypy.checkmember here, since this plugin runs before type checking. - # We only support some basic scanerios here, which is hopefully sufficient for + # We only support some basic scenarios here, which is hopefully sufficient for # the vast majority of use cases. if not isinstance(t, Instance): return default diff --git a/mypy/server/deps.py b/mypy/server/deps.py index b2c91d8db4888..29cd8ac8b4635 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -671,6 +671,13 @@ def visit_member_expr(self, e: MemberExpr) -> None: if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) + if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, MypyFile): + # Also depend on the name as accessed through this module. The + # fullname above may point to the original definition (e.g. via + # a re-export using "from ... import *"), but we must also + # recheck if the name is removed from the accessed module's + # namespace. + self.add_dependency(make_trigger(e.expr.node.fullname + "." + e.name)) else: # Reference to a non-module (or missing) attribute if e.expr not in self.type_map: diff --git a/mypy/server/update.py b/mypy/server/update.py index 64ce0fb5d3f8a..cb7ff60e14cba 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -783,31 +783,37 @@ def calculate_active_triggers( else: snapshot2 = snapshot_symbol_table(id, new.names) diff = compare_symbol_table_snapshots(id, snapshot1, snapshot2) - package_nesting_level = id.count(".") - for item in diff.copy(): - if item.count(".") <= package_nesting_level + 1 and item.split(".")[-1] not in ( - "__builtins__", - "__file__", - "__name__", - "__package__", - "__doc__", - ): - # Activate catch-all wildcard trigger for top-level module changes (used for - # "from m import *"). This also gets triggered by changes to module-private - # entries, but as these unneeded dependencies only result in extra processing, - # it's a minor problem. - # - # TODO: Some __* names cause mistriggers. Fix the underlying issue instead of - # special casing them here. - diff.add(id + WILDCARD_TAG) - if item.count(".") > package_nesting_level + 1: - # These are for changes within classes, used by protocols. - diff.add(item.rsplit(".", 1)[0] + WILDCARD_TAG) - + diff |= wildcard_triggers_for_changes(id, diff) names |= diff return {make_trigger(name) for name in names} +def wildcard_triggers_for_changes(module_id: str, diff: set[str]) -> set[str]: + """Return catch-all wildcard triggers activated by a set of changed names.""" + result: set[str] = set() + package_nesting_level = module_id.count(".") + for item in diff: + if item.count(".") <= package_nesting_level + 1 and item.split(".")[-1] not in ( + "__builtins__", + "__file__", + "__name__", + "__package__", + "__doc__", + ): + # Activate catch-all wildcard trigger for top-level module changes (used for + # "from m import *"). This also gets triggered by changes to module-private + # entries, but as these unneeded dependencies only result in extra processing, + # it's a minor problem. + # + # TODO: Some __* names cause mistriggers. Fix the underlying issue instead of + # special casing them here. + result.add(module_id + WILDCARD_TAG) + if item.count(".") > package_nesting_level + 1: + # These are for changes within classes, used by protocols. + result.add(item.rsplit(".", 1)[0] + WILDCARD_TAG) + return result + + def replace_modules_with_new_variants( manager: BuildManager, graph: dict[str, State], @@ -1044,6 +1050,7 @@ def key(node: FineGrainedDeferredNode) -> int: changed = compare_symbol_table_snapshots( file_node.fullname, old_symbols_snapshot, new_symbols_snapshot ) + changed |= wildcard_triggers_for_changes(module_id, changed) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 259bb3791deaf..2d97dd534d9f5 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2278,7 +2278,7 @@ def infer_variance(info: TypeInfo, i: int) -> bool: self_type = fill_typevars(info) for member in all_non_object_members(info): # __mypy-replace is an implementation detail of the dataclass plugin - if member in ("__init__", "__new__", "__mypy-replace"): + if member in {"__init__", "__new__", "__replace__", "__mypy-replace"}: continue if isinstance(self_type, TupleType): diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index e9c8d91fdbd71..96eb98131db63 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -245,8 +245,6 @@ posixpath: 3.0- pprint: 3.0- profile: 3.0- profiling: 3.15- -profiling.sampling: 3.15- -profiling.tracing: 3.15- pstats: 3.0- pty: 3.0- pwd: 3.0- diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index 449cf75dad422..fcd0da4c465c7 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -423,7 +423,7 @@ class window: # undocumented def chgat(self, y: int, x: int, num: int, attr: int) -> None: ... def clear(self) -> None: ... - def clearok(self, yes: int) -> None: ... + def clearok(self, flag: bool, /) -> None: ... def clrtobot(self) -> None: ... def clrtoeol(self) -> None: ... def cursyncup(self) -> None: ... @@ -480,9 +480,9 @@ class window: # undocumented @overload def hline(self, y: int, x: int, ch: _ChType, n: int) -> None: ... - def idcok(self, flag: bool) -> None: ... - def idlok(self, yes: bool) -> None: ... - def immedok(self, flag: bool) -> None: ... + def idcok(self, flag: bool, /) -> None: ... + def idlok(self, flag: bool, /) -> None: ... + def immedok(self, flag: bool, /) -> None: ... @overload def inch(self) -> int: ... @@ -494,7 +494,7 @@ class window: # undocumented @overload def insch(self, y: int, x: int, ch: _ChType, attr: int = ...) -> None: ... - def insdelln(self, nlines: int) -> None: ... + def insdelln(self, nlines: int, /) -> None: ... def insertln(self) -> None: ... @overload @@ -514,13 +514,13 @@ class window: # undocumented def is_linetouched(self, line: int, /) -> bool: ... def is_wintouched(self) -> bool: ... - def keypad(self, yes: bool, /) -> None: ... - def leaveok(self, yes: bool) -> None: ... - def move(self, new_y: int, new_x: int) -> None: ... - def mvderwin(self, y: int, x: int) -> None: ... - def mvwin(self, new_y: int, new_x: int) -> None: ... - def nodelay(self, yes: bool) -> None: ... - def notimeout(self, yes: bool) -> None: ... + def keypad(self, flag: bool, /) -> None: ... + def leaveok(self, flag: bool, /) -> None: ... + def move(self, new_y: int, new_x: int, /) -> None: ... + def mvderwin(self, y: int, x: int, /) -> None: ... + def mvwin(self, new_y: int, new_x: int, /) -> None: ... + def nodelay(self, flag: bool, /) -> None: ... + def notimeout(self, flag: bool, /) -> None: ... @overload def noutrefresh(self) -> None: ... @@ -550,9 +550,9 @@ class window: # undocumented @overload def refresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... - def resize(self, nlines: int, ncols: int) -> None: ... + def resize(self, nlines: int, ncols: int, /) -> None: ... def scroll(self, lines: int = 1) -> None: ... - def scrollok(self, flag: bool) -> None: ... + def scrollok(self, flag: bool, /) -> None: ... def setscrreg(self, top: int, bottom: int, /) -> None: ... def standend(self) -> None: ... def standout(self) -> None: ... @@ -568,9 +568,9 @@ class window: # undocumented def subwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... def syncdown(self) -> None: ... - def syncok(self, flag: bool) -> None: ... + def syncok(self, flag: bool, /) -> None: ... def syncup(self) -> None: ... - def timeout(self, delay: int) -> None: ... + def timeout(self, delay: int, /) -> None: ... def touchline(self, start: int, count: int, changed: bool = True) -> None: ... def touchwin(self) -> None: ... def untouchwin(self) -> None: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 14a98b9a5fcaf..a1993773ee25f 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -956,14 +956,27 @@ class DictComp(expr): else: value: expr generators: list[comprehension] - if sys.version_info >= (3, 13): + if sys.version_info >= (3, 15): + def __init__( + self, key: expr, value: expr | None = None, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + elif sys.version_info >= (3, 13): def __init__( self, key: expr, value: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] ) -> None: ... else: def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - if sys.version_info >= (3, 14): + if sys.version_info >= (3, 15): + def __replace__( + self, + *, + key: expr = ..., + value: expr | None = ..., + generators: list[comprehension] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + elif sys.version_info >= (3, 14): def __replace__( self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] ) -> Self: ... @@ -2147,6 +2160,10 @@ class NodeVisitor: def visit_TypeVarTuple(self, node: TypeVarTuple) -> Any: ... def visit_TypeAlias(self, node: TypeAlias) -> Any: ... + if sys.version_info >= (3, 14): + def visit_TemplateStr(self, node: TemplateStr) -> Any: ... + def visit_Interpolation(self, node: Interpolation) -> Any: ... + # visit methods for deprecated nodes def visit_ExtSlice(self, node: ExtSlice) -> Any: ... def visit_Index(self, node: Index) -> Any: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index d773f98e90b6b..8497f9e2e1037 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -33,7 +33,7 @@ from _typeshed import ( from collections.abc import Awaitable, Callable, Iterable, Iterator, MutableSet, Reversible, Set as AbstractSet, Sized from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper from os import PathLike -from types import CellType, CodeType, EllipsisType, GenericAlias, NotImplementedType, TracebackType +from types import CellType, CodeType, EllipsisType, GenericAlias, NotImplementedType, TracebackType, UnionType # mypy crashes if any of {ByteString, Sequence, MutableSequence, Mapping, MutableMapping} # are imported from collections.abc in builtins.pyi @@ -1277,7 +1277,6 @@ if sys.version_info >= (3, 15): cls: type[frozendict[str, _VT]], iterable: Iterable[tuple[str, _VT]], /, **kwargs: _VT ) -> frozendict[str, _VT]: ... - def __init__(self) -> None: ... def copy(self) -> frozendict[_KT, _VT]: ... @overload @@ -2080,11 +2079,13 @@ if sys.version_info >= (3, 15): class sentinel: __name__: str __module__: str - def __new__(cls, name: str, /, *, repr: str | None = None) -> Self: ... - def __copy__(self, /) -> Self: ... - def __deepcopy__(self, memo: Any, /) -> Self: ... - def __or__(self, other: Any, /) -> Any: ... - def __ror__(self, other: Any, /) -> Any: ... + def __new__(cls, name: str, /, *, repr: str | None = None) -> sentinel: ... + def __copy__(self, /) -> sentinel: ... + def __deepcopy__(self, memo: Any, /) -> sentinel: ... + # `other` can be any legal form for unions. + # `x | x` creates a `sentinel` instance if `x` is a sentinel, not a `UnionType` instance. + def __or__(self, other: Any, /) -> UnionType | sentinel: ... + def __ror__(self, other: Any, /) -> UnionType | sentinel: ... @overload def sorted( diff --git a/mypy/typeshed/stdlib/fnmatch.pyi b/mypy/typeshed/stdlib/fnmatch.pyi index 345c4576497de..018139dc482bf 100644 --- a/mypy/typeshed/stdlib/fnmatch.pyi +++ b/mypy/typeshed/stdlib/fnmatch.pyi @@ -1,15 +1,16 @@ import sys from collections.abc import Iterable +from os import PathLike from typing import AnyStr __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] if sys.version_info >= (3, 14): __all__ += ["filterfalse"] -def fnmatch(name: AnyStr, pat: AnyStr) -> bool: ... +def fnmatch(name: AnyStr | PathLike[AnyStr], pat: AnyStr | PathLike[AnyStr]) -> bool: ... def fnmatchcase(name: AnyStr, pat: AnyStr) -> bool: ... -def filter(names: Iterable[AnyStr], pat: AnyStr) -> list[AnyStr]: ... +def filter(names: Iterable[AnyStr | PathLike[AnyStr]], pat: AnyStr | PathLike[AnyStr]) -> list[AnyStr]: ... def translate(pat: str) -> str: ... if sys.version_info >= (3, 14): - def filterfalse(names: Iterable[AnyStr], pat: AnyStr) -> list[AnyStr]: ... + def filterfalse(names: Iterable[AnyStr | PathLike[AnyStr]], pat: AnyStr | PathLike[AnyStr]) -> list[AnyStr]: ... diff --git a/mypy/typeshed/stdlib/html/__init__.pyi b/mypy/typeshed/stdlib/html/__init__.pyi index 8ad72f1265882..71e971d150449 100644 --- a/mypy/typeshed/stdlib/html/__init__.pyi +++ b/mypy/typeshed/stdlib/html/__init__.pyi @@ -1,4 +1,7 @@ +import re + __all__ = ["escape", "unescape"] def escape(s: str, quote: bool = True) -> str: ... def unescape(s: str) -> str: ... +def _replace_charref(s: re.Match[str]) -> str: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index 003ecc10f072e..1bd2b9dee325b 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -38,7 +38,7 @@ from typing import ( overload, type_check_only, ) -from typing_extensions import Self, TypeIs, deprecated, disjoint_base +from typing_extensions import Never, Self, TypeIs, deprecated, disjoint_base if sys.version_info >= (3, 14): from annotationlib import Format @@ -217,7 +217,7 @@ if sys.version_info >= (3, 11): def getmodulename(path: StrPath) -> str | None: ... def ismodule(object: object) -> TypeIs[ModuleType]: ... -def isclass(object: object) -> TypeIs[type[Any]]: ... +def isclass(object: object) -> TypeIs[type[object]]: ... def ismethod(object: object) -> TypeIs[MethodType]: ... if sys.version_info >= (3, 14): @@ -245,7 +245,7 @@ def iscoroutinefunction(obj: Callable[_P, object]) -> TypeGuard[Callable[_P, Cor @overload def iscoroutinefunction(obj: object) -> TypeGuard[Callable[..., CoroutineType[Any, Any, Any]]]: ... -def isgenerator(object: object) -> TypeIs[GeneratorType[Any, Any, Any]]: ... +def isgenerator(object: object) -> TypeIs[GeneratorType[object, Never, object]]: ... def iscoroutine(object: object) -> TypeIs[CoroutineType[Any, Any, Any]]: ... def isawaitable(object: object) -> TypeIs[Awaitable[Any]]: ... @@ -264,7 +264,7 @@ class _SupportsSet(Protocol[_T_contra, _V_contra]): class _SupportsDelete(Protocol[_T_contra]): def __delete__(self, instance: _T_contra, /) -> None: ... -def isasyncgen(object: object) -> TypeIs[AsyncGeneratorType[Any, Any]]: ... +def isasyncgen(object: object) -> TypeIs[AsyncGeneratorType[object, Never]]: ... def istraceback(object: object) -> TypeIs[TracebackType]: ... def isframe(object: object) -> TypeIs[FrameType]: ... def iscode(object: object) -> TypeIs[CodeType]: ... @@ -289,7 +289,7 @@ def ismethoddescriptor(object: object) -> TypeIs[MethodDescriptorType]: ... def ismemberdescriptor(object: object) -> TypeIs[MemberDescriptorType]: ... def isabstract(object: object) -> bool: ... def isgetsetdescriptor(object: object) -> TypeIs[GetSetDescriptorType]: ... -def isdatadescriptor(object: object) -> TypeIs[_SupportsSet[Any, Any] | _SupportsDelete[Any]]: ... +def isdatadescriptor(object: object) -> TypeIs[_SupportsSet[Never, Never] | _SupportsDelete[Never]]: ... # # Retrieving source code diff --git a/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi b/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi index 6072d4f2359af..666fd42c3ea5f 100644 --- a/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi +++ b/mypy/typeshed/stdlib/profiling/sampling/gecko_collector.pyi @@ -1,11 +1,109 @@ -from _typeshed import StrOrBytesPath -from collections.abc import Sequence +from _typeshed import Incomplete, StrOrBytesPath, StrPath +from collections.abc import Generator, Sequence +from tempfile import TemporaryDirectory +from typing import Any, ClassVar, Final, TypedDict, type_check_only from _remote_debugging import AwaitedInfo, InterpreterInfo from .collector import Collector, _Timestamps +@type_check_only +class _GeckoCategory(TypedDict): + name: str + color: str + subcategories: list[str] + +THREAD_STATUS_HAS_GIL: Final[int] +THREAD_STATUS_ON_CPU: Final[int] +THREAD_STATUS_UNKNOWN: Final[int] +THREAD_STATUS_GIL_REQUESTED: Final[int] +THREAD_STATUS_HAS_EXCEPTION: Final[int] +THREAD_STATUS_MAIN_THREAD: Final[int] + +GECKO_CATEGORIES: Final[list[_GeckoCategory]] + +CATEGORY_OTHER: Final = 0 +CATEGORY_PYTHON: Final = 1 +CATEGORY_NATIVE: Final = 2 +CATEGORY_GC: Final = 3 +CATEGORY_GIL: Final = 4 +CATEGORY_CPU: Final = 5 +CATEGORY_CODE_TYPE: Final = 6 +CATEGORY_OPCODES: Final = 7 +CATEGORY_EXCEPTION: Final = 8 + +DEFAULT_SUBCATEGORY: Final = 0 + +GECKO_FORMAT_VERSION: Final = 32 +GECKO_PREPROCESSED_VERSION: Final = 57 + +RESOURCE_TYPE_LIBRARY: Final = 1 + +FRAME_ADDRESS_NONE: Final = -1 +FRAME_INLINE_DEPTH_ROOT: Final = 0 + +PROCESS_TYPE_MAIN: Final = 0 +STACKWALK_DISABLED: Final = 0 + +DEFAULT_SPILL_BUFFER_BYTES: Final[int] + +class SpillColumn: + path: str + buffer: bytearray + + def __init__(self, directory: StrPath, basename: StrPath, *, buffer_bytes: int | None = None) -> None: ... + # "value" accepts the same types as json.JSONEncoder.encode() + def append(self, value: Any) -> None: ... + def flush(self) -> None: ... + def iter_tokens(self) -> Generator[str]: ... + +class GeckoThreadSpill: + sample_count: int + marker_count: int + def __init__(self, directory: StrPath, tid: int) -> None: ... + def append_sample(self, stack_index: int, time_ms: float) -> None: ... + def append_marker( + self, name_idx: int, start_time: float, end_time: float, phase: int, category: int, data: dict[str, Any] + ) -> None: ... + def prepare_read(self) -> None: ... + class GeckoCollector(Collector): + aggregating: ClassVar[bool] + + sample_interval_usec: int + skip_idle: bool + opcodes_enabled: bool + start_time: float + + global_strings: list[str] + global_string_map: dict[str, int] + + threads: dict[int, dict[str, Any]] + spill_dir: TemporaryDirectory[str] | None + exported: bool + + libs: list[Incomplete] + + sample_count: int + last_sample_time: float + interval: float + + has_gil_start: dict[Incomplete, Incomplete] + no_gil_start: dict[Incomplete, Incomplete] + on_cpu_start: dict[Incomplete, Incomplete] + off_cpu_start: dict[Incomplete, Incomplete] + python_code_start: dict[Incomplete, Incomplete] + native_code_start: dict[Incomplete, Incomplete] + gil_wait_start: dict[Incomplete, Incomplete] + exception_start: dict[Incomplete, Incomplete] + no_exception_start: dict[Incomplete, Incomplete] + + gc_start_per_thread: dict[int, float] + + initialized_threads: set[Incomplete] + + opcode_state: dict[int, tuple[Incomplete, int, int, str, str, float]] + def __init__(self, sample_interval_usec: int, *, skip_idle: bool = False, opcodes: bool = False) -> None: ... def collect( self, stack_frames: Sequence[InterpreterInfo] | Sequence[AwaitedInfo], timestamps_us: _Timestamps = None diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index b9771ffc72dad..68b6b3fbe41d7 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -703,8 +703,10 @@ class GenericAlias: @property def __typing_unpacked_tuple_args__(self) -> tuple[Any, ...] | None: ... - def __or__(self, value: Any, /) -> UnionType: ... - def __ror__(self, value: Any, /) -> UnionType: ... + # `other` can be any legal form for unions. + # `list[int] | list[int]` creates a `GenericAlias` instance, not a `UnionType` instance + def __or__(self, value: Any, /) -> UnionType | GenericAlias: ... + def __ror__(self, value: Any, /) -> UnionType | GenericAlias: ... # GenericAlias delegates attr access to `__origin__` def __getattr__(self, name: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index fdbba495c579f..80341175e6e14 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -59,7 +59,6 @@ from typing import ( # noqa: Y022,Y037,Y038,Y039,UP035 Tuple as Tuple, Type as Type, TypeAlias as TypeAlias, - TypedDict as TypedDict, TypeGuard as TypeGuard, TypeVar as _TypeVar, Union as Union, @@ -72,6 +71,9 @@ from typing import ( # noqa: Y022,Y037,Y038,Y039,UP035 type_check_only, ) +if sys.version_info >= (3, 14): + from _typeshed import EvaluateFunc + # Please keep order the same as at runtime. __all__ = [ # Super-special typing primitives. @@ -143,6 +145,7 @@ __all__ = [ "override", "Protocol", "Sentinel", + "sentinel", "reveal_type", "runtime", "runtime_checkable", @@ -232,6 +235,11 @@ Literal: _SpecialForm def IntVar(name: str) -> Any: ... # returns a new TypeVar +# Kept as a distinct symbol to `typing.TypedDict` so that type checkers can more easily +# distinguish between the two on Python 3.14, on which `typing_extensions.TypedDict` +# exposes `__closed__` and `__extra_items__` but `typing.TypedDict` does not +TypedDict: _SpecialForm + # Internal mypy fallback type for all typed dicts (does not exist at runtime) # N.B. Keep this mostly in sync with typing._TypedDict/mypy_extensions._TypedDict @type_check_only @@ -456,7 +464,6 @@ if sys.version_info >= (3, 13): ReadOnly as ReadOnly, TypeIs as TypeIs, TypeVar as TypeVar, - TypeVarTuple as TypeVarTuple, get_protocol_members as get_protocol_members, is_protocol as is_protocol, ) @@ -545,19 +552,58 @@ else: def has_default(self) -> bool: ... def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + ReadOnly: _SpecialForm + TypeIs: _SpecialForm + +if sys.version_info >= (3, 15): + from typing import TypeVarTuple as TypeVarTuple +else: @final class TypeVarTuple: @property def __name__(self) -> str: ... @property + def __bound__(self) -> AnnotationForm | None: ... + @property + def __covariant__(self) -> bool: ... + @property + def __contravariant__(self) -> bool: ... + @property + def __infer_variance__(self) -> bool: ... + @property def __default__(self) -> AnnotationForm: ... - def __init__(self, name: str, *, default: AnnotationForm = ...) -> None: ... + if sys.version_info >= (3, 11): + def __new__( + cls, + name: str, + *, + bound: AnnotationForm | None = None, + covariant: bool = False, + contravariant: bool = False, + infer_variance: bool = False, + default: AnnotationForm = ..., + ) -> Self: ... + else: + def __init__( + self, + name: str, + *, + bound: AnnotationForm | None = None, + covariant: bool = False, + contravariant: bool = False, + infer_variance: bool = False, + default: AnnotationForm = ..., + ) -> None: ... + def __iter__(self) -> Any: ... # Unpack[Self] def has_default(self) -> bool: ... - def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Never, /) -> Never: ... - ReadOnly: _SpecialForm - TypeIs: _SpecialForm + def __typing_prepare_subst__(self, alias: Any, args: Any, /) -> tuple[Any, ...]: ... + if sys.version_info >= (3, 14): + @property + def evaluate_default(self) -> EvaluateFunc | None: ... # TypeAliasType was added in Python 3.12, but had significant changes in 3.14. if sys.version_info >= (3, 14): @@ -680,11 +726,21 @@ else: def type_repr(value: object) -> str: ... # PEP 661 -class Sentinel: - def __init__(self, name: str, repr: str | None = None) -> None: ... - if sys.version_info >= (3, 14): - def __or__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions - def __ror__(self, other: Any) -> UnionType: ... # other can be any type form legal for unions - else: - def __or__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions - def __ror__(self, other: Any) -> _SpecialForm: ... # other can be any type form legal for unions +if sys.version_info >= (3, 15): + from builtins import sentinel as sentinel +else: + class sentinel: + def __init__(self, name: str, /, *, repr: str | None = None) -> None: ... + __name__: str + __module__: str + if sys.version_info >= (3, 14): + # `other`` can be any type form legal for unions. + # `x | x` creates a `sentinel` instance if `x` is a sentinel, not a `UnionType` instance + def __or__(self, other: Any) -> UnionType | sentinel: ... + def __ror__(self, other: Any) -> UnionType | sentinel: ... + else: + # other can be any type form legal for unions + def __or__(self, other: Any) -> _SpecialForm: ... + def __ror__(self, other: Any) -> _SpecialForm: ... + +Sentinel = sentinel diff --git a/mypy/typeshed/stubs/librt/librt/strings.pyi b/mypy/typeshed/stubs/librt/librt/strings.pyi index 94e3b69abf243..af9ae60936d8e 100644 --- a/mypy/typeshed/stubs/librt/librt/strings.pyi +++ b/mypy/typeshed/stubs/librt/librt/strings.pyi @@ -42,7 +42,8 @@ def read_f64_le(b: bytes, index: i64, /) -> float: ... def read_f64_be(b: bytes, index: i64, /) -> float: ... # Codepoint classification helpers operating on i32 codepoints (typically -# obtained via ord(s[i])). Negative inputs return False. +# obtained via ord(s[i])). Out-of-range inputs (negative, or past the maximum +# Unicode code point 0x10FFFF) return False. def isspace(c: i32, /) -> bool: ... def isdigit(c: i32, /) -> bool: ... def isalnum(c: i32, /) -> bool: ... @@ -53,7 +54,8 @@ def isidentifier(c: i32, /) -> bool: ... # uppercase / lowercase expands to multiple codepoints (e.g. U+00DF # uppercases to "SS", U+FB01 to "FI"), returns the input unchanged so # the signature stays i32 -> i32. Use str.upper() / str.lower() for full -# Unicode case conversion when those cases matter. Negative inputs are -# returned unchanged. +# Unicode case conversion when those cases matter. Out-of-range inputs +# (negative, or past the maximum Unicode code point 0x10FFFF) are returned +# unchanged. def toupper(c: i32, /) -> i32: ... def tolower(c: i32, /) -> i32: ... diff --git a/mypy/typeshed/stubs/librt/librt/threading.pyi b/mypy/typeshed/stubs/librt/librt/threading.pyi new file mode 100644 index 0000000000000..a3571ac7f4745 --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/threading.pyi @@ -0,0 +1,16 @@ +from types import TracebackType +from typing import final + +@final +class Lock: + def acquire(self, blocking: bool = True) -> bool: ... + def release(self) -> None: ... + def locked(self) -> bool: ... + def __enter__(self) -> bool: ... + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + /, + ) -> None: ... diff --git a/mypy/version.py b/mypy/version.py index a33ca938708f4..b4d4e0d35d054 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "2.2.0+dev" +__version__ = "2.3.1" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/mypyc/build.py b/mypyc/build.py index 13bd50fef3b1a..8c6eabead17c9 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -54,7 +54,12 @@ class ModDesc(NamedTuple): LIBRT_MODULES = [ ModDesc("librt.internal", ["internal/librt_internal.c"], [], ["internal"]), - ModDesc("librt.strings", ["strings/librt_strings.c"], [], ["strings"]), + ModDesc( + "librt.strings", + ["strings/librt_strings.c"], + ["strings/librt_strings.h", "strings/librt_strings_common.h"], + ["strings"], + ), ModDesc( "librt.base64", [ @@ -122,6 +127,12 @@ class ModDesc(NamedTuple): ), ModDesc("librt.time", ["time/librt_time.c"], ["time/librt_time.h"], []), ModDesc("librt.random", ["random/librt_random.c"], ["random/librt_random.h"], ["random"]), + ModDesc( + "librt.threading", + ["threading/librt_threading.c"], + ["threading/librt_threading.h"], + ["threading"], + ), ] try: @@ -323,34 +334,36 @@ def generate_c( emit_messages(options, e.messages, time.time() - t0, serious=(not e.use_stdout)) sys.exit(1) - t1 = time.time() - if result.errors: - emit_messages(options, result.errors, t1 - t0) - sys.exit(1) - - if compiler_options.verbose: - print(f"Parsed and typechecked in {t1 - t0:.3f}s") - - errors = Errors(options) - modules, ctext, mapper = emitmodule.compile_modules_to_c( - result, compiler_options=compiler_options, errors=errors, groups=groups - ) - t2 = time.time() - emit_messages(options, errors.new_messages(), t2 - t1) - if errors.num_errors: - # No need to stop the build if only warnings were emitted. - sys.exit(1) - - if compiler_options.verbose: - print(f"Compiled to C in {t2 - t1:.3f}s") - - if options.mypyc_annotation_file: - generate_annotated_html(options.mypyc_annotation_file, result, modules, mapper) + try: + t1 = time.time() + if result.errors: + emit_messages(options, result.errors, t1 - t0) + sys.exit(1) - # Collect SourceDep dependencies - source_deps = sorted(emitmodule.collect_source_dependencies(modules), key=lambda d: d.path) + if compiler_options.verbose: + print(f"Parsed and typechecked in {t1 - t0:.3f}s") - return ctext, "\n".join(format_modules(modules)), source_deps + errors = Errors(options) + modules, ctext, mapper = emitmodule.compile_modules_to_c( + result, compiler_options=compiler_options, errors=errors, groups=groups + ) + t2 = time.time() + emit_messages(options, errors.new_messages(), t2 - t1) + if errors.num_errors: + # No need to stop the build if only warnings were emitted. + sys.exit(1) + + if compiler_options.verbose: + print(f"Compiled to C in {t2 - t1:.3f}s") + + if options.mypyc_annotation_file: + generate_annotated_html(options.mypyc_annotation_file, result, modules, mapper) + + # Collect SourceDep dependencies + source_deps = sorted(emitmodule.collect_source_dependencies(modules), key=lambda d: d.path) + return ctext, "\n".join(format_modules(modules)), source_deps + finally: + result.manager.metastore.close() def build_using_shared_lib( diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index b89c91343e66c..57cce6a3fe8fe 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -2,14 +2,12 @@ from __future__ import annotations -import pprint import sys -import textwrap from collections.abc import Callable from typing import Final from mypyc.codegen.cstring import c_string_initializer -from mypyc.codegen.literals import Literals +from mypyc.codegen.literals import Literals, literal_sort_key from mypyc.common import ( ATTR_PREFIX, BITMAP_BITS, @@ -237,24 +235,16 @@ def attr(self, name: str) -> str: return ATTR_PREFIX + name def object_annotation(self, obj: object, line: str) -> str: - """Build a C comment with an object's string representation. + """Build a C comment with a literal value's string representation. - If the comment exceeds the line length limit, it's wrapped into a - multiline string (with the extra lines indented to be aligned with - the first line's comment). + This is a debugging aid that makes generated C easier to read. - If it contains illegal characters, an empty string is returned.""" - line_width = self._indent + len(line) - formatted = pprint.pformat(obj, compact=True, indent=1, width=max(90 - line_width, 20)) - if any(x in formatted for x in ("/*", "*/", "\0")): + If it contains illegal characters or is too long, return an empty string. + """ + formatted = stable_literal_repr(obj) + if any(x in formatted for x in ("/*", "*/", "\0")) or len(formatted) >= 256: return "" - - if "\n" in formatted: - first_line, rest = formatted.split("\n", maxsplit=1) - comment_continued = textwrap.indent(rest, (line_width + 3) * " ") - return f" /* {first_line}\n{comment_continued} */" - else: - return f" /* {formatted} */" + return f" /* {formatted} */" def emit_line(self, line: str = "", *, ann: object = None) -> None: if line.startswith("}"): @@ -1486,3 +1476,21 @@ def native_function_doc_initializer(func: FuncIR) -> str: return "NULL" docstring = f"{text_sig}\n--\n\n" return c_string_initializer(docstring.encode("ascii", errors="backslashreplace")) + + +def stable_literal_repr(obj: object) -> str: + """Return a single-line repr of a literal value. + + Behaves like repr() for most values, but renders frozenset members in a + deterministic order (frozenset iteration order is hash-seed dependent). + """ + if isinstance(obj, frozenset): + if not obj: + return "frozenset()" + items = ", ".join(stable_literal_repr(item) for item in sorted(obj, key=literal_sort_key)) + return "frozenset({" + items + "})" + elif isinstance(obj, tuple): + if len(obj) == 1: + return "(" + stable_literal_repr(obj[0]) + ",)" + return "(" + ", ".join(stable_literal_repr(item) for item in obj) + ")" + return repr(obj) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index db94f1de9406e..aeea3374353f9 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -29,6 +29,7 @@ BITMAP_BITS, BITMAP_TYPE, CPYFUNCTION_NAME, + IS_FREE_THREADED, MYPYC_DEFAULTS_SETUP, NATIVE_PREFIX, PREFIX, @@ -43,7 +44,7 @@ FuncIR, get_text_signature, ) -from mypyc.ir.rtypes import RTuple, RType, object_rprimitive +from mypyc.ir.rtypes import RTuple, RType, is_simple_refcounted_pointer, object_rprimitive from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type @@ -251,7 +252,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: getseters_name = f"{name_prefix}_getseters" vtable_name = f"{name_prefix}_vtable" traverse_name = f"{name_prefix}_traverse" - clear_name = f"{name_prefix}_clear" + clear_name = emitter.native_function_name(cl.clear) dealloc_name = f"{name_prefix}_dealloc" methods_name = f"{name_prefix}_methods" vtable_setup_name = f"{name_prefix}_trait_vtable_setup" @@ -270,7 +271,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: fields["tp_dealloc"] = f"(destructor){name_prefix}_dealloc" if not cl.is_acyclic: fields["tp_traverse"] = f"(traverseproc){name_prefix}_traverse" - fields["tp_clear"] = f"(inquiry){name_prefix}_clear" + fields["tp_clear"] = f"(inquiry){clear_name}" # Populate .tp_finalize and generate a finalize method only if __del__ is defined for this class. del_method = next((e.method for e in cl.vtable_entries if e.name == "__del__"), None) if del_method: @@ -343,7 +344,11 @@ def emit_line() -> None: if not cl.is_acyclic: generate_traverse_for_class(cl, traverse_name, emitter) emit_line() - generate_clear_for_class(cl, clear_name, emitter) + generate_clear_for_class(cl, cl.clear, emitter) + emit_line() + generate_clear_for_class( + cl, cl.clear_on_completion, emitter, skip_attrs=cl.attrs_to_keep_alive_on_completion + ) emit_line() generate_dealloc_for_class(cl, dealloc_name, clear_name, bool(del_method), emitter) emit_line() @@ -894,12 +899,21 @@ def generate_traverse_for_class(cl: ClassIR, func_name: str, emitter: Emitter) - emitter.emit_line("}") -def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> None: - emitter.emit_line("static int") - emitter.emit_line(f"{func_name}({cl.struct_name(emitter.names)} *self)") +def generate_clear_for_class( + cl: ClassIR, func_decl: FuncDecl, emitter: Emitter, skip_attrs: set[str] | None = None +) -> None: + if skip_attrs is None: + skip_attrs = set() + emitter.emit_line("static " + native_function_header(func_decl, emitter)) emitter.emit_line("{") + emitter.emit_line( + f"{cl.struct_name(emitter.names)} *self = " + f"({cl.struct_name(emitter.names)} *)cpy_r_self;" + ) for base in reversed(cl.base_mro): for attr, rtype in base.attributes.items(): + if attr in skip_attrs: + continue emitter.emit_gc_clear(f"self->{emitter.attr(attr)}", rtype) base_args = "(PyObject *)self" if cl.builtin_base: @@ -950,7 +964,7 @@ def generate_dealloc_for_class( if not cl.is_acyclic: emitter.emit_line("PyObject_GC_UnTrack(self);") if cl.builtin_base: - emitter.emit_line(f"{clear_func_name}(self);") + emitter.emit_line(f"{clear_func_name}((PyObject *)self);") # For native subclasses of builtins such as dict, the base deallocator # is responsible for tearing down base-owned storage and freeing memory. # Re-track self if base is GC-aware to match cpython's subtype_dealloc. @@ -965,7 +979,7 @@ def generate_dealloc_for_class( emit_reuse_dealloc(cl, emitter) # The trashcan is needed to handle deep recursive deallocations emitter.emit_line(f"CPy_TRASHCAN_BEGIN(self, {dealloc_func_name})") - emitter.emit_line(f"{clear_func_name}(self);") + emitter.emit_line(f"{clear_func_name}((PyObject *)self);") emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);") emitter.emit_line("CPy_TRASHCAN_END(self)") emitter.emit_line("done: ;") @@ -1065,12 +1079,14 @@ def generate_getseter_declarations(cl: ClassIR, emitter: Emitter) -> None: getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) ) ) - emitter.emit_line("static int") - emitter.emit_line( - "{}({} *self, PyObject *value, void *closure);".format( - setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + # Final attributes are read-only, so they have no setter. + if attr not in cl.final_attributes: + emitter.emit_line("static int") + emitter.emit_line( + "{}({} *self, PyObject *value, void *closure);".format( + setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names) + ) ) - ) for prop, (getter, setter) in cl.properties.items(): if getter.decl.implicit: @@ -1099,11 +1115,15 @@ def generate_getseters_table(cl: ClassIR, name: str, emitter: Emitter) -> None: if not cl.is_trait: for attr in cl.attributes: emitter.emit_line(f'{{"{attr}",') - emitter.emit_line( - " (getter){}, (setter){},".format( - getter_name(cl, attr, emitter.names), setter_name(cl, attr, emitter.names) + if attr in cl.final_attributes: + # Final attributes are read-only, so emit a NULL setter. + emitter.emit_line(f" (getter){getter_name(cl, attr, emitter.names)}, NULL,") + else: + emitter.emit_line( + " (getter){}, (setter){},".format( + getter_name(cl, attr, emitter.names), setter_name(cl, attr, emitter.names) + ) ) - ) emitter.emit_line(" NULL, NULL},") for prop, (getter, setter) in cl.properties.items(): if getter.decl.implicit: @@ -1129,8 +1149,10 @@ def generate_getseters(cl: ClassIR, emitter: Emitter) -> None: if not cl.is_trait: for i, (attr, rtype) in enumerate(cl.attributes.items()): generate_getter(cl, attr, rtype, emitter) - emitter.emit_line("") - generate_setter(cl, attr, rtype, emitter) + # Final attributes are read-only, so they have no setter. + if attr not in cl.final_attributes: + emitter.emit_line("") + generate_setter(cl, attr, rtype, emitter) if i < len(cl.attributes) - 1: emitter.emit_line("") for prop, (getter, setter) in cl.properties.items(): @@ -1157,6 +1179,32 @@ def generate_getter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("{") attr_expr = f"self->{attr_field}" + if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + # In free-threaded builds, load the attribute and take a new reference + # atomically to avoid a use-after-free race with a concurrent setter. + # CPy_GetAttrRef returns NULL if the attribute is undefined (NULL field), + # which is exactly the error/undefined value for a 'PyObject *' field. + # + # Final attributes are never rebound (no setter), so there is no concurrent + # writer to race with: a plain load + incref is safe. Use the cheaper + # CPy_GetAttrRefFinal, which skips the try-incref and _Py_NewRefWithLock + # slow path entirely (an unconditional Py_INCREF needs no maybe-weakref). + # This getter is generated per defining class, so a direct membership test + # matches the read-only getset table above (no need to walk the MRO). + if attr in cl.final_attributes: + getattr_ref = f"CPy_GetAttrRefFinal((PyObject **)&{attr_expr})" + else: + getattr_ref = f"CPy_GetAttrRef((PyObject **)&{attr_expr})" + emitter.emit_line(f"PyObject *retval = {getattr_ref};") + emitter.emit_line("if (unlikely(retval == NULL)) {") + emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") + emitter.emit_line(f' "attribute {repr(attr)} of {repr(cl.name)} undefined");') + emitter.emit_line("return NULL;") + emitter.emit_line("}") + emitter.emit_line("return retval;") + emitter.emit_line("}") + return + # HACK: Don't consider refcounted values as always defined, since it's possible to # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted # values is benign. @@ -1194,6 +1242,30 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("return -1;") emitter.emit_line("}") + if IS_FREE_THREADED and is_simple_refcounted_pointer(rtype): + # In free-threaded builds, publish the new value atomically via + # CPy_SetAttrRef so a concurrent reader (see CPy_GetAttrRef) never sees a + # torn pointer or a freed old value. CPy_SetAttrRef steals its value and + # reclaims the old one, so we cast/type-check the incoming value, take a + # new reference (the setter only borrows 'value'), then hand it over. + # A NULL value deletes the attribute (reclaims the old value, stores NULL). + if deletable: + emitter.emit_line("if (value != NULL) {") + if is_same_type(rtype, object_rprimitive): + emitter.emit_line("PyObject *tmp = value;") + else: + emitter.emit_cast("value", "tmp", rtype, declare_dest=True) + emitter.emit_lines("if (!tmp)", " return -1;") + emitter.emit_inc_ref("tmp", rtype) + emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&self->{attr_field}, tmp);") + if deletable: + emitter.emit_line("} else {") + emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&self->{attr_field}, NULL);") + emitter.emit_line("}") + emitter.emit_line("return 0;") + emitter.emit_line("}") + return + # HACK: Don't consider refcounted values as always defined, since it's possible to # access uninitialized values via 'gc.get_objects()'. Accessing non-refcounted # values is benign. @@ -1211,7 +1283,15 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_line("if (value != NULL) {") if rtype.is_unboxed: - emitter.emit_unbox("value", "tmp", rtype, error=ReturnHandler("-1"), declare_dest=True) + # Borrow the unboxed value: emit_inc_ref below takes the single owned + # reference, matching the borrowed-then-incref pattern of the other two + # branches. Without borrow=True, emit_unbox already creates a new + # reference for refcounted unboxed types (e.g. CPyTagged boxed ints, + # tuples with refcounted fields), so the emit_inc_ref would double the + # reference and leak the stored value on every set via this setter. + emitter.emit_unbox( + "value", "tmp", rtype, error=ReturnHandler("-1"), declare_dest=True, borrow=True + ) elif is_same_type(rtype, object_rprimitive): emitter.emit_line("PyObject *tmp = value;") else: diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index dcb606f6ab51b..4c3c17d047c8a 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -12,7 +12,13 @@ TracebackAndGotoHandler, c_array_initializer, ) -from mypyc.common import GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, NATIVE_PREFIX, REG_PREFIX +from mypyc.common import ( + GENERATOR_ATTRIBUTE_PREFIX, + HAVE_IMMORTAL, + IS_FREE_THREADED, + NATIVE_PREFIX, + REG_PREFIX, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( @@ -83,6 +89,7 @@ is_int_rprimitive, is_none_rprimitive, is_pointer_rprimitive, + is_simple_refcounted_pointer, is_tagged, ) @@ -394,6 +401,35 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st cast = f"({decl_cl.struct_name(self.emitter.names)} *)" return f"({cast}{obj})->{self.emitter.attr(op.attr)}" + def emit_load_attr_take_ref( + self, dest: str, obj: str, op: GetAttr, cl: ClassIR, attr_rtype: RType, attr_expr: str + ) -> bool: + """Emit the load of a native attribute into 'dest', taking a new reference. + + On free-threaded builds, reading a single reference-counted 'PyObject *' field + and taking a new reference must be done atomically to avoid a use-after-free + race with a concurrent setter. CPy_GetAttrRef performs the load and incref + atomically and returns a new reference (or NULL if undefined), so callers must + NOT emit a separate inc_ref. Return True in that case so the caller can skip it. + + Final attributes are never rebound (no setter), so there is no concurrent writer + and no use-after-free window; an owned read uses the cheaper CPy_GetAttrRefFinal + (a plain load + incref). Borrowed reads keep the plain load: they are only emitted + for attributes safe to borrow on free-threaded builds (Final and vec attrs -- see + transform_member_expr in irbuild), whose values live as long as their container. + The default (GIL) build always takes the plain-load path and increfs separately. + """ + use_get_attr_ref = ( + IS_FREE_THREADED and is_simple_refcounted_pointer(attr_rtype) and not op.is_borrowed + ) + if use_get_attr_ref and cl.is_final_attr(op.attr): + self.emitter.emit_line(f"{dest} = CPy_GetAttrRefFinal((PyObject **)&{attr_expr});") + elif use_get_attr_ref: + self.emitter.emit_line(f"{dest} = CPy_GetAttrRef((PyObject **)&{attr_expr});") + else: + self.emitter.emit_line(f"{dest} = {attr_expr};") + return use_get_attr_ref + def visit_get_attr(self, op: GetAttr) -> None: if op.allow_error_value: self.get_attr_with_allow_error_value(op) @@ -427,7 +463,9 @@ def visit_get_attr(self, op: GetAttr) -> None: else: # Otherwise, use direct or offset struct access. attr_expr = self.get_attr_expr(obj, op, decl_cl) - self.emitter.emit_line(f"{dest} = {attr_expr};") + use_get_attr_ref = self.emit_load_attr_take_ref( + dest, obj, op, cl, attr_rtype, attr_expr + ) always_defined = cl.is_always_defined(op.attr) merged_branch = None if not always_defined: @@ -458,7 +496,7 @@ def visit_get_attr(self, op: GetAttr) -> None: ) ) - if attr_rtype.is_refcounted and not op.is_borrowed: + if attr_rtype.is_refcounted and not op.is_borrowed and not use_get_attr_ref: if not merged_branch and not always_defined: self.emitter.emit_line("} else {") self.emitter.emit_inc_ref(dest, attr_rtype) @@ -480,16 +518,20 @@ def get_attr_with_allow_error_value(self, op: GetAttr) -> None: cl = rtype.class_ir attr_rtype, decl_cl = cl.attr_details(op.attr) - # Direct struct access without NULL check attr_expr = self.get_attr_expr(obj, op, decl_cl) - self.emitter.emit_line(f"{dest} = {attr_expr};") - - # Only emit inc_ref if not NULL - if attr_rtype.is_refcounted and not op.is_borrowed: - check = self.error_value_check(op, "!=") - self.emitter.emit_line(f"if ({check}) {{") - self.emitter.emit_inc_ref(dest, attr_rtype) - self.emitter.emit_line("}") + # On free-threaded builds this takes a new reference atomically (see + # emit_load_attr_take_ref). CPy_GetAttrRef returns NULL when the field is + # undefined, which is precisely the error value here, so the "NULL without + # AttributeError" behavior is preserved (this op has error_kind ERR_NEVER). + # is_simple_refcounted_pointer excludes tuples/vecs, so error_value_check's + # special cases only matter on the plain-load path (where no ref was taken). + if not self.emit_load_attr_take_ref(dest, obj, op, cl, attr_rtype, attr_expr): + # Only emit inc_ref if not NULL + if attr_rtype.is_refcounted and not op.is_borrowed: + check = self.error_value_check(op, "!=") + self.emitter.emit_line(f"if ({check}) {{") + self.emitter.emit_inc_ref(dest, attr_rtype) + self.emitter.emit_line("}") def next_branch(self) -> Branch | None: if self.op_index + 1 < len(self.ops): @@ -529,6 +571,23 @@ def visit_set_attr(self, op: SetAttr) -> None: op.attr, ) ) + elif IS_FREE_THREADED and is_simple_refcounted_pointer(attr_rtype): + # In free-threaded builds, publishing a single reference-counted + # 'PyObject *' field must be atomic so a concurrent reader (see + # CPy_GetAttrRef) never observes a torn pointer or a freed value. + # Both helpers steal the reference to src. + attr_expr = self.get_attr_expr(obj, op, decl_cl) + if op.is_init: + # The attribute is known to be previously undefined (NULL), so + # there is no old value to reclaim; a relaxed store suffices + # (self's later publication provides the release barrier -- see + # CPy_InitAttrRef). + self.emitter.emit_line(f"CPy_InitAttrRef((PyObject **)&{attr_expr}, {src});") + else: + # Atomically swap in the new value and reclaim the old one. + self.emitter.emit_line(f"CPy_SetAttrRef((PyObject **)&{attr_expr}, {src});") + if op.error_kind == ERR_FALSE: + self.emitter.emit_line(f"{dest} = 1;") else: # ...and struct access for normal attributes. attr_expr = self.get_attr_expr(obj, op, decl_cl) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index e2cd0a829dd77..3b320b5321232 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -62,6 +62,7 @@ LIBRT_BASE64, LIBRT_RANDOM, LIBRT_STRINGS, + LIBRT_THREADING, LIBRT_TIME, LIBRT_VECS, Capsule, @@ -210,15 +211,18 @@ def parse_and_typecheck( ) -> BuildResult: assert options.strict_optional, "strict_optional must be turned on" mypyc_plugin = MypycPlugin(options, compiler_options, groups) - result = build( - sources=sources, - options=options, - alt_lib_path=alt_lib_path, - fscache=fscache, - extra_plugins=[mypyc_plugin], - ) - mypyc_plugin.metastore.close() + try: + result = build( + sources=sources, + options=options, + alt_lib_path=alt_lib_path, + fscache=fscache, + extra_plugins=[mypyc_plugin], + ) + finally: + mypyc_plugin.metastore.close() if result.errors: + result.manager.metastore.close() raise CompileError(result.errors) return result @@ -1258,6 +1262,10 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_time() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") + if LIBRT_THREADING in module.dependencies: + emitter.emit_line("if (import_librt_threading() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") if LIBRT_VECS in module.dependencies: emitter.emit_line("if (import_librt_vecs() < 0) {") emitter.emit_line("return -1;") diff --git a/mypyc/codegen/literals.py b/mypyc/codegen/literals.py index ed1ff93277167..a8d4650b5c294 100644 --- a/mypyc/codegen/literals.py +++ b/mypyc/codegen/literals.py @@ -65,7 +65,8 @@ def record_literal(self, value: LiteralValue) -> None: elif isinstance(value, frozenset): frozenset_literals = self.frozenset_literals if value not in frozenset_literals: - for item in value: + # Sort members so that we don't depend on frozenset iteration order. + for item in sorted(value, key=literal_sort_key): assert _is_literal_value(item) self.record_literal(item) frozenset_literals[value] = len(frozenset_literals) @@ -140,10 +141,14 @@ def encoded_tuple_values(self) -> list[str]: return self._encode_collection_values(self.tuple_literals) def encoded_frozenset_values(self) -> list[str]: - return self._encode_collection_values(self.frozenset_literals) + # Ensure deterministic frozenset item order by sorting items. + return self._encode_collection_values(self.frozenset_literals, sort_items=True) def _encode_collection_values( - self, values: dict[tuple[object, ...], int] | dict[frozenset[object], int] + self, + values: dict[tuple[object, ...], int] | dict[frozenset[object], int], + *, + sort_items: bool = False, ) -> list[str]: """Encode tuple/frozenset values into a C array. @@ -164,7 +169,8 @@ def _encode_collection_values( for i in range(count): value = value_by_index[i] result.append(str(len(value))) - for item in value: + items = sorted(value, key=literal_sort_key) if sort_items else value + for item in items: assert _is_literal_value(item) index = self.literal_index(item) result.append(str(index)) @@ -299,3 +305,13 @@ def _encode_complex_values(values: dict[complex, int]) -> list[str]: result.append(float_to_c(value.real)) result.append(float_to_c(value.imag)) return result + + +def literal_sort_key(value: object) -> tuple[object, ...]: + """Return a sort key for a literal value.""" + if isinstance(value, frozenset): + # Sort items to avoid depending on the unpredictable iteration order. + return ("frozenset", tuple(sorted(literal_sort_key(item) for item in value))) + elif isinstance(value, tuple): + return ("tuple", tuple(literal_sort_key(item) for item in value)) + return (type(value).__name__, repr(value)) diff --git a/mypyc/common.py b/mypyc/common.py index 382d640a84083..fa34647c5c729 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -68,6 +68,14 @@ BITMAP_TYPE: Final = "uint32_t" BITMAP_BITS: Final = 32 +# Constant for keeping a (often borrowed) op alive for a short time (a few subexpressions, +# no arbitrary computation or memory allocations), until flush_keep_alives() is called. +KEEP_ALIVE_SHORT_LIVED: Final = 0 +# Keep value alive longer, up to until the current top-level expression has been fully +# evaluated. This allows keeping alive values across function calls and arbitrary +# computation. Note that some expressions (e.g. lambdas), restrict the scope of borrowing. +KEEP_ALIVE_WHOLE_EXPRESSION: Final = 1 + # Runtime C library files that are always included (some ops may bring # extra dependencies via mypyc.ir.deps.SourceDep or mypyc.ir.deps.HeaderDep) RUNTIME_C_FILES: Final = [ diff --git a/mypyc/doc/differences_from_python.rst b/mypyc/doc/differences_from_python.rst index 414476723125b..7e6cf37154d3e 100644 --- a/mypyc/doc/differences_from_python.rst +++ b/mypyc/doc/differences_from_python.rst @@ -237,62 +237,82 @@ nested function calls, typically due to out-of-control recursion. Final values ------------ -Compiled code replaces a reference to an attribute declared ``Final`` with -the value of the attribute computed at compile time. This is an example of -:ref:`early binding `. Example:: +Mypy treats variables and attributes defined as ``Final`` specially. +For instance attributes of native classes, mypyc prevents reassignment +of these attributes. + +Mypyc replaces references to a variable or attribute declared ``Final`` +with the value of the attribute computed at compile time, when it can +be determined during compilation. Example:: MAX: Final = 100 def limit_to_max(x: int) -> int: - if x > MAX: - return MAX - return x + if x > MAX: + return MAX + return x The two references to ``MAX`` don't involve any module namespace lookups, and are equivalent to this code:: def limit_to_max(x: int) -> int: - if x > 100: - return 100 - return x + if x > 100: + return 100 + return x When run as interpreted, the first example will execute slower due to -the extra namespace lookups. In interpreted code final attributes can -also be modified. +the extra namespace lookups. + +For a final class attribute or global variable whose value can't be +determined during compilation, mypyc defines a hidden native variable +that stores the value when the initialization assignment is evaluated. +Redefining the attribute using ``setattr`` or direct namespace access +has no effect for mypyc-compiled code (in the same compilation unit). + +Final global variables and class attributes are faster to read in +compiled code, since they don't have to be looked up from a namespace +dictionary. Thus using a ``Final`` object that holds mutable state in an +attribute tends to be faster than a plain global variable in compiled code:: + + class Counter: + def __init__(self) -> None: + self.value = 0 + + x: Final = Counter() + y = 0 + + def inc() -> None: + # Faster: no Python namespace lookup + x.value += 1 + # Slower: access through globals() namespace dictionary + global y + y += 1 .. _free-threading: Free threading -------------- -Mypyc has basic support for free threading, but it doesn't provide the -same memory safety guarantees as Python in compiled modules, since -in current Python versions this would cause an unacceptable performance -impact. - -The exact details of the memory safety in the presence of data races -are likely to evolve in the future. Currently, compiled code must -ensure that proper synchronization is used to prevent data races. In -particular, these operations require explicit synchronization, such as -via ``threading.Lock``, if there is a possibility of data races -(the list is not exhaustive): - -* Reads or writes of non-final instance data attributes of native - classes. -* List item access or iteration using a ``list`` static type (using - ``Sequence`` or ``MutableSequence`` as the type ensure correct - implicit synchronization). -* Dict item access using a static ``dict`` type (using ``Mapping`` - or ``MutableMapping`` ensures correct implicit synchronization). - -As libraries often won't be able to control the concurrent access by -user code, we recommend that modules document that multi-threaded -access is only supported via public interfaces that ensure correct -synchronization. Marking attributes as internal using an underscore -attribute prefix is another possibility, but this is not enforced at -runtime. Another option is to document that multithreaded access is -not supported, or that particular objects should not be used from -multiple threads concurrently. +Mypyc supports free threading. However, optimized primitive operations in +compiled code may have different atomicity properties compared to CPython. +Use explicit synchronization if code depends on operations being atomic and +race conditions are possible. This is already the recommended approach for +normal Python code. You can often use :ref:`librt.threading.Lock ` +(or :py:class:`threading.Lock`, which is less efficient than +``librt.threading.Lock``) to fix data races. + +Since mypyc 2.3, the vast majority of operations are memory safe even if +there are race conditions (unlike earlier mypyc releases). This includes +list operations and access to native instance attributes (except for +a few less common use cases that will be fixed in future releases). + +.. note:: + + Operations that aren't safe under race conditions in interpreted CPython + are not expected to be memory safe in compiled code either. + Some :ref:`librt ` features are heavily optimized for performance and + don't guarantee memory safety when there are race conditions + (notably the :ref:`vec ` type). It's always safe to perform read-only operations concurrently. Using objects with final attributes and tuple objects can help prevent diff --git a/mypyc/doc/index.rst b/mypyc/doc/index.rst index aacf275de9885..3f74595dd47fb 100644 --- a/mypyc/doc/index.rst +++ b/mypyc/doc/index.rst @@ -35,6 +35,7 @@ generate fast code. librt_base64 librt_random librt_strings + librt_threading librt_time librt_vecs diff --git a/mypyc/doc/librt.rst b/mypyc/doc/librt.rst index 23206f8cfe806..681efe47e7603 100644 --- a/mypyc/doc/librt.rst +++ b/mypyc/doc/librt.rst @@ -30,6 +30,8 @@ Follow submodule links in the table to a detailed description of each submodule. - Pseudorandom number generation * - :doc:`librt.strings ` - String and bytes utilities + * - :doc:`librt.threading ` + - Threading primitives * - :doc:`librt.time ` - Time utilities * - :doc:`librt.vecs ` diff --git a/mypyc/doc/librt_strings.rst b/mypyc/doc/librt_strings.rst index aece015f4f752..6bb4beef6f25e 100644 --- a/mypyc/doc/librt_strings.rst +++ b/mypyc/doc/librt_strings.rst @@ -15,7 +15,7 @@ Thread safety ``BytesWriter`` and ``StringWriter`` objects are unsafe to access from another thread if they are concurrently modified (on free-threaded Python builds). They are optimized for maximal performance, and they aren't fully synchronized. Read-only access from multiple -threads is safe. +threads is safe, as always. BytesWriter ^^^^^^^^^^^ @@ -101,6 +101,9 @@ StringWriter Functions --------- +Reading and writing binary data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + The ``write_*`` and ``read_*`` functions allow interpreting bytes as packed binary data. They can be used as (much) more efficient but lower-level alternatives to the stdlib :mod:`struct` module in compiled code. @@ -210,3 +213,60 @@ This example writes two binary values and reads them afterwards:: Read a 64-bit floating-point value starting at the given index as a big-endian binary value (8 bytes). + +Code point classification +^^^^^^^^^^^^^^^^^^^^^^^^^ + +These functions classify a single Unicode code point, passed as an ``i32`` integer. They are +faster alternatives to calling the corresponding :py:class:`str` methods on a one-character +string in compiled code. A code point is often obtained via ``ord(s[i])``, which is a +fast operation in compiled code when ``s`` has type :py:class:`str`. + +Each function agrees with the matching :py:class:`str` method applied to the one-character +string ``chr(c)``. Out-of-range inputs (negative values, or values past the maximum Unicode +code point ``0x10FFFF``) return ``False``. + +.. function:: isspace(c: i32, /) -> bool + + Return whether the code point is whitespace. Equivalent to ``chr(c).isspace()``. + +.. function:: isalpha(c: i32, /) -> bool + + Return whether the code point is alphabetic. Equivalent to ``chr(c).isalpha()``. + +.. function:: isdigit(c: i32, /) -> bool + + Return whether the code point is a digit. Equivalent to ``chr(c).isdigit()``. + +.. function:: isalnum(c: i32, /) -> bool + + Return whether the code point is alphanumeric. Equivalent to ``chr(c).isalnum()``. + +.. function:: isidentifier(c: i32, /) -> bool + + Return whether the code point is valid as the first character of a Python identifier. + Equivalent to ``chr(c).isidentifier()``. + +Code point case conversion +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These functions convert the case of a single Unicode code point, passed as an ``i32`` integer, +and return the converted code point as an ``i32``. They are faster alternatives to +:py:meth:`str.upper` / :py:meth:`str.lower` on a one-character string in compiled code. + +For the rare code points whose Unicode uppercase or lowercase form has multiple code points +(e.g. U+00DF ``ß`` has the upper case form ``"SS"``, and U+FB01 ``fi`` maps to ``"FI"``), the +input is returned unchanged, so the signature can stay ``i32 -> i32``. Use :py:meth:`str.upper` +/ :py:meth:`str.lower` when full Unicode case conversion matters, or implement the logic to +handle the special cases explicitly. Out-of-range inputs (negative values, or values past the +maximum Unicode code point ``0x10FFFF``) are returned unchanged. + +.. function:: toupper(c: i32, /) -> i32 + + Return the uppercase of the code point, or the input unchanged if the uppercase does not + consist of exactly one code point. + +.. function:: tolower(c: i32, /) -> i32 + + Return the lowercase of the code point, or the input unchanged if the lowercase does not + consist of exactly one code point. diff --git a/mypyc/doc/librt_threading.rst b/mypyc/doc/librt_threading.rst new file mode 100644 index 0000000000000..0af795bc0daab --- /dev/null +++ b/mypyc/doc/librt_threading.rst @@ -0,0 +1,56 @@ +.. _librt-threading: + +librt.threading +=============== + +The ``librt.threading`` module is part of the ``librt`` package on PyPI, and it includes +threading primitives. + +Classes +------- + +.. _librt-threading-lock: + +Lock +^^^^ + +.. class:: Lock + + A fast mutual exclusion lock. This can be used as a faster replacement for + :py:class:`threading.Lock` in compiled code. + + Like :py:class:`threading.Lock`, a ``Lock`` is *unowned*: it may be released by a thread + other than the one that acquired it, and it doesn't support reentrant (recursive) locking. + A newly created lock is unlocked. + + ``Lock`` can be used as a context manager. The lock is acquired (blocking) on entry and + released on exit, including when the body raises an exception:: + + def example(lock: Lock) -> None: + with lock: + ... # Critical section; the lock is held here. + + ``Lock`` cannot be subclassed. ``Lock`` cannot be used with :py:class:`threading.Condition`. + + .. method:: acquire(blocking: bool = True) -> bool + + Acquire the lock. + + When *blocking* is true (the default), block (if needed) until the lock is available, + acquire it, and return ``True``. When *blocking* is false, acquire the lock only if it + can be done without blocking: return ``True`` if the lock could be acquired, or + ``False`` otherwise (it was already locked by some thread). + + Unlike :py:meth:`threading.Lock.acquire`, there is no *timeout* argument. + + .. method:: release() -> None + + Release the lock, allowing another thread (if any) that is blocked on :meth:`acquire` + to proceed. Since the lock is unowned, it may be released from a thread other than the + one that acquired it. + + Raise :py:exc:`RuntimeError` if the lock is not currently held. + + .. method:: locked() -> bool + + Return ``True`` if the lock is currently held (by any thread). diff --git a/mypyc/doc/librt_vecs.rst b/mypyc/doc/librt_vecs.rst index dad3be621ff4c..a7f5bc3ffbd96 100644 --- a/mypyc/doc/librt_vecs.rst +++ b/mypyc/doc/librt_vecs.rst @@ -1,3 +1,5 @@ +.. _librt-vecs: + librt.vecs ========== @@ -198,15 +200,21 @@ with no unnecessary temporary objects. Thread safety ------------- +The ``vec`` type is heavily optimized for performance, and this means that ``vec`` +gives fewer thread safety guarantees than built-in types such as ``list``. +``vec`` is a lower-level type where implicit synchronization would have a very +significant performance cost. However, since vec lengths are immutable, some race +conditions that lists can be susceptible to are not possible with vecs. + In free-threaded Python builds, it's unsafe to write or modify an item if other threads might be concurrently accessing *the same item*. For example, writing ``v[4]`` is not safe to do if another thread might be reading ``v[4]``. Similarly, two threads concurrently calling ``append`` or ``remove`` on the same vec object is not safe. -This is different from list objects, since vec is a lower-level type where implicit -synchronization would have a significant performance cost. However, since vec lengths -are immutable, some race conditions that lists can be susceptible to are not possible -with vecs. +Similarly, assigning to an instance attribute of a native class with a ``vec`` type +is unsafe if another thread might be accessing the same attribute concurrently +(in a free-threaded Python build only). Using ``Final`` attributes is a way +to prevent this race condition, since ``Final`` attributes cannot be rebound. Implementation details ---------------------- diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index f754275480edc..3b54331cb2f07 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -7,7 +7,7 @@ from mypyc.common import PROPSET_PREFIX, JsonDict from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import DeserMaps, Value -from mypyc.ir.rtypes import RInstance, RType, deserialize_type, object_rprimitive +from mypyc.ir.rtypes import RInstance, RType, c_int_rprimitive, deserialize_type, object_rprimitive from mypyc.namegen import NameGenerator, exported_name # Some notes on the vtable layout: Each concrete class has a vtable @@ -143,8 +143,27 @@ def __init__( module_name, FuncSignature([RuntimeArg("type", object_rprimitive)], RInstance(self)), ) + self.clear = FuncDecl( + name + "_clear", + None, + module_name, + FuncSignature([RuntimeArg("self", RInstance(self))], c_int_rprimitive), + internal=True, + ) + self.clear_on_completion = FuncDecl( + name + "_clear_on_completion", + None, + module_name, + FuncSignature([RuntimeArg("self", RInstance(self))], c_int_rprimitive), + internal=True, + ) # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} + # Attributes that must survive generator/coroutine completion because + # escaped nested functions may still read them as closure variables. + self.attrs_to_keep_alive_on_completion: set[str] = set() + # Final attributes defined in the class (not inherited) + self.final_attributes: set[str] = set() # Deletable attributes self.deletable: list[str] = [] # We populate method_types with the signatures of every method before @@ -267,6 +286,22 @@ def attr_details(self, name: str) -> tuple[RType, ClassIR]: def attr_type(self, name: str) -> RType: return self.attr_details(name)[0] + def is_final_attr(self, name: str) -> bool: + """Is the (possibly inherited) attribute Final, i.e. never rebound? + + A Final attribute is read-only at runtime (it has no setter) and is assigned + exactly once during construction, so it can never be reassigned afterwards. + This makes it safe to borrow on free-threaded builds (no concurrent store can + invalidate a borrowed reference) and lets reads skip the concurrent-writer + guard. Returns False for properties and for attributes this class doesn't have. + """ + for ir in self.mro: + if name in ir.attributes: + return name in ir.final_attributes + if name in ir.property_types: + return False + return False + def method_decl(self, name: str) -> FuncDecl: for ir in self.mro: if name in ir.method_decls: @@ -394,8 +429,12 @@ def serialize(self) -> JsonDict: "_serializable": self._serializable, "builtin_base": self.builtin_base, "ctor": self.ctor.serialize(), + "clear": self.clear.serialize(), + "clear_on_completion": self.clear_on_completion.serialize(), # We serialize dicts as lists to ensure order is preserved "attributes": [(k, t.serialize()) for k, t in self.attributes.items()], + "attrs_to_keep_alive_on_completion": sorted(self.attrs_to_keep_alive_on_completion), + "final_attributes": sorted(self.final_attributes), # We try to serialize a name reference, but if the decl isn't in methods # then we can't be sure that will work so we serialize the whole decl. "method_decls": [ @@ -455,7 +494,11 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir._serializable = data["_serializable"] ir.builtin_base = data["builtin_base"] ir.ctor = FuncDecl.deserialize(data["ctor"], ctx) + ir.clear = FuncDecl.deserialize(data["clear"], ctx) + ir.clear_on_completion = FuncDecl.deserialize(data["clear_on_completion"], ctx) ir.attributes = {k: deserialize_type(t, ctx) for k, t in data["attributes"]} + ir.attrs_to_keep_alive_on_completion = set(data["attrs_to_keep_alive_on_completion"]) + ir.final_attributes = set(data["final_attributes"]) ir.method_decls = { k: ctx.functions[v].decl if isinstance(v, str) else FuncDecl.deserialize(v, ctx) for k, v in data["method_decls"] diff --git a/mypyc/ir/deps.py b/mypyc/ir/deps.py index 751845d3a324c..b1a27f85fcfe6 100644 --- a/mypyc/ir/deps.py +++ b/mypyc/ir/deps.py @@ -110,6 +110,7 @@ def get_header(self) -> str: LIBRT_VECS: Final = Capsule("librt.vecs") LIBRT_TIME: Final = Capsule("librt.time") LIBRT_RANDOM: Final = Capsule("librt.random") +LIBRT_THREADING: Final = Capsule("librt.threading") BYTES_EXTRA_OPS: Final = SourceDep("bytes_extra_ops.c") BYTES_WRITER_EXTRA_OPS: Final = SourceDep("byteswriter_extra_ops.c") diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 485aa84886b32..14e12559be1eb 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -31,7 +31,7 @@ class to enable the new behavior. Sometimes adding a new abstract from mypy_extensions import trait -from mypyc.common import PROPSET_PREFIX +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, PROPSET_PREFIX from mypyc.ir.deps import Dependency from mypyc.ir.rtypes import ( RArray, @@ -39,6 +39,7 @@ class to enable the new behavior. Sometimes adding a new abstract RStruct, RTuple, RType, + RTypeVar, RUnion, RVec, RVoid, @@ -703,7 +704,7 @@ def __init__( self, name: str, arg_types: list[RType], - return_type: RType, # TODO: What about generic? + return_type: RType, var_arg_type: RType | None, truncated_type: RType | None, c_function_name: str | None, @@ -716,6 +717,7 @@ def __init__( is_pure: bool, experimental: bool, dependencies: list[Dependency] | None, + type_params: list[RTypeVar] | None, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -749,6 +751,7 @@ def __init__( # If this flag is set, the primitive has native integer types and must # be matched using more complex rules. self.is_ambiguous = any(has_fixed_width_int(t) for t in arg_types) + self.type_params = None if not type_params else type_params def __repr__(self) -> str: return f"" @@ -776,11 +779,23 @@ class PrimitiveOp(RegisterOp): code paths for short and long representations. """ - def __init__(self, args: list[Value], desc: PrimitiveDescription, line: int = -1) -> None: + def __init__( + self, + args: list[Value], + desc: PrimitiveDescription, + line: int = -1, + *, + arg_types: list[RType] | None = None, + return_type: RType | None = None, + type_args: list[RType] | None = None, + ) -> None: self.error_kind = desc.error_kind super().__init__(line) self.args = args - self.type = desc.return_type + self.arg_types = arg_types if arg_types is not None else desc.arg_types + self.type = return_type if return_type is not None else desc.return_type + self.is_borrowed = desc.is_borrowed + self.type_args = type_args self.desc = desc def sources(self) -> list[Value]: @@ -883,6 +898,7 @@ def __init__( *, borrow: bool = False, allow_error_value: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> None: super().__init__(line) self.obj = obj @@ -897,6 +913,8 @@ def __init__( elif attr_type.error_overlap: self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted + # How long a borrowed result of this op stays valid (a KEEP_ALIVE_* constant). + self.borrow_scope = borrow_scope def sources(self) -> list[Value]: return [self.obj] diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index d0db9f2460a1d..734426ca42de9 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -231,10 +231,15 @@ def visit_call_c(self, op: CallC) -> str: def visit_primitive_op(self, op: PrimitiveOp) -> str: args_str = ", ".join(self.format("%r", arg) for arg in op.args) + if op.type_args: + joined = ", ".join(str(arg) for arg in op.type_args) + type_args = f"[{joined}]" + else: + type_args = "" if op.is_void: - return self.format("%s %s", op.desc.name, args_str) + return self.format("%s%s %s", op.desc.name, type_args, args_str) else: - return self.format("%r = %s %s", op, op.desc.name, args_str) + return self.format("%r = %s%s %s", op, op.desc.name, type_args, args_str) def visit_truncate(self, op: Truncate) -> str: return self.format("%r = truncate %r: %t to %t", op, op.src, op.src_type, op.type) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index db29f9e304d8d..afe3d3e9df39b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -41,7 +41,7 @@ class to enable the new behavior. In rare cases, adding a new from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, Union, final from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name -from mypyc.ir.deps import LIBRT_RANDOM, LIBRT_STRINGS, LIBRT_VECS, Dependency +from mypyc.ir.deps import LIBRT_RANDOM, LIBRT_STRINGS, LIBRT_THREADING, LIBRT_VECS, Dependency from mypyc.namegen import NameGenerator if TYPE_CHECKING: @@ -152,6 +152,9 @@ def visit_rprimitive(self, typ: RPrimitive, /) -> T: def visit_rinstance(self, typ: RInstance, /) -> T: raise NotImplementedError + def visit_rtypevar(self, typ: RTypeVar, /) -> T: + raise RuntimeError("RTypeVar should not be encountered here") + @abstractmethod def visit_rvec(self, typ: RVec, /) -> T: raise NotImplementedError @@ -547,18 +550,38 @@ def __hash__(self) -> int: } | { "librt.random.Random": RPrimitive( "librt.random.Random", is_unboxed=False, is_refcounted=True, dependencies=(LIBRT_RANDOM,) - ) + ), + "librt.threading.Lock": RPrimitive( + "librt.threading.Lock", + is_unboxed=False, + is_refcounted=True, + dependencies=(LIBRT_THREADING,), + ), } bytes_writer_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.strings.BytesWriter"] string_writer_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.strings.StringWriter"] random_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.random.Random"] +lock_rprimitive: Final = KNOWN_NATIVE_TYPES["librt.threading.Lock"] def is_native_rprimitive(rtype: RType) -> bool: return isinstance(rtype, RPrimitive) and rtype.name in KNOWN_NATIVE_TYPES +def is_simple_refcounted_pointer(rtype: RType) -> bool: + """Is rtype represented at runtime as a single, reference-counted 'PyObject *'? + + This covers 'object', 'str', containers, instances of native classes and + optional/union types -- everything whose C representation is exactly one + 'PyObject *' field that owns a reference. It excludes unboxed types (tagged + 'int', fixed-width ints, floats, bools), inline tuples ('RTuple'), vectors + ('RVec') and C structs ('RStruct'), which need different treatment for + free-threaded memory safety. + """ + return rtype.is_refcounted and not rtype.is_unboxed and not isinstance(rtype, RStruct) + + def is_tagged(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive or rtype is short_int_rprimitive @@ -747,6 +770,12 @@ def visit_rarray(self, t: RArray) -> str: def visit_rvoid(self, t: RVoid) -> str: assert False, "rvoid in tuple?" + def visit_rtypevar(self, typ: RTypeVar) -> str: + # We need to return something to support generic RTuples, etc. Make sure + # the return value is invalid C so that generic RTuples must be expanded + # before they can be used in IR. + return f"!RTypeVar {typ.id} invalid!" + @final class RTuple(RType): @@ -1013,6 +1042,50 @@ def serialize(self) -> str: return self.name +@final +class RTypeVar(RType): + """Type variable type used for generic primitive ops. + + This allows having generic primitive operations like vec get item, which is + parametrized by the vec item type. + + These types are not valid in any other context outside PrimitiveDescription, + and they will always be substituted during the construction of a PrimitiveOp. + + NOTE: This is not related to mypy's TypeVarType! + """ + + def __init__(self, id: int) -> None: + self.id = id + + @property + def may_be_immortal(self) -> bool: + # RTypeVar must always be substituted before use, so this should never matter. + return False + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rtypevar(self) + + def __str__(self) -> str: + return f"" + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> TypeGuard[RTypeVar]: + return isinstance(other, RTypeVar) and other.id == self.id + + def __hash__(self) -> int: + return self.id ^ 12345 + + def serialize(self) -> JsonDict: + return {".class": "RTypeVar", "id": self.id} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RTypeVar: + return RTypeVar(data["id"]) + + @final class RVec(RType): """librt.vecs.vec[T]""" diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 587be873a46d9..f4e3745836cf5 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -20,12 +20,17 @@ TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, ArgKind, + AssignmentExpr, + AwaitExpr, CallExpr, Decorator, + DictionaryComprehension, Expression, FuncDef, + GeneratorExpr, IndexExpr, IntExpr, + LambdaExpr, Lvalue, MemberExpr, MypyFile, @@ -41,7 +46,10 @@ TypeInfo, TypeParam, Var, + YieldExpr, + YieldFromExpr, ) +from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, DeletedType, @@ -62,6 +70,9 @@ BITMAP_BITS, EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, + IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -79,6 +90,7 @@ BasicBlock, Branch, Call, + Cast, ComparisonOp, GetAttr, InitStatic, @@ -146,7 +158,12 @@ from mypyc.options import CompilerOptions from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op from mypyc.primitives.generic_ops import iter_op, next_op, py_setattr_op -from mypyc.primitives.list_ops import list_get_item_unsafe_op, list_pop_last, to_list +from mypyc.primitives.list_ops import ( + list_get_item_int64_op, + list_get_item_unsafe_op, + list_pop_last, + to_list, +) from mypyc.primitives.misc_ops import ( check_unpack_count_op, get_module_dict_op, @@ -278,6 +295,20 @@ def __init__( self.imports: dict[str, None] = {} self.can_borrow = False + self.expression_depth = 0 + # Symbols (local vars) reassigned via a walrus expression within the current + # top-level expression. Used to avoid borrowing an attribute over the whole + # expression when the borrow root could be rebound (and thus freed) partway. + self.reassigned_in_expr: set[SymbolNode] = set() + # Whether the current top-level expression contains a suspension point + # (await, yield or yield from). A whole-expression borrow can't span such a + # point, since the borrowed value (and its root) live in registers that are + # not spilled into the generator environment across the suspend. + self.expr_has_suspend = False + # Saved expression state for enclosing functions (see enter()/leave()). + self.expression_depth_stack: list[int] = [] + self.reassigned_in_expr_stack: list[set[SymbolNode]] = [] + self.expr_has_suspend_stack: list[bool] = [] # When set, load_globals_dict uses this module instead of self.module_name. # Used by generate_attr_defaults_init for cross-module inherited defaults. @@ -309,6 +340,10 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V """ with self.catch_errors(node.line): if isinstance(node, Expression): + self.expression_depth += 1 + if self.expression_depth == 1: + self.reassigned_in_expr = find_walrus_targets(node) + self.expr_has_suspend = expr_has_suspend(node) old_can_borrow = self.can_borrow self.can_borrow = can_borrow try: @@ -323,6 +358,11 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V self.can_borrow = old_can_borrow if not can_borrow: self.flush_keep_alives(node.line) + self.expression_depth -= 1 + if self.expression_depth == 0: + self.flush_keep_alives(node.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) + self.reassigned_in_expr = set() + self.expr_has_suspend = False return res else: try: @@ -331,8 +371,8 @@ def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> V pass return None - def flush_keep_alives(self, line: int) -> None: - self.builder.flush_keep_alives(line) + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + self.builder.flush_keep_alives(line, scope=scope) # Pass through methods for the most common low-level builder ops, for convenience. @@ -893,7 +933,10 @@ def process_sequence_assignment( index: Value if is_list_rprimitive(rvalue.type): index = Integer(i, c_pyssize_t_rprimitive) - item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) + if not IS_FREE_THREADED: + item_value = self.primitive_op(list_get_item_unsafe_op, [rvalue, index], line) + else: + item_value = self.primitive_op(list_get_item_int64_op, [rvalue, index], line) elif is_tuple_rprimitive(rvalue.type): index = Integer(i, c_pyssize_t_rprimitive) item_value = self.call_c(tuple_get_item_unsafe_op, [rvalue, index], line) @@ -1183,7 +1226,9 @@ def is_synthetic_type(self, typ: TypeInfo) -> bool: return typ.is_named_tuple or typ.is_newtype or typ.typeddict_type is not None def get_final_ref(self, expr: MemberExpr) -> tuple[str, Var, bool] | None: - """Check if `expr` is a final attribute. + """Check if `expr` is a final class or module attribute. + + Return False for instance attributes. This needs to be done differently for class and module attributes to correctly determine fully qualified name. Return a tuple that consists of @@ -1332,6 +1377,15 @@ def enter(self, fn_info: FuncInfo | str = "", *, ret_type: RType = none_rprimiti self.fn_info = fn_info self.fn_infos.append(self.fn_info) self.ret_types.append(ret_type) + # A function body is its own top-level expression context, even when the + # function (e.g. a lambda) is being generated in the middle of an outer + # expression. Save the outer expression state and start fresh. + self.expression_depth_stack.append(self.expression_depth) + self.reassigned_in_expr_stack.append(self.reassigned_in_expr) + self.expr_has_suspend_stack.append(self.expr_has_suspend) + self.expression_depth = 0 + self.reassigned_in_expr = set() + self.expr_has_suspend = False if fn_info.is_generator: self.nonlocal_control.append(GeneratorNonlocalControl()) else: @@ -1345,6 +1399,9 @@ def leave(self) -> tuple[list[Register], list[RuntimeArg], list[BasicBlock], RTy ret_type = self.ret_types.pop() fn_info = self.fn_infos.pop() self.nonlocal_control.pop() + self.expression_depth = self.expression_depth_stack.pop() + self.reassigned_in_expr = self.reassigned_in_expr_stack.pop() + self.expr_has_suspend = self.expr_has_suspend_stack.pop() self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] return builder.args, runtime_args, builder.blocks, ret_type, fn_info @@ -1380,6 +1437,27 @@ def enter_scope(self, fn_info: FuncInfo) -> Iterator[None]: self.builder = self.builders[-1] self.fn_info = self.fn_infos[-1] + @contextmanager + def enter_borrow_scope(self, line: int) -> Iterator[None]: + """Enter new borrow scope from which borrows can't leak to outer expressions. + + This is a borrow region (see LowLevelIRBuilder.borrow_region) that also + resets the per-expression borrowing heuristic state, since the body forms + its own top-level expression context (e.g. a comprehension iteration or a + lambda body). + """ + old_expression_depth = self.expression_depth + old_reassigned_in_expr = self.reassigned_in_expr + old_expr_has_suspend = self.expr_has_suspend + self.expression_depth = 0 + try: + with self.builder.borrow_region(line): + yield + finally: + self.expression_depth = old_expression_depth + self.reassigned_in_expr = old_reassigned_in_expr + self.expr_has_suspend = old_expr_has_suspend + @contextmanager def enter_method( self, @@ -1490,12 +1568,15 @@ def add_var_to_env_class( base: FuncInfo | ImplicitClass, reassign: bool = False, always_defined: bool = False, + keep_alive_on_completion: bool = False, prefix: str = "", ) -> AssignmentTarget: # First, define the variable name as an attribute of the environment class, and then # construct a target for that attribute. name = prefix + remangle_redefinition_name(var.name) self.fn_info.env_class.attributes[name] = rtype + if keep_alive_on_completion: + self.fn_info.env_class.attrs_to_keep_alive_on_completion.add(name) if always_defined: self.fn_info.env_class.attrs_with_defaults.add(name) attr_target = AssignmentTargetAttr(base.curr_env_reg, name) @@ -1556,6 +1637,46 @@ def is_native_attr_ref(self, expr: MemberExpr) -> bool: and any(expr.name in ir.attributes for ir in obj_rtype.class_ir.mro) ) + def is_final_native_attr_ref(self, expr: MemberExpr) -> bool: + """Is expr a direct reference to a Final native (struct) attribute of an instance? + + A Final attribute is read-only at runtime (it has no setter), so it can never be + reassigned after construction. This makes it safe to borrow even on free-threaded + builds, since no concurrent store can invalidate the borrowed reference. + """ + obj_rtype = self.node_type(expr.expr) + return ( + isinstance(obj_rtype, RInstance) + and obj_rtype.class_ir.is_ext_class + and obj_rtype.class_ir.is_final_attr(expr.name) + ) + + def root_is_reassigned(self, v: Value) -> bool: + """Is the root local variable a borrow chain 'v' reads from reassigned this expression? + + A whole-expression borrow of an attribute keeps the borrow root alive only + via the register holding it. If that register belongs to a local variable + that is rebound (via a walrus assignment) during the same top-level + expression, the old value may be freed while the borrow is still live. + """ + if not self.reassigned_in_expr: + return False + # Peel borrowed links back to the root value the chain reads from. + while True: + if isinstance(v, GetAttr) and v.is_borrowed: + v = v.obj + elif isinstance(v, Cast) and v.is_borrowed: + v = v.src + else: + break + if not isinstance(v, Register): + return False + for symbol in self.reassigned_in_expr: + target = self.symtables[-1].get(symbol) + if isinstance(target, AssignmentTargetRegister) and target.register is v: + return True + return False + def mark_block_unreachable(self) -> None: """Mark statements in the innermost block being processed as unreachable. @@ -1670,6 +1791,78 @@ def get_call_target_fullname(ref: RefExpr) -> str: return ref.fullname +class WalrusTargetCollector(TraverserVisitor): + """Collect the symbols assigned to by walrus expressions in a subtree.""" + + def __init__(self) -> None: + self.targets: set[SymbolNode] = set() + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + if o.target.node is not None: + self.targets.add(o.target.node) + super().visit_assignment_expr(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own expression context, so don't descend into it. + pass + + +def find_walrus_targets(expr: Expression) -> set[SymbolNode]: + """Return the symbols reassigned via a walrus expression within 'expr'. + + Walrus (':=') is the only way to rebind a variable in the middle of evaluating + an expression, so this is the complete set of in-expression reassignments. + """ + collector = WalrusTargetCollector() + expr.accept(collector) + return collector.targets + + +class SuspendDetector(TraverserVisitor): + """Detect await/yield/yield from expressions in a subtree.""" + + def __init__(self) -> None: + self.found = False + + def visit_await_expr(self, o: AwaitExpr) -> None: + self.found = True + + def visit_yield_expr(self, o: YieldExpr) -> None: + self.found = True + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + self.found = True + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + # An 'async for' clause suspends via an implicit await on __anext__ that + # isn't represented as an AwaitExpr node in the AST (list/set comprehensions + # delegate to a GeneratorExpr, so they are covered here too). + if any(o.is_async): + self.found = True + super().visit_generator_expr(o) + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + if any(o.is_async): + self.found = True + super().visit_dictionary_comprehension(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + # A lambda body forms its own function (and suspension) context. + pass + + +def expr_has_suspend(expr: Expression) -> bool: + """Does evaluating 'expr' involve a suspension point (await/yield/yield from)? + + A whole-expression borrow can't safely span a suspension point, since the + borrowed value and its borrow root are held in registers that aren't spilled + into the generator environment across the suspend. + """ + detector = SuspendDetector() + expr.accept(detector) + return detector.found + + def create_type_params( builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int ) -> list[Value]: diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 4fd9a2e2cc748..61587b46161f7 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -760,7 +760,6 @@ def find_attr_initializers( if cls.builtin_base: return set(), [] - cls_type = dataclass_type(cdef) attrs_with_defaults: set[str] = set() default_assignments: list[tuple[AssignmentStmt, str]] = [] @@ -774,10 +773,11 @@ def find_attr_initializers( info_ir = builder.mapper.type_to_ir.get(info) if info_ir is None: continue + info_cls_type = dataclass_type(info.defn) for stmt in info.defn.defs.body: if not isinstance(stmt, AssignmentStmt): continue - name = default_attr_name(stmt, info_ir, cls_type) + name = default_attr_name(stmt, info_ir, info_cls_type) if name is None: continue attrs_with_defaults.add(name) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 3128543d4cd2c..c8c5f7fa68593 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -17,7 +17,7 @@ def g() -> int: from __future__ import annotations -from mypy.nodes import Argument, FuncDef, SymbolNode, Var +from mypy.nodes import Argument, FuncDef, FuncItem, SymbolNode, Var from mypyc.common import ( BITMAP_BITS, ENV_ATTR_NAME, @@ -60,6 +60,8 @@ class is generated, the function environment has not yet been # If the function is nested, its environment class must contain an environment # attribute pointing to its encapsulating functions' environment class. env_class.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + if builder.fn_info.contains_nested: + env_class.attrs_to_keep_alive_on_completion.add(ENV_ATTR_NAME) env_class.mro = [env_class] builder.fn_info.env_class = env_class builder.classes.append(env_class) @@ -229,7 +231,17 @@ def add_args_to_env( rtype = builder.type_to_rtype(arg.variable.type) assert base is not None, "base cannot be None for adding nonlocal args" builder.add_var_to_env_class( - arg.variable, rtype, base, reassign=reassign, prefix=prefix + arg.variable, + rtype, + base, + reassign=reassign, + keep_alive_on_completion=( + is_free_variable(builder, arg.variable) + or is_free_variable_in_nested_func( + builder, builder.fn_info.fitem, arg.variable + ) + ), + prefix=prefix, ) @@ -256,7 +268,12 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None: if isinstance(var, Var): rtype = builder.type_to_rtype(var.type) builder.add_var_to_env_class( - var, rtype, env_for_func, reassign=False, prefix=prefix + var, + rtype, + env_for_func, + reassign=False, + keep_alive_on_completion=True, + prefix=prefix, ) if builder.fn_info.fitem in builder.encapsulating_funcs: @@ -267,10 +284,16 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None: # the same name and signature across conditional blocks # will generate different callable classes, so the callable # class that gets instantiated must be generic. + nested_prefix = prefix if nested_fn.is_generator or nested_fn.is_coroutine: - prefix = GENERATOR_ATTRIBUTE_PREFIX + nested_prefix = GENERATOR_ATTRIBUTE_PREFIX builder.add_var_to_env_class( - nested_fn, object_rprimitive, env_for_func, reassign=False, prefix=prefix + nested_fn, + object_rprimitive, + env_for_func, + reassign=False, + keep_alive_on_completion=is_free_variable(builder, nested_fn), + prefix=nested_prefix, ) @@ -308,3 +331,14 @@ def setup_func_for_recursive_call( def is_free_variable(builder: IRBuilder, symbol: SymbolNode) -> bool: fitem = builder.fn_info.fitem return fitem in builder.free_variables and symbol in builder.free_variables[fitem] + + +def is_free_variable_in_nested_func( + builder: IRBuilder, fitem: FuncItem, symbol: SymbolNode +) -> bool: + for nested in builder.encapsulating_funcs.get(fitem, []): + if symbol in builder.free_variables.get(nested, set()): + return True + if is_free_variable_in_nested_func(builder, nested, symbol): + return True + return False diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index cd7295ef709ad..91e1e55d8da26 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -58,7 +58,12 @@ TypeType, get_proper_type, ) -from mypyc.common import MAX_SHORT_INT +from mypyc.common import ( + IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, + MAX_SHORT_INT, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.ir.ops import ( @@ -66,11 +71,14 @@ Assign, BasicBlock, CallC, + Cast, ComparisonOp, + GetAttr, Integer, LoadAddress, LoadLiteral, PrimitiveDescription, + PrimitiveOp, RaiseStandardError, Register, TupleGet, @@ -252,7 +260,7 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if expr.fullname in ("typing.TYPE_CHECKING", "typing_extensions.TYPE_CHECKING"): return builder.false(expr.line) - # First check if this is maybe a final attribute. + # First check if this is maybe a final class/module attribute. final = builder.get_final_ref(expr) if final is not None: fullname, final_var, native = final @@ -272,9 +280,16 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: if isinstance(expr.node, MypyFile) and expr.node.fullname in builder.imports: return builder.load_module(expr.node.fullname) - can_borrow = builder.is_native_attr_ref(expr) - obj = builder.accept(expr.expr, can_borrow=can_borrow) rtype = builder.node_type(expr) + # Borrowing a native attribute read is unsafe on free-threaded builds, since another + # thread could concurrently reassign the attribute and free the old value. We still borrow + # in two cases: + # - Native Final attributes are read-only at runtime, so they can never be reassigned. + # - Vec-typed attributes require manual synchronization, so we borrow them liberally. + can_borrow = builder.is_native_attr_ref(expr) and ( + not IS_FREE_THREADED or isinstance(rtype, RVec) or builder.is_final_native_attr_ref(expr) + ) + obj = builder.accept(expr.expr, can_borrow=can_borrow) if ( is_object_rprimitive(obj.type) @@ -298,8 +313,43 @@ def transform_member_expr(builder: IRBuilder, expr: MemberExpr) -> Value: check_instance_attribute_access_through_class(builder, expr, typ) - borrow = can_borrow and builder.can_borrow - return builder.builder.get_attr(obj, expr.name, rtype, expr.line, borrow=borrow) + is_final = builder.is_final_native_attr_ref(expr) + scope = KEEP_ALIVE_SHORT_LIVED + if ( + is_final + and builder.expression_depth > 1 + and value_borrow_scope(builder, obj) >= KEEP_ALIVE_WHOLE_EXPRESSION + # Don't borrow across the whole expression if the borrow root can be + # rebound via a walrus assignment + and not builder.root_is_reassigned(obj) + # Don't borrow across a suspension point (await/yield/yield from), since + # the borrow would not survive the suspend (registers aren't spilled). + and not builder.expr_has_suspend + ): + scope = KEEP_ALIVE_WHOLE_EXPRESSION + borrow = (can_borrow and builder.can_borrow) or scope == KEEP_ALIVE_WHOLE_EXPRESSION + return builder.builder.get_attr( + obj, expr.name, rtype, expr.line, borrow=borrow, borrow_scope=scope + ) + + +def value_borrow_scope(builder: IRBuilder, v: Value) -> int: + """Compute how long an existing borrowed value can safely be kept alive. + + Returns a KEEP_ALIVE_* constant (or a large value if 'v' is not borrowed and + thus has no borrowing constraint). + """ + if isinstance(v, GetAttr) and v.is_borrowed: + return min(v.borrow_scope, value_borrow_scope(builder, v.obj)) + elif isinstance(v, Cast) and v.is_borrowed: + # A cast just propagates the value (when successful), so the scope is + # determined by the source. + return value_borrow_scope(builder, v.src) + elif isinstance(v, (CallC, PrimitiveOp)) and v.is_borrowed: + # Values borrowed from a C function (e.g. a borrowed list/vec item) may be + # invalidated by arbitrary computation, so they are only short-lived. + return KEEP_ALIVE_SHORT_LIVED + return 999 def check_instance_attribute_access_through_class( @@ -773,8 +823,14 @@ def try_optimize_int_floor_divide(builder: IRBuilder, expr: OpExpr) -> OpExpr: def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: index = expr.index base_type = builder.node_type(expr.base) - can_borrow = is_list_rprimitive(base_type) or isinstance(base_type, RVec) - can_borrow_base = can_borrow and is_borrow_friendly_expr(builder, index) + # We can borrow a list item safely only if GIL is enabled. The vec type is optimized for + # performance, so we'll do unsafe borrowing. + can_borrow = (is_list_rprimitive(base_type) and not IS_FREE_THREADED) or isinstance( + base_type, RVec + ) + can_borrow_base = ( + is_list_rprimitive(base_type) or isinstance(base_type, RVec) + ) and is_borrow_friendly_expr(builder, index) # Check for dunder specialization for non-slice indexing if not isinstance(index, SliceExpr): @@ -796,7 +852,7 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: if value: return value - index_reg = builder.accept(expr.index, can_borrow=can_borrow) + index_reg = builder.accept(expr.index, can_borrow=can_borrow or can_borrow_base) return builder.builder.get_item( base, index_reg, builder.node_type(expr), expr.line, can_borrow=builder.can_borrow ) @@ -862,15 +918,17 @@ def transform_conditional_expr(builder: IRBuilder, expr: ConditionalExpr) -> Val target = Register(expr_type) builder.activate_block(if_body) - true_value = builder.accept(expr.if_expr) - true_value = builder.coerce(true_value, expr_type, expr.line) - builder.add(Assign(target, true_value, expr.line)) + with builder.builder.borrow_region(expr.line): + true_value = builder.accept(expr.if_expr) + true_value = builder.coerce(true_value, expr_type, expr.line) + builder.add(Assign(target, true_value, expr.line)) builder.goto(next_block) builder.activate_block(else_body) - false_value = builder.accept(expr.else_expr) - false_value = builder.coerce(false_value, expr_type, expr.line) - builder.add(Assign(target, false_value, expr.line)) + with builder.builder.borrow_region(expr.line): + false_value = builder.accept(expr.else_expr) + false_value = builder.coerce(false_value, expr_type, expr.line) + builder.add(Assign(target, false_value, expr.line)) builder.goto(next_block) builder.activate_block(next_block) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 95894dcedaba8..5a36dcb80ff7d 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -28,6 +28,7 @@ Var, ) from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types +from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -81,7 +82,12 @@ ) from mypyc.primitives.exc_ops import no_err_occurred_op, propagate_if_error_op from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op -from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op +from mypyc.primitives.list_ops import ( + list_append_op, + list_get_item_int64_op, + list_get_item_unsafe_op, + new_list_set_item_op, +) from mypyc.primitives.misc_ops import stop_async_iteration_op from mypyc.primitives.registry import CFunctionDescription from mypyc.primitives.set_ops import set_add_op @@ -287,7 +293,8 @@ def sequence_from_generator_preallocate_helper( target_op = empty_op_llbuilder(length, line) def set_item(item_index: Value) -> None: - e = builder.accept(gen.left_expr) + with builder.enter_borrow_scope(line): + e = builder.accept(gen.left_expr) set_item_op(target_op, item_index, e, line) for_loop_helper_with_index( @@ -440,9 +447,10 @@ def loop_contents( remaining_loop_params: the parameters for any further nested loops; if it's empty we'll instead evaluate the "gen_inner_stmts" function """ - # Check conditions, in order, short circuiting them. + # Check conditions, in order, short-circuiting them. for cond in conds: - cond_val = builder.accept(cond) + with builder.enter_borrow_scope(line): + cond_val = builder.accept(cond) cont_block, rest_block = BasicBlock(), BasicBlock() # If the condition is true we'll skip the continue. builder.add_bool_branch(cond_val, rest_block, cont_block) @@ -456,7 +464,8 @@ def loop_contents( else: # We finally reached the actual body of the generator. # Generate the IR for the inner loop body. - gen_inner_stmts() + with builder.enter_borrow_scope(line): + gen_inner_stmts() handle_loop(loop_params) @@ -866,7 +875,10 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> # since we want to use __getitem__ if we don't have an unsafe version, # so we just check manually. if is_list_rprimitive(target.type): - return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) + if not IS_FREE_THREADED: + return builder.primitive_op(list_get_item_unsafe_op, [target, index], line) + else: + return builder.primitive_op(list_get_item_int64_op, [target, index], line) elif is_tuple_rprimitive(target.type): return builder.call_c(tuple_get_item_unsafe_op, [target, index], line) elif is_str_rprimitive(target.type): diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 82fc74f9b1ce5..555baaada0e81 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -23,6 +23,7 @@ Call, Goto, Integer, + LoadErrorValue, MethodCall, RaiseStandardError, Register, @@ -49,7 +50,7 @@ load_outer_envs, setup_func_for_recursive_call, ) -from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl +from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl, gen_generator_func_cleanup from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.primitives.exc_ops import ( error_catch_op, @@ -107,6 +108,8 @@ class that implements the function (each function gets a separate class). setup_func_for_recursive_call( builder, fitem, builder.fn_info.generator_class, prefix=GENERATOR_ATTRIBUTE_PREFIX ) + cleanup_on_error = BasicBlock() + builder.builder.push_error_handler(cleanup_on_error) create_switch_for_generator_class(builder) add_raise_exception_blocks_to_generator_class(builder, fitem.line) @@ -114,6 +117,11 @@ class that implements the function (each function gets a separate class). builder.accept(fitem.body) builder.maybe_add_implicit_return() + builder.builder.pop_error_handler() + + builder.activate_block(cleanup_on_error) + gen_generator_func_cleanup(builder, fitem.line) + builder.add(Return(builder.add(LoadErrorValue(object_rprimitive)))) populate_switch_for_generator_class(builder) @@ -166,6 +174,12 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: builder.fn_info.env_class = generator_class_ir else: generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) + if not builder.fn_info.fitem.is_coroutine: + # After completion generators still need generator.__mypyc_env__ for subsequent + # __next__() calls to observe the terminal next-label and raise StopIteration. + # Coroutines can't be resumed after completion, so keeping the environment alive + # there would just extend local lifetimes unnecessarily. + generator_class_ir.attrs_to_keep_alive_on_completion.add(ENV_ATTR_NAME) builder.classes.append(generator_class_ir) return generator_class_ir diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c19eded77464e..aafdb73fed84e 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -7,7 +7,8 @@ from __future__ import annotations import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from typing import Final, TypeGuard, cast from mypy.argmap import map_actuals_to_formals @@ -19,6 +20,7 @@ FAST_ISINSTANCE_MAX_SUBCLASSES, FAST_PREFIX, IS_FREE_THREADED, + KEEP_ALIVE_SHORT_LIVED, MAX_LITERAL_SHORT_INT, MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, @@ -206,6 +208,7 @@ new_tuple_with_length_op, sequence_tuple_op, ) +from mypyc.rt_expandtype import expand_rtype from mypyc.rt_subtype import is_runtime_subtype from mypyc.sametype import is_same_type from mypyc.subtype import is_subtype @@ -249,6 +252,15 @@ BOOL_BINARY_OPS: Final = {"&", "&=", "|", "|=", "^", "^=", "==", "!=", "<", "<=", ">", ">="} +class PendingKeepAlive: + __slots__ = ("value", "scope", "seq") + + def __init__(self, value: Value, scope: int, seq: int) -> None: + self.value = value + self.scope = scope + self.seq = seq + + class LowLevelIRBuilder: """A "low-level" IR builder class. @@ -275,7 +287,10 @@ def __init__(self, errors: Errors | None, options: CompilerOptions) -> None: self.error_handlers: list[BasicBlock | None] = [None] # Values that we need to keep alive as long as we have borrowed # temporaries. Use flush_keep_alives() to mark the end of the live range. - self.keep_alives: list[Value] = [] + # The second value is the scope/duration of keep alive (KEEP_ALIVE_* constant). + # Different values must be kept alive for different durations. + self.keep_alives: list[PendingKeepAlive] = [] + self.next_keep_alive_seq = 0 def set_module(self, module_name: str, module_path: str) -> None: """Set the name and path of the current module.""" @@ -366,10 +381,55 @@ def self(self) -> Register: """ return self.args[0] - def flush_keep_alives(self, line: int) -> None: - if self.keep_alives: - self.add(KeepAlive(self.keep_alives.copy(), line)) - self.keep_alives = [] + def flush_keep_alives(self, line: int, *, scope: int = KEEP_ALIVE_SHORT_LIVED) -> None: + if any(entry.scope == scope for entry in self.keep_alives): + self.add( + KeepAlive( + [entry.value for entry in self.keep_alives if entry.scope == scope], line + ) + ) + self.keep_alives = [entry for entry in self.keep_alives if entry.scope != scope] + + def keep_alive_checkpoint(self) -> int: + return self.next_keep_alive_seq + + def flush_keep_alives_since(self, line: int, checkpoint: int) -> None: + """Flush keep-alives added after 'checkpoint' without touching earlier ones.""" + new_keep_alives = [entry for entry in self.keep_alives if entry.seq >= checkpoint] + if new_keep_alives: + self.add(KeepAlive([entry.value for entry in new_keep_alives], line)) + self.keep_alives = [entry for entry in self.keep_alives if entry.seq < checkpoint] + + @contextmanager + def borrow_region(self, line: int) -> Iterator[None]: + """Confine borrows created in this region: flush them when leaving it. + + Keep-alives added inside the region (regardless of their duration scope) + are flushed on exit, so borrows can't leak past a lexical boundary such as + a conditional branch, an 'and'/'or' branch or a comprehension iteration. + """ + checkpoint = self.keep_alive_checkpoint() + yield + self.flush_keep_alives_since(line, checkpoint) + + def add_keep_alive(self, value: Value, scope: int) -> None: + self.keep_alives.append(PendingKeepAlive(value, scope, self.next_keep_alive_seq)) + self.next_keep_alive_seq += 1 + + def keep_borrow_source_alive(self, value: Value, borrow_scope: int) -> None: + """Ensure the source of borrowed value will be alive for given borrow scope. + + A borrow keeps reading from 'value', so 'value' must stay valid for the + borrow's whole scope. When 'value' is itself a borrowed cast it holds no + reference of its own (and coerce() only kept its source alive short-term), + so we follow the cast to the underlying owned value and keep *that* alive + for the requested scope too. Otherwise, a longer-lived borrow through a cast + (e.g. a Final attribute read via cast(T, make())) could read freed memory. + """ + self.add_keep_alive(value, borrow_scope) + while isinstance(value, Cast) and value.is_borrowed: + value = value.src + self.add_keep_alive(value, borrow_scope) def debug_print(self, toprint: str | Value) -> None: if isinstance(toprint, str): @@ -399,7 +459,7 @@ def unbox_or_cast( return self.add(Unbox(src, target_type, line)) else: if can_borrow: - self.keep_alives.append(src) + self.add_keep_alive(src, KEEP_ALIVE_SHORT_LIVED) return self.add(Cast(src, target_type, line, borrow=can_borrow, unchecked=unchecked)) def coerce( @@ -804,19 +864,32 @@ def coerce_nullable(self, src: Value, target_type: RType, line: int) -> Value: # Attribute access def get_attr( - self, obj: Value, attr: str, result_type: RType, line: int, *, borrow: bool = False + self, + obj: Value, + attr: str, + result_type: RType, + line: int, + *, + borrow: bool = False, + borrow_scope: int = KEEP_ALIVE_SHORT_LIVED, ) -> Value: - """Get a native or Python attribute of an object.""" + """Get a native or Python attribute of an object. + + If the result is borrowed, borrow_scope controls how long it stays valid + (a KEEP_ALIVE_* constant). The caller is responsible for choosing a scope + that is actually safe (e.g. downgrading to short-lived if the borrow root + may be rebound during the borrow's live range). + """ if ( isinstance(obj.type, RInstance) and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr) ): - op = GetAttr(obj, attr, line, borrow=borrow) + op = GetAttr(obj, attr, line, borrow=borrow, borrow_scope=borrow_scope) # For non-refcounted attribute types, the borrow might be # disabled even if requested, so don't check 'borrow'. if op.is_borrowed: - self.keep_alives.append(obj) + self.keep_borrow_source_alive(obj, borrow_scope) return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) @@ -2111,18 +2184,20 @@ def shortcircuit_helper( # it is the right side if the left is false. true_body, false_body = (right_body, left_body) if op == "and" else (left_body, right_body) - left_value = left() - self.add_bool_branch(left_value, true_body, false_body) + with self.borrow_region(line): + left_value = left() + self.add_bool_branch(left_value, true_body, false_body) - self.activate_block(left_body) - left_coerced = self.coerce(left_value, expr_type, line) - self.add(Assign(target, left_coerced, line)) + self.activate_block(left_body) + left_coerced = self.coerce(left_value, expr_type, line) + self.add(Assign(target, left_coerced, line)) self.goto(next_block) self.activate_block(right_body) - right_value = right() - right_coerced = self.coerce(right_value, expr_type, line) - self.add(Assign(target, right_coerced, line)) + with self.borrow_region(line): + right_value = right() + right_coerced = self.coerce(right_value, expr_type, line) + self.add(Assign(target, right_coerced, line)) self.goto(next_block) self.activate_block(next_block) @@ -2280,7 +2355,7 @@ def call_c( # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE @@ -2334,6 +2409,7 @@ def primitive_op( args: list[Value], line: int, result_type: RType | None = None, + type_args: list[RType] | None = None, ) -> Value: """Add a primitive op.""" # Does this primitive map into calling a Python C API @@ -2363,10 +2439,20 @@ def primitive_op( # This primitive gets transformed in a lowering pass to # lower-level IR ops using a custom transform function. + # Evaluate argument and return types for generic primitives + return_type = None + if desc.type_params is not None: + assert type_args is not None, "Generic primitive op requires explicit type arguments" + assert len(type_args) == len(desc.type_params) + arg_types = [expand_rtype(arg_type, type_args) for arg_type in desc.arg_types] + return_type = expand_rtype(desc.return_type, type_args) + else: + arg_types = desc.arg_types + coerced = [] # Coerce fixed number arguments - for i in range(min(len(args), len(desc.arg_types))): - formal_type = desc.arg_types[i] + for i in range(min(len(args), len(arg_types))): + formal_type = arg_types[i] arg = args[i] assert formal_type is not None # TODO arg = self.coerce(arg, formal_type, line) @@ -2374,14 +2460,23 @@ def primitive_op( assert desc.ordering is None assert desc.var_arg_type is None assert not desc.extra_int_constants - target = self.add(PrimitiveOp(coerced, desc, line=line)) + target = self.add( + PrimitiveOp( + coerced, + desc, + line=line, + arg_types=arg_types, + return_type=return_type, + type_args=type_args, + ) + ) if desc.is_borrowed: # If the result is borrowed, force the arguments to be # kept alive afterwards, as otherwise the result might be # immediately freed, at the risk of a dangling pointer. for arg in coerced: if not isinstance(arg, (Integer, LoadLiteral)): - self.keep_alives.append(arg) + self.add_keep_alive(arg, KEEP_ALIVE_SHORT_LIVED) if desc.error_kind == ERR_NEG_INT: comp = ComparisonOp(target, Integer(0, desc.return_type, line), ComparisonOp.SGE, line) comp.error_kind = ERR_FALSE diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index e0a23769770d7..009a1d6a986bc 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -8,10 +8,13 @@ from abc import abstractmethod from typing import TYPE_CHECKING +from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import ( + ERR_NEVER, NO_TRACEBACK_LINE_NO, BasicBlock, Branch, + Call, Goto, Integer, Register, @@ -90,10 +93,7 @@ class GeneratorNonlocalControl(BaseNonlocalControl): """Default nonlocal control in a generator function outside statements.""" def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: - # Assign an invalid next label number so that the next time - # __next__ is called, we jump to the case in which - # StopIteration is raised. - builder.assign(builder.fn_info.generator_class.next_label_target, Integer(-1), line) + gen_generator_func_cleanup(builder, line) # Raise a StopIteration containing a field for the value that # should be returned. Before doing so, create a new block @@ -132,6 +132,25 @@ def gen_return(self, builder: IRBuilder, value: Value, line: int) -> None: builder.add(Return(Integer(0, object_rprimitive))) +def gen_class_clear_on_completion(builder: IRBuilder, obj: Value, cl: ClassIR, line: int) -> None: + call = Call(cl.clear_on_completion, [obj], line) + call.error_kind = ERR_NEVER + builder.add(call) + + +def gen_generator_func_cleanup(builder: IRBuilder, line: int) -> None: + """Clear references held by a completed generator or coroutine.""" + cls = builder.fn_info.generator_class + + # Assign an invalid next label number so that the next time __next__ is + # called, we jump to the case in which StopIteration is raised. + builder.assign(cls.next_label_target, Integer(-1), line) + + gen_class_clear_on_completion(builder, cls.curr_env_reg, builder.fn_info.env_class, line) + if builder.fn_info.env_class is not cls.ir: + gen_class_clear_on_completion(builder, cls.self_reg, cls.ir, line) + + class CleanupNonlocalControl(NonlocalControl): """Abstract nonlocal control that runs some cleanup code.""" diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 8b73b10bf8064..6e697823593a6 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -645,6 +645,8 @@ def prepare_methods_and_attributes( add_getter_declaration(ir, name, attr_rtype, module_name) add_setter_declaration(ir, name, attr_rtype, module_name) ir.attributes[name] = attr_rtype + if node.node.is_final: + ir.final_attributes.add(name) elif isinstance(node.node, (FuncDef, Decorator)): prepare_method_def(ir, module_name, cdef, mapper, node.node, options) elif isinstance(node.node, OverloadedFuncDef): diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 428650a810dca..21f47b190c302 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,7 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import TEMP_ATTR_NAME +from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -80,12 +80,19 @@ c_pyssize_t_rprimitive, exc_rtuple, is_tagged, + lock_rprimitive, none_rprimitive, object_pointer_rprimitive, object_rprimitive, ) from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional -from mypyc.irbuild.builder import IRBuilder, create_type_params, int_borrow_friendly_op +from mypyc.irbuild.builder import ( + IRBuilder, + create_type_params, + expr_has_suspend, + find_walrus_targets, + int_borrow_friendly_op, +) from mypyc.irbuild.for_helpers import for_loop_helper from mypyc.irbuild.generator import add_raise_exception_blocks_to_generator_class from mypyc.irbuild.nonlocalcontrol import ( @@ -115,6 +122,7 @@ restore_exc_info_op, ) from mypyc.primitives.generic_ops import iter_op, next_raw_op, py_delattr_op +from mypyc.primitives.librt_threading_ops import lock_acquire_op, lock_release_op from mypyc.primitives.misc_ops import ( check_stop_op, coro_op, @@ -159,10 +167,17 @@ def transform_expression_stmt(builder: IRBuilder, stmt: ExpressionStmt) -> None: if isinstance(stmt.expr, StrExpr): # Docstring. Ignore return - # ExpressionStmts do not need to be coerced like other Expressions, so we shouldn't - # call builder.accept here. + # ExpressionStmts do not need to be coerced like other Expressions, so + # we shouldn't call builder.accept here. + builder.expression_depth += 1 + builder.reassigned_in_expr = find_walrus_targets(stmt.expr) + builder.expr_has_suspend = expr_has_suspend(stmt.expr) stmt.expr.accept(builder.visitor) - builder.flush_keep_alives(stmt.line) + builder.expression_depth -= 1 + builder.reassigned_in_expr = set() + builder.expr_has_suspend = False + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_SHORT_LIVED) + builder.flush_keep_alives(stmt.line, scope=KEEP_ALIVE_WHOLE_EXPRESSION) def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: @@ -1108,6 +1123,10 @@ def transform_with( al = "a" if is_async else "" mgr_v = builder.accept(expr) + + if not is_async and mgr_v.type == lock_rprimitive: + transform_with_lock(builder, mgr_v, target, body, line) + return is_native = isinstance(mgr_v.type, RInstance) if is_native: value = builder.add(MethodCall(mgr_v, f"__{al}enter__", args=[], line=line)) @@ -1179,6 +1198,32 @@ def finally_body() -> None: ) +def transform_with_lock( + builder: IRBuilder, mgr_v: Value, target: Lvalue | None, body: GenFunc, line: int +) -> None: + """Optimized 'with' for librt.threading.Lock. + + Generate a simple try/finally with direct acquire/release calls. + Lock.__exit__ never suppresses exceptions, so we don't need the + full PEP 343 try/except/finally machinery. + """ + # __enter__: acquire the lock + value = builder.primitive_op(lock_acquire_op, [mgr_v], line) + + mgr = builder.maybe_spill(mgr_v) + + def try_body() -> None: + if target: + builder.assign(builder.get_assignment_target(target), value, line) + body() + + def finally_body() -> None: + # __exit__: release the lock (ignoring exception info) + builder.primitive_op(lock_release_op, [builder.read(mgr, line)], line) + + transform_try_finally_stmt(builder, try_body, finally_body, line) + + def transform_with_stmt(builder: IRBuilder, o: WithStmt) -> None: # Generate separate logic for each expr in it, left to right def generate(i: int) -> None: diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index bfcfabee45c21..8dffc12a22130 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final -from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE +from mypyc.common import IS_32_BIT_PLATFORM, KEEP_ALIVE_SHORT_LIVED, PLATFORM_SIZE from mypyc.ir.ops import ( ERR_MAGIC, Assign, @@ -52,6 +52,7 @@ vec_api_by_item_type, vec_item_type_tags, ) +from mypyc.primitives.librt_vecs_ops import vec_get_item_unsafe_borrow_op, vec_get_item_unsafe_op if TYPE_CHECKING: from mypyc.irbuild.ll_builder import LowLevelIRBuilder @@ -316,9 +317,10 @@ def vec_get_item( ) -> Value: """Generate inlined vec __getitem__ call. - We inline this, since it's simple but performance-critical. + We inline the length and bounds check, since they are simple but + performance-critical. The actual item load is emitted as a generic primitive + op that is lowered later. """ - # TODO: Support more item types # TODO: Support more index types len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) @@ -328,13 +330,28 @@ def vec_get_item( def vec_get_item_unsafe( builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False ) -> Value: - """Get vec item, assuming index is non-negative and within bounds.""" + """Get vec item, assuming index is non-negative and within bounds. + + This emits a generic primitive op that is inlined during lowering. + """ + assert isinstance(base.type, RVec) + if can_borrow: + desc = vec_get_item_unsafe_borrow_op + else: + desc = vec_get_item_unsafe_op + return builder.primitive_op(desc, [base, index], line, type_args=[base.type.item_type]) + + +def vec_get_item_unsafe_lower( + builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False +) -> Value: + """Generate the low-level IR for an unsafe vec item load.""" assert isinstance(base.type, RVec) index = as_platform_int(builder, index, line) vtype = base.type item_addr = vec_item_ptr(builder, base, index) result = vec_load_mem_item(builder, item_addr, vtype.item_type, can_borrow=can_borrow) - builder.keep_alives.append(base) + builder.add_keep_alive(base, KEEP_ALIVE_SHORT_LIVED) return result diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c22c4162669bd..458db90efd530 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -664,16 +664,68 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end); // List operations -PyObject *CPyList_Build(Py_ssize_t len, ...); +#ifndef Py_GIL_DISABLED + PyObject *CPyList_GetItem(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index); +PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); +bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); +bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); + +#else + +PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index); +bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value); + +static inline PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { + if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) { + // Inlined fast path + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return PyList_GetItemRef(list, n); + } else { + return CPyList_GetItem_(list, index); + } +} + +static inline PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + if (n < 0) { + n += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, n); +} + +static inline PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { + if (index < 0) { + index += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, index); +} + +static inline bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { + if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) { + // Inlined fast path + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + return PyList_SetItem(list, n, value) >= 0; + } else { + return CPyList_SetItem_(list, index, value); + } +} + +static inline bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { + if (index < 0) { + index += PyList_GET_SIZE(list); + } + return PyList_SetItem(list, index, value) >= 0; +} + +#endif + PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index); PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index); -PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index); PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index); -bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value); void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value); -bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value); +PyObject *CPyList_Build(Py_ssize_t len, ...); PyObject *CPyList_PopLast(PyObject *obj); PyObject *CPyList_Pop(PyObject *obj, CPyTagged index); CPyTagged CPyList_Count(PyObject *obj, PyObject *value); diff --git a/mypyc/lib-rt/function_wrapper.c b/mypyc/lib-rt/function_wrapper.c index 16b1b59303481..dfea6aed1d1c3 100644 --- a/mypyc/lib-rt/function_wrapper.c +++ b/mypyc/lib-rt/function_wrapper.c @@ -196,6 +196,7 @@ static PyObject* CPyFunction_Vectorcall(PyObject *func, PyObject *const *args, s } +// Steals ml, name, and code. Borrows module. static CPyFunction* CPyFunction_Init(CPyFunction *op, PyMethodDef *ml, PyObject* name, PyObject *module, PyObject* code, bool set_self) { PyCFunctionObject *cf = (PyCFunctionObject *)op; @@ -206,12 +207,10 @@ static CPyFunction* CPyFunction_Init(CPyFunction *op, PyMethodDef *ml, PyObject* Py_XINCREF(module); cf->m_module = module; - Py_INCREF(name); op->func_name = name; ((PyCMethodObject *)op)->mm_class = NULL; - Py_XINCREF(code); op->func_code = code; CPyFunction_func_vectorcall(op) = CPyFunction_Vectorcall; @@ -243,7 +242,7 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu PyCFunction func, int func_flags, const char *func_doc, int first_line, int code_flags, bool has_self_arg) { PyMethodDef *method = NULL; - PyObject *code = NULL, *op = NULL; + PyObject *code = NULL, *name = NULL, *op = NULL; bool set_self = false; #ifdef Py_GIL_DISABLED @@ -280,18 +279,24 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu if (unlikely(!code)) { goto err; } + name = PyUnicode_FromString(funcname); + if (unlikely(!name)) { + goto err; + } // Set m_self inside the function wrapper only if the wrapped function has no self arg // to pass m_self as the self arg when the function is called. // When the function has a self arg, it will come in the args vector passed to the // vectorcall handler. set_self = !has_self_arg; - op = (PyObject *)CPyFunction_Init(PyObject_GC_New(CPyFunction, CPyFunctionType), - method, PyUnicode_FromString(funcname), module, - code, set_self); - if (unlikely(!op)) { + CPyFunction *raw = PyObject_GC_New(CPyFunction, CPyFunctionType); + if (unlikely(!raw)) { goto err; } + op = (PyObject *)CPyFunction_Init(raw, method, name, module, code, set_self); + method = NULL; + name = NULL; + code = NULL; PyObject_GC_Track(op); return op; @@ -300,5 +305,7 @@ PyObject* CPyFunction_New(PyObject *module, const char *filename, const char *fu if (method) { PyMem_Free(method); } + Py_XDECREF(name); + Py_XDECREF(code); return NULL; } diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index 8b7eaa1acf3f0..e8fd061ada687 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -49,6 +49,8 @@ PyObject *CPyList_Copy(PyObject *list) { return PyObject_CallMethodNoArgs(list, mypyc_interned_str.copy); } +#ifndef Py_GIL_DISABLED + PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); Py_ssize_t size = PyList_GET_SIZE(list); @@ -69,24 +71,6 @@ PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) { return result; } -PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - Py_ssize_t size = PyList_GET_SIZE(list); - if (n >= 0) { - if (n >= size) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } else { - n += size; - if (n < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } - return PyList_GET_ITEM(list, n); -} - PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); @@ -112,29 +96,6 @@ PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) { } } -PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { - if (CPyTagged_CheckShort(index)) { - Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); - Py_ssize_t size = PyList_GET_SIZE(list); - if (n >= 0) { - if (n >= size) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } else { - n += size; - if (n < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - } - return PyList_GET_ITEM(list, n); - } else { - PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); - return NULL; - } -} - PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { size_t size = PyList_GET_SIZE(list); if (likely((uint64_t)index < size)) { @@ -156,23 +117,6 @@ PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) { return result; } -PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { - size_t size = PyList_GET_SIZE(list); - if (likely((uint64_t)index < size)) { - return PyList_GET_ITEM(list, index); - } - if (index >= 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - index += size; - if (index < 0) { - PyErr_SetString(PyExc_IndexError, "list index out of range"); - return NULL; - } - return PyList_GET_ITEM(list, index); -} - bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) { if (CPyTagged_CheckShort(index)) { Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); @@ -220,6 +164,95 @@ bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) { return true; } +#else /* Py_GIL_DISABLED */ + +PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + if (n < 0) { + n += PyList_GET_SIZE(list); + } + return PyList_GetItemRef(list, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n < 0) { + n += size; + } + return PyList_SetItem(list, n, value) >= 0; + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return false; + } +} + +#endif + +PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); +} + +PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) { + if (CPyTagged_CheckShort(index)) { + Py_ssize_t n = CPyTagged_ShortAsSsize_t(index); + Py_ssize_t size = PyList_GET_SIZE(list); + if (n >= 0) { + if (n >= size) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } else { + n += size; + if (n < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + } + return PyList_GET_ITEM(list, n); + } else { + PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG); + return NULL; + } +} + +PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) { + size_t size = PyList_GET_SIZE(list); + if (likely((uint64_t)index < size)) { + return PyList_GET_ITEM(list, index); + } + if (index >= 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + index += size; + if (index < 0) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GET_ITEM(list, index); +} + // This function should only be used to fill in brand new lists. void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value) { PyList_SET_ITEM(list, index, value); diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 4a4552d059b8c..d7a3eb3214bed 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -156,6 +156,10 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) { return x << 1; } +static inline int CPyTagged_IsNegative(CPyTagged x) { + return ((Py_ssize_t)x) < 0; +} + // Are we targeting Python 3.X or newer? #define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000) #define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000) diff --git a/mypyc/lib-rt/pythonsupport.c b/mypyc/lib-rt/pythonsupport.c index 0a99f0ae2e29b..a8f5a4f4ad4ea 100644 --- a/mypyc/lib-rt/pythonsupport.c +++ b/mypyc/lib-rt/pythonsupport.c @@ -5,6 +5,27 @@ #include "pythonsupport.h" +#ifdef Py_GIL_DISABLED +// Cold slow path of CPy_GetAttrRef (declared in pythonsupport.h). Reached only +// when the inline fast-path try-incref fails: the value is owned by another +// thread, so taking a reference requires an atomic shared-refcount operation. +// Kept out-of-line so the inline fast path stays small. +// +// First try the lock-free shared-refcount CAS. If the value has not had +// maybe-weakref set yet (for example, it was published by CPy_InitAttrRef), force +// a cross-thread reference via _Py_NewRefWithLock, which cannot fail and sets +// maybe-weakref so subsequent reads take the fast path. The value was already +// observed in the field by CPy_GetAttrRef; CPy_SetAttrRef's QSBR-delayed decref +// keeps any replaced value alive long enough for this reader. +CPy_NOINLINE +PyObject *CPy_GetAttrRefSlow(PyObject *v) { + if (_Py_TryIncRefShared(v)) { + return v; + } + return _Py_NewRefWithLock(v); // sets maybe-weakref; cannot fail +} +#endif + ///////////////////////////////////////// // Adapted from bltinmodule.c in Python 3.7.0 PyObject* diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 1b0583543fe48..33c5a596d1747 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -23,6 +23,10 @@ #include "internal/pycore_setobject.h" // _PySet_Update #endif +#ifdef Py_GIL_DISABLED +#include "internal/pycore_object.h" // _Py_TryIncrefFast, _Py_TryIncRefShared +#endif + #if CPY_3_12_FEATURES #include "internal/pycore_frame.h" #endif @@ -34,6 +38,136 @@ extern "C" { } // why isn't emacs smart enough to not indent this #endif +#ifdef Py_GIL_DISABLED +// Read a native attribute that is a single reference-counted 'PyObject *' field, +// returning a new reference (or NULL if the field is NULL/undefined). +// +// On free-threaded builds a plain load followed by an incref races with a +// concurrent setter that may decref the old value to zero and free it before the +// incref runs (use-after-free). CPy_SetAttrRef avoids that by reclaiming old +// values through QSBR-delayed decref, so a value observed in the field remains +// safe to touch while this reader is running. +// +// Only the hot case is inlined here: an incref of a value owned by this thread or +// immortal, via '_Py_TryIncrefFast' (no CAS, no loop). Everything colder -- the +// cross-thread shared-refcount CAS and the _Py_NewRefWithLock fallback -- lives +// out-of-line in 'CPy_GetAttrRefSlow'. Splitting it this way keeps each call +// site's fast path small enough to inline, which is measurably faster than +// letting the compiler auto-out-line the whole helper (that merges every read +// site's branch history into one shared copy and mispredicts). It is only used in +// free-threaded builds; the default (GIL) build keeps the plain load + incref +// generated inline by mypyc. +PyObject *CPy_GetAttrRefSlow(PyObject *v); + +static inline PyObject *CPy_GetAttrRef(PyObject **field) { + PyObject *v = (PyObject *)_Py_atomic_load_ptr_acquire(field); + if (v == NULL) { + return NULL; + } + if (_Py_TryIncrefFast(v)) { + return v; + } + return CPy_GetAttrRefSlow(v); +} + +// Read a native attribute that is a single reference-counted 'PyObject *' field +// AND is Final (assigned once during construction, never rebound -- mypyc emits no +// setter for it), returning a new reference (or NULL if undefined). +// +// A Final attribute has no concurrent writer after 'self' is published, so the +// use-after-free race that CPy_GetAttrRef guards against cannot happen: the field +// holds a strong reference for the object's whole lifetime, and any thread reading +// it necessarily holds 'self', which keeps the value alive. So the try-incref + +// _Py_NewRefWithLock fallback are unnecessary here -- a plain load + Py_INCREF is +// safe. A cross-thread Py_INCREF is an unconditional +// atomic add on ob_ref_shared, so (unlike CPy_GetAttrRef's try-incref) it needs no +// maybe-weakref and has no slow path. The load is relaxed rather than acquire: the +// reader reached 'self' through a synchronization edge (self's own publication) +// that already ordered the construction stores before it, exactly as with +// CPy_InitAttrRef's relaxed store. Relaxed keeps it TSan-clean at zero cost (plain +// mov/ldr). +static inline PyObject *CPy_GetAttrRefFinal(PyObject **field) { + PyObject *v = (PyObject *)_Py_atomic_load_ptr_relaxed(field); + if (v != NULL) { + Py_INCREF(v); + } + return v; +} + +// Reclaim the previous value of a native attribute after it has been replaced. +// +// CPy_GetAttrRef reads the field optimistically without holding any lock the +// writer also takes, so a reader can load the old pointer and then try to take a +// reference after this store. The old value must therefore stay alive until every +// thread has passed a quiescent point, which is exactly what a QSBR-deferred +// decref guarantees. So all mortal old values are +// reclaimed via _PyObject_XDecRefDelayed, matching CPython's own replace-a-slot +// paths (e.g. _PyObject_SetDict / _PyObject_SetManagedDict). +// +// We deliberately do NOT take a "local refcount > 1, owned by this thread" fast +// path: dropping the field's reference is not the only decref of the object, so a +// non-freeing local decrement here does not prevent an unrelated reference holder +// from driving the object to zero (a plain, non-deferred Py_DECREF -> _Py_Dealloc) +// while an in-flight reader still holds the stale pointer -- a use-after-free that +// only QSBR deferral closes. Immortal objects are never freed, so skipping their +// decref entirely is safe and avoids queuing a no-op onto the delayed-free list. +static inline void CPy_DecRefAttrOld(PyObject *op) { + if (op == NULL) { + return; + } + if (_Py_IsImmortal(op)) { + return; + } + _PyObject_XDecRefDelayed(op); +} + +// Set a native attribute that is a single reference-counted 'PyObject *' field, +// stealing the reference to 'value' (which may be NULL to delete the attribute) +// and safely reclaiming the previous value. +// +// Memory safety does NOT depend on SetMaybeWeakref here, so (unlike an earlier +// version) we do not call it. Two things keep this safe: +// - The old value is reclaimed via CPy_DecRefAttrOld, a QSBR-deferred decref, so +// it cannot be freed while an in-flight CPy_GetAttrRef still holds the stale +// pointer. +// - A concurrent cross-thread reader whose inline fast-path try-incref fails on +// an unflagged 'value' still cannot fail and never blocks on this writer: +// CPy_GetAttrRef falls into CPy_GetAttrRefSlow, which is fully lock-free. It +// retries with a lock-free shared-refcount CAS (_Py_TryIncRefShared) and, only +// if that also fails, forces a reference via _Py_NewRefWithLock, which cannot +// fail and lazily sets maybe-weakref so later cross-thread reads take the CAS. +// This mirrors CPy_InitAttrRef, which already omits SetMaybeWeakref for the same +// reason. Setting the flag here would only be a possible performance tuning knob +// (it would let that first cross-thread reader succeed on the cheaper +// _Py_TryIncRefShared CAS instead of falling through to _Py_NewRefWithLock); it is +// not needed for correctness. The atomic exchange publishes the new pointer and +// hands back the old one without a writer/writer race. +static inline void CPy_SetAttrRef(PyObject **field, PyObject *value) { + PyObject *old = (PyObject *)_Py_atomic_exchange_ptr(field, value); + CPy_DecRefAttrOld(old); +} + +// Initialize a native attribute that is known to be previously undefined (NULL), +// stealing the reference to 'value'. +// +// Initializer stores only happen while 'self' is still thread-local (the +// attribute-definedness analysis marks a SetAttr as an initializer only before +// 'self' can leak -- see mypyc/analysis/attrdefined.py). So there is no old value +// to reclaim, no competing writer, and the field store is not itself the +// publication point: 'self' is published later (when it escapes __init__ or is +// returned), and that publication carries the release barrier making all the +// construction stores visible. A relaxed store therefore suffices. +// +// Unlike CPy_SetAttrRef, this deliberately does NOT call SetMaybeWeakref (its CAS +// is pure overhead here, ~+2.6ns per fresh store, and construction-heavy code +// pays it on every attribute of every new object). The cost is moved off this hot +// path onto CPy_GetAttrRef's cold slow path, which sets maybe-weakref lazily on +// the first cross-thread read that needs it. +static inline void CPy_InitAttrRef(PyObject **field, PyObject *value) { + _Py_atomic_store_ptr_relaxed(field, value); +} +#endif + PyObject* update_bases(PyObject *bases); int init_subclass(PyTypeObject *type, PyObject *kwds); @@ -205,6 +339,25 @@ list_count(PyListObject *self, PyObject *value) Py_ssize_t count = 0; Py_ssize_t i; +#ifdef Py_GIL_DISABLED + for (i = 0; i < PyList_GET_SIZE(self); i++) { + PyObject *item = PyList_GetItemRef((PyObject *)self, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + if (PyErr_ExceptionMatches(PyExc_IndexError)) { + PyErr_Clear(); + break; + } + return CPY_INT_TAG; + } + int cmp = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (cmp > 0) + count++; + else if (cmp < 0) + return CPY_INT_TAG; + } +#else for (i = 0; i < Py_SIZE(self); i++) { int cmp = PyObject_RichCompareBool(self->ob_item[i], value, Py_EQ); if (cmp > 0) @@ -212,6 +365,7 @@ list_count(PyListObject *self, PyObject *value) else if (cmp < 0) return CPY_INT_TAG; } +#endif return CPyTagged_ShortFromSsize_t(count); } diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 371b322ca18b2..15d31a443ff86 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -164,5 +164,11 @@ def run(self) -> None: include_dirs=["."], extra_compile_args=cflags, ), + Extension( + "librt.threading", + ["threading/librt_threading.c"], + include_dirs=[".", "threading"], + extra_compile_args=cflags, + ), ] ) diff --git a/mypyc/lib-rt/strings/librt_strings.h b/mypyc/lib-rt/strings/librt_strings.h index 6c1942667ba44..33af57dc2a2a8 100644 --- a/mypyc/lib-rt/strings/librt_strings.h +++ b/mypyc/lib-rt/strings/librt_strings.h @@ -31,8 +31,9 @@ typedef struct { } StringWriterObject; // Codepoint classification helpers. Inputs are signed i32 for compatibility -// with mypyc's int32_rprimitive; negative values are non-codepoints and -// return false. Defined `static inline` so they compile statically into +// with mypyc's int32_rprimitive; out-of-range values (negative, or past the +// maximum Unicode code point 0x10FFFF) are non-codepoints and return false. +// Defined `static inline` so they compile statically into // both the librt.strings module and any mypyc-compiled extension that // includes this header, avoiding the capsule indirection that would dwarf // the work of a single Py_UNICODE_IS* macro call. @@ -58,12 +59,14 @@ static inline bool LibRTStrings_IsAlpha(int32_t c) { // PyUnicode_IsIdentifier on a 1-character string. Aborts via // CPyError_OutOfMemory on allocation failure to keep this ERR_NEVER. static inline bool LibRTStrings_IsIdentifier(int32_t c) { - if (c < 0) return false; - if (c < 128) { + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; } + // Reject negatives and code points past the Unicode maximum. + if ((uint32_t)c > 0x10FFFF) return false; PyObject *s = PyUnicode_FromOrdinal((int)c); if (s == NULL) { CPyError_OutOfMemory(); @@ -101,9 +104,13 @@ static inline int32_t LibRTStrings_ChangeCase_slow(int32_t c, const char *method // non-ASCII delegates to str.upper on a 1-character string. Returns the // input unchanged when uppercasing expands to multiple codepoints. static inline int32_t LibRTStrings_ToUpper(int32_t c) { - if (c < 0) return c; - if (c >= 'a' && c <= 'z') return c - 32; - if (c < 128) return c; + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { + if (c >= 'a' && c <= 'z') return c - 32; + return c; + } + // Negatives and code points past the Unicode maximum are returned unchanged. + if ((uint32_t)c > 0x10FFFF) return c; return LibRTStrings_ChangeCase_slow(c, "upper"); } @@ -111,9 +118,13 @@ static inline int32_t LibRTStrings_ToUpper(int32_t c) { // non-ASCII delegates to str.lower on a 1-character string. Returns the // input unchanged when lowercasing expands to multiple codepoints. static inline int32_t LibRTStrings_ToLower(int32_t c) { - if (c < 0) return c; - if (c >= 'A' && c <= 'Z') return c + 32; - if (c < 128) return c; + // Unsigned compare: negatives wrap to large values and skip the fast path. + if ((uint32_t)c < 128) { + if (c >= 'A' && c <= 'Z') return c + 32; + return c; + } + // Negatives and code points past the Unicode maximum are returned unchanged. + if ((uint32_t)c > 0x10FFFF) return c; return LibRTStrings_ChangeCase_slow(c, "lower"); } diff --git a/mypyc/lib-rt/threading/librt_threading.c b/mypyc/lib-rt/threading/librt_threading.c new file mode 100644 index 0000000000000..84d7346a75872 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading.c @@ -0,0 +1,714 @@ +#include "pythoncapi_compat.h" + +#define PY_SSIZE_T_CLEAN +#include +#include "librt_threading.h" +#include "mypyc_util.h" + +#if !defined(_WIN32) +#include +#include +#endif + +#if CPY_3_14_FEATURES + +// Python 3.14+ (all platforms, with or without the GIL): Use PyMutex (1-byte +// atomic lock with parking lot). PyMutex gives better interruptibility than the +// pthread fallback, and its fast release path is competitive with sem_t. +// PyMutex_LockFast, _PyMutex_LockTimed, and _PY_LOCK_DETACH are internal +// CPython APIs that might change across minor releases. +#define LOCK_BACKEND_PYMUTEX +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "internal/pycore_lock.h" + +#elif defined(_WIN32) + +// Python <3.14 on Windows: Use Slim Reader/Writer Lock +#define LOCK_BACKEND_SRWLOCK +#include + +#else + +// Python <3.14 on POSIX. +// +// Prefer a POSIX unnamed semaphore when the platform supports it well and the +// GIL is enabled, and fall back to a pthread mutex + condition variable +// otherwise. We use the same test CPython uses to pick its semaphore-based lock +// (see Python/thread_pthread.h): an unnamed semaphore is only usable when +// sem_init() actually works AND a timed wait is available. Notably this is true +// on Linux but false on macOS (whose sem_init() is a non-functional stub), so +// macOS uses the mutex+condvar fallback. Free-threaded builds also use the +// mutex+condvar fallback because the semaphore backend's `locked` bookkeeping +// relies on GIL serialization. +#if !defined(Py_GIL_DISABLED) && \ + defined(_POSIX_SEMAPHORES) && (_POSIX_SEMAPHORES + 0) != -1 && \ + (defined(HAVE_SEM_TIMEDWAIT) || defined(HAVE_SEM_CLOCKWAIT)) +#define LOCK_BACKEND_SEM +#include +#else +#define LOCK_BACKEND_PTHREAD +#include +#endif + +#endif + +// +// Lock +// +// A fast mutex lock for use from mypyc-compiled code. +// +// On Python 3.14+ (all platforms), this uses CPython's PyMutex, a 1-byte atomic +// lock backed by a parking lot for contended waits. PyMutex automatically +// releases the GIL when blocking. +// +// On Python 3.13 and earlier with Windows, this uses an SRWLOCK (Slim +// Reader/Writer Lock) plus a CONDITION_VARIABLE guarding a `locked` flag. The SRWLOCK only +// protects the flag and is never held across the user's critical section, so +// release() may be called from a thread other than the acquirer (matching +// threading.Lock semantics). This mirrors CPython's Windows lock (NRMUTEX in +// Python/thread_nt.h) and is the Windows twin of the POSIX pthread+condvar +// backend below. +// +// On Python 3.13 and earlier with POSIX systems, there are two backends, both +// of which allow release() from a thread other than the one that acquired the +// lock (matching threading.Lock semantics): +// +// - Where unnamed POSIX semaphores work well (e.g. Linux) and the GIL is +// enabled, this uses a sem_t initialized to 1: acquire is sem_wait, release +// is sem_post. Semaphores have no ownership concept, so cross-thread release +// is directly well-defined. +// +// - Otherwise (e.g. macOS, whose sem_init() is a non-functional stub, and +// free-threaded builds), this uses a pthread mutex + condition variable +// guarding a `locked` flag. The mutex only protects the flag and is never +// held across the user's critical section, so the OS mutex is always +// unlocked on the same thread that locked it. +// + +// ---------- Platform-specific lock state ---------- + +#if defined(LOCK_BACKEND_PYMUTEX) + +typedef struct { + PyObject_HEAD + PyMutex mutex; +} LockObject; + +#elif defined(LOCK_BACKEND_SRWLOCK) + +typedef struct { + PyObject_HEAD + // The SRWLOCK below does NOT represent the Python lock; like the pthread + // fallback's `mut`, it only guards `locked` and the condition variable, + // and is held just long enough to inspect/flip the flag -- never across + // the user's critical section. That is what allows release() from a + // thread other than the acquirer: SRWLOCK's same-thread-release rule is + // never violated, and clearing `locked` is just a guarded store. This + // mirrors CPython's Windows lock (NRMUTEX in Python/thread_nt.h). + SRWLOCK srw; + CONDITION_VARIABLE lock_released; + int locked; // 0=unlocked, 1=locked; protected by `srw` +} LockObject; + +#elif defined(LOCK_BACKEND_SEM) + +typedef struct { + PyObject_HEAD + sem_t sem; // counting semaphore, initialized to 1 + // Tracks the locked state for locked() and release-unlocked detection. + // The semaphore itself is the source of truth for mutual exclusion; this + // flag is advisory bookkeeping. It is set after a successful acquire and + // cleared before sem_post in release. + // + // This backend is only selected when the GIL is enabled, so this flag is a + // plain int relying on the GIL for serialization: it is only ever touched + // with the GIL held (the blocking sem_wait drops the GIL, but the flag + // store happens after the GIL is reacquired). This mirrors the old + // PyThread_type_lock wrapper bookkeeping used by CPython 3.12 and earlier, + // where _thread lock kept a plain `char locked` for sanity checks and + // locked(). + int locked; +} LockObject; + +#else // pthread mutex + condvar fallback + +typedef struct { + PyObject_HEAD + // The pthread mutex below does NOT represent the Python lock; it only + // guards the `locked` flag and the condition variable. The Python lock + // state is `locked` itself. This indirection (matching CPython's POSIX + // lock) is what allows release() from a thread other than the acquirer: + // `mut` is always locked and unlocked within a single call on a single + // thread, so pthread's same-thread-unlock rule is never violated. + pthread_mutex_t mut; + pthread_cond_t lock_released; + // Always accessed while holding `mut` (including the locked() reader), + // so a plain int is sufficient -- the mutex provides the ordering. + // Matches CPython's pthread_lock.locked (a plain char). + int locked; // 0=unlocked, 1=locked; protected by `mut` +} LockObject; + +#endif + +// ---------- Platform-specific init/acquire/release ---------- + +static inline int +Lock_init_internal(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + self->mutex = (PyMutex){0}; +#elif defined(LOCK_BACKEND_SRWLOCK) + InitializeSRWLock(&self->srw); + InitializeConditionVariable(&self->lock_released); + self->locked = 0; +#elif defined(LOCK_BACKEND_SEM) + if (sem_init(&self->sem, 0, 1) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + self->locked = 0; +#else + int status = pthread_mutex_init(&self->mut, NULL); + if (status != 0) { + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + status = pthread_cond_init(&self->lock_released, NULL); + if (status != 0) { + pthread_mutex_destroy(&self->mut); + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + self->locked = 0; +#endif + return 0; +} + +// Try to acquire the lock. Returns 1 (true) on success, 0 (false) if +// non-blocking and the lock is held, or -1 if interrupted by an error-raising +// signal handler. +static int +Lock_acquire_impl(LockObject *self, int blocking) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + if (!blocking) { + PyLockStatus r = _PyMutex_LockTimed(&self->mutex, 0, _Py_LOCK_DONT_DETACH); + return r == PY_LOCK_ACQUIRED; + } + if (PyMutex_LockFast(&self->mutex)) { + return 1; + } + PyLockStatus r = _PyMutex_LockTimed(&self->mutex, -1, + _PY_LOCK_DETACH | _PY_LOCK_HANDLE_SIGNALS); + if (r == PY_LOCK_INTR) { + return -1; + } + return 1; + +#elif defined(LOCK_BACKEND_SRWLOCK) + // `srw` only guards `locked` and the condition variable; it is held just + // long enough to inspect/flip the flag, never across the user's critical + // section. This is what lets a different thread call release(). + // + // Fast path: grab the lock without releasing the GIL if it is free. This is + // also the whole story in the non-blocking case. + AcquireSRWLockExclusive(&self->srw); + if (!self->locked) { + self->locked = 1; + ReleaseSRWLockExclusive(&self->srw); + return 1; + } + ReleaseSRWLockExclusive(&self->srw); + if (!blocking) { + return 0; + } + + // Slow path: wait for the lock to be released, with the GIL dropped so + // other Python threads can run (and release the lock). + // SleepConditionVariableSRW atomically releases the SRWLOCK while sleeping + // and reacquires it on wake, exactly like pthread_cond_wait. + Py_BEGIN_ALLOW_THREADS + AcquireSRWLockExclusive(&self->srw); + while (self->locked) { + SleepConditionVariableSRW(&self->lock_released, &self->srw, INFINITE, 0); + } + self->locked = 1; + ReleaseSRWLockExclusive(&self->srw); + Py_END_ALLOW_THREADS + return 1; + +#elif defined(LOCK_BACKEND_SEM) + // A semaphore has no ownership: any thread may sem_post a token that + // another thread consumed via sem_wait, so cross-thread release is + // directly well-defined. `locked` is advisory bookkeeping for locked() + // and for guarding against releasing an unheld lock. + // + // Fast path: try a non-blocking acquire first to avoid GIL release/reacquire + // overhead in the common uncontended case. This is also the whole story in + // the non-blocking case. + { + int status; + do { + status = sem_trywait(&self->sem); + } while (status == -1 && errno == EINTR); + if (status == 0) { + self->locked = 1; + return 1; + } + } + if (!blocking) { + return 0; // EAGAIN: already held + } + + // Slow path: block with the GIL dropped so other Python threads can run + // (and release the lock). If a signal interrupts sem_wait(), run pending + // Python signal handlers with the GIL held; retry unless a handler raises. + for (;;) { + int status; + int err = 0; + + Py_BEGIN_ALLOW_THREADS + status = sem_wait(&self->sem); + if (status == -1) { + err = errno; + } + Py_END_ALLOW_THREADS + + if (status == 0) { + self->locked = 1; + return 1; + } + + if (err != EINTR) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + +#else // pthread mutex + condvar fallback + // `mut` only guards `locked` and the condition variable; it is held just + // long enough to inspect/flip the flag, never across the user's critical + // section. This is what lets a different thread call release(). + // + // Fast path: grab the lock without releasing the GIL if it is free. This is + // also the whole story in the non-blocking case. + pthread_mutex_lock(&self->mut); + if (!self->locked) { + self->locked = 1; + pthread_mutex_unlock(&self->mut); + return 1; + } + pthread_mutex_unlock(&self->mut); + if (!blocking) { + return 0; + } + + // Slow path: wait for the lock to be released, with the GIL dropped so + // other Python threads can run (and release the lock). If we wake but do + // not get the lock, give pending Python signal handlers a chance to run, + // matching CPython's pthread fallback. + for (;;) { + int acquired = 0; + int interrupted = 0; + int status; + + Py_BEGIN_ALLOW_THREADS + status = pthread_mutex_lock(&self->mut); + if (status == 0) { + while (self->locked) { + status = pthread_cond_wait(&self->lock_released, &self->mut); + if (status != 0) { + break; + } + if (self->locked) { + interrupted = 1; + break; + } + } + if (status == 0 && !interrupted) { + self->locked = 1; + acquired = 1; + } + int unlock_status = pthread_mutex_unlock(&self->mut); + if (status == 0) { + status = unlock_status; + } + } + Py_END_ALLOW_THREADS + + if (status != 0) { + errno = status; + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + if (acquired) { + return 1; + } + if (interrupted && Py_MakePendingCalls() < 0) { + return -1; + } + } +#endif +} + +// Release the lock. Returns 0 on success, -1 if the lock was not held. +static int +Lock_release_impl(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + // threading.Lock is unowned, so release() may be called from a different + // thread than acquire(). CPython's atomic _PyMutex_TryUnlock() is not part + // of the public API and is not exported by all CPython builds. This fast + // partial-replacement path deliberately avoids a second guard mutex: + // ordinary release() of an unlocked lock raises RuntimeError, but racy + // erroneous release() calls are undefined behavior and may make + // PyMutex_Unlock() abort. + if (!PyMutex_IsLocked(&self->mutex)) { + return -1; + } + PyMutex_Unlock(&self->mutex); + return 0; + +#elif defined(LOCK_BACKEND_SRWLOCK) + AcquireSRWLockExclusive(&self->srw); + if (!self->locked) { + ReleaseSRWLockExclusive(&self->srw); + return -1; + } + self->locked = 0; + // Wake one waiter (if any). Signalling under `srw` is fine and avoids a + // lost-wakeup race. + WakeConditionVariable(&self->lock_released); + ReleaseSRWLockExclusive(&self->srw); + return 0; + +#elif defined(LOCK_BACKEND_SEM) + // Check-then-clear the flag, then post. This backend is only selected when + // the GIL is enabled, so the check and clear are serialized without an + // atomic. + if (!self->locked) { + return -1; + } + self->locked = 0; + sem_post(&self->sem); + return 0; + +#else // pthread mutex + condvar fallback + pthread_mutex_lock(&self->mut); + if (!self->locked) { + pthread_mutex_unlock(&self->mut); + return -1; + } + self->locked = 0; + // Wake one waiter (if any). Signalling under `mut` is fine and avoids a + // lost-wakeup race. + pthread_cond_signal(&self->lock_released); + pthread_mutex_unlock(&self->mut); + return 0; +#endif +} + +static inline int +Lock_is_locked(LockObject *self) +{ +#if defined(LOCK_BACKEND_PYMUTEX) + return PyMutex_IsLocked(&self->mutex); +#elif defined(LOCK_BACKEND_SRWLOCK) + // locked() is not expected to be on a perf-critical path, so take the + // SRWLOCK (shared) for a clean read of the guarded flag rather than + // relying on an atomic/volatile field. + AcquireSRWLockShared(&self->srw); + int result = self->locked; + ReleaseSRWLockShared(&self->srw); + return result; +#elif defined(LOCK_BACKEND_SEM) + // The flag is GIL-serialized (see the struct comment); locked() is a + // plain read, matching the old CPython _thread lock bookkeeping described + // above. + return self->locked != 0; +#else // pthread mutex + condvar fallback + // `locked` is only ever accessed under `mut`; take it for a clean read. + // locked() is not expected to be on a perf-critical path. + pthread_mutex_lock(&self->mut); + int result = self->locked; + pthread_mutex_unlock(&self->mut); + return result; +#endif +} + +// ---------- Python type methods (shared across platforms) ---------- + +static PyTypeObject LockType; + +static PyObject * +Lock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &LockType) { + PyErr_SetString(PyExc_TypeError, "Lock cannot be subclassed"); + return NULL; + } + + LockObject *self = (LockObject *)type->tp_alloc(type, 0); + if (self != NULL && Lock_init_internal(self) < 0) { + type->tp_free((PyObject *)self); + return NULL; + } + return (PyObject *)self; +} + +static int +Lock_init(LockObject *self, PyObject *args, PyObject *kwds) +{ + if (!PyArg_ParseTuple(args, "")) { + return -1; + } + + if (kwds != NULL && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "Lock() takes no keyword arguments"); + return -1; + } + + return 0; +} + +static void +Lock_dealloc(LockObject *self) +{ +#if defined(LOCK_BACKEND_SEM) + sem_destroy(&self->sem); +#elif defined(LOCK_BACKEND_PTHREAD) + // `mut` is only ever held transiently within a single call, so it is + // always unlocked here even if the Python lock is still "locked". + // Some pthread implementations require the cond to be destroyed first. + pthread_cond_destroy(&self->lock_released); + pthread_mutex_destroy(&self->mut); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject * +Lock_acquire(LockObject *self, PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) +{ + int blocking = 1; + + Py_ssize_t nkw = kwnames ? PyTuple_GET_SIZE(kwnames) : 0; + if (nargs + nkw > 1) { + PyErr_SetString(PyExc_TypeError, "acquire() takes at most 1 argument"); + return NULL; + } + + if (nargs == 1) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) + return NULL; + } else if (nkw == 1) { + PyObject *key = PyTuple_GET_ITEM(kwnames, 0); + if (PyUnicode_CompareWithASCIIString(key, "blocking") != 0) { + PyErr_Format(PyExc_TypeError, + "acquire() got an unexpected keyword argument '%U'", + key); + return NULL; + } + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) + return NULL; + } + + int result = Lock_acquire_impl(self, blocking); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject * +Lock_release(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (Lock_release_impl(self) < 0) { + PyErr_SetString(PyExc_RuntimeError, "cannot release an unlocked lock"); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +Lock_locked(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyBool_FromLong(Lock_is_locked(self)); +} + +static PyObject * +Lock_enter(LockObject *self, PyObject *Py_UNUSED(ignored)) +{ + int result = Lock_acquire_impl(self, 1); + if (result < 0) + return NULL; + return PyBool_FromLong(result); +} + +static PyObject * +Lock_exit(LockObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + return Lock_release(self, NULL); +} + +static PyMethodDef Lock_methods[] = { + {"acquire", (PyCFunction)(void(*)(void))Lock_acquire, METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("Acquire the lock, blocking or non-blocking.\n" + "Returns True if the lock was acquired, False otherwise.")}, + {"release", (PyCFunction)Lock_release, METH_NOARGS, + PyDoc_STR("Release the lock.")}, + {"locked", (PyCFunction)Lock_locked, METH_NOARGS, + PyDoc_STR("Return True if the lock is currently held.")}, + {"__enter__", (PyCFunction)Lock_enter, METH_NOARGS, + PyDoc_STR("Acquire the lock.")}, + {"__exit__", (PyCFunction)Lock_exit, METH_FASTCALL, + PyDoc_STR("Release the lock.")}, + {NULL} +}; + +static PyTypeObject LockType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "Lock", + .tp_doc = PyDoc_STR("A fast mutual exclusion lock"), + .tp_basicsize = sizeof(LockObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = Lock_new, + .tp_init = (initproc)Lock_init, + .tp_dealloc = (destructor)Lock_dealloc, + .tp_methods = Lock_methods, +}; + +static PyTypeObject * +Lock_type_internal(void) { + return &LockType; +} + +// Create a new Lock object (for use from compiled code) +static PyObject * +Lock_new_internal(void) { + LockObject *self = (LockObject *)LockType.tp_alloc(&LockType, 0); + if (self != NULL && Lock_init_internal(self) < 0) { + LockType.tp_free((PyObject *)self); + return NULL; + } + return (PyObject *)self; +} + +// Acquire the lock (blocking), for use from compiled code. +// Returns true on success, sets error and returns 2 (ERR_MAGIC) on failure. +static char +Lock_acquire_internal(PyObject *self) { + int result = Lock_acquire_impl((LockObject *)self, 1); + if (result < 0) { + return 2; + } + return (char)result; +} + +// Acquire the lock with explicit blocking arg, for use from compiled code. +// Returns true if acquired, false otherwise. Sets error and returns 2 +// (ERR_MAGIC) on failure. +static char +Lock_acquire_blocking_internal(PyObject *self, char blocking) { + int result = Lock_acquire_impl((LockObject *)self, blocking); + if (result < 0) { + return 2; + } + return (char)result; +} + +// Release the lock, for use from compiled code. +// Returns 0 (None) on success, sets error and returns 2 (ERR_MAGIC) on failure. +static char +Lock_release_internal(PyObject *self) { + if (Lock_release_impl((LockObject *)self) < 0) { + PyErr_SetString(PyExc_RuntimeError, "cannot release an unlocked lock"); + return 2; + } + return 0; +} + +// Check if the lock is held, for use from compiled code. +static char +Lock_locked_internal(PyObject *self) { + return (char)Lock_is_locked((LockObject *)self); +} + +static PyMethodDef librt_threading_module_methods[] = { + {NULL, NULL, 0, NULL} +}; + +static int +threading_abi_version(void) { + return LIBRT_THREADING_ABI_VERSION; +} + +static int +threading_api_version(void) { + return LIBRT_THREADING_API_VERSION; +} + +static int +librt_threading_module_exec(PyObject *m) +{ + if (PyType_Ready(&LockType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Lock", (PyObject *)&LockType) < 0) { + return -1; + } + + // Export mypyc internal C API via capsule + static void *threading_api[LIBRT_THREADING_API_LEN] = { + (void *)threading_abi_version, + (void *)threading_api_version, + (void *)Lock_type_internal, + (void *)Lock_new_internal, + (void *)Lock_acquire_internal, + (void *)Lock_release_internal, + (void *)Lock_locked_internal, + (void *)Lock_acquire_blocking_internal, + }; + PyObject *c_api_object = PyCapsule_New((void *)threading_api, "librt.threading._C_API", NULL); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot librt_threading_module_slots[] = { + {Py_mod_exec, librt_threading_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef librt_threading_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "threading", + .m_doc = "Fast threading primitives optimized for mypyc", + .m_size = 0, + .m_methods = librt_threading_module_methods, + .m_slots = librt_threading_module_slots, +}; + +PyMODINIT_FUNC +PyInit_threading(void) +{ + return PyModuleDef_Init(&librt_threading_module); +} diff --git a/mypyc/lib-rt/threading/librt_threading.h b/mypyc/lib-rt/threading/librt_threading.h new file mode 100644 index 0000000000000..a22284f3fdbcd --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading.h @@ -0,0 +1,10 @@ +#ifndef LIBRT_THREADING_H +#define LIBRT_THREADING_H + +#include + +#define LIBRT_THREADING_ABI_VERSION 1 +#define LIBRT_THREADING_API_VERSION 1 +#define LIBRT_THREADING_API_LEN 8 + +#endif // LIBRT_THREADING_H diff --git a/mypyc/lib-rt/threading/librt_threading_api.c b/mypyc/lib-rt/threading/librt_threading_api.c new file mode 100644 index 0000000000000..156975d243638 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading_api.c @@ -0,0 +1,43 @@ +#include "librt_threading_api.h" + +void *LibRTThreading_API[LIBRT_THREADING_API_LEN] = {0}; + +int +import_librt_threading(void) +{ + PyObject *mod = PyImport_ImportModule("librt.threading"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + void **capsule = (void **)PyCapsule_Import("librt.threading._C_API", 0); + if (capsule == NULL) + return -1; + + // Only after version validation succeeds can we safely copy the full table. + int (*abi_version)(void) = (int (*)(void))capsule[0]; + int (*api_version)(void) = (int (*)(void))capsule[1]; + if (abi_version() != LIBRT_THREADING_ABI_VERSION) { + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.threading, expected %d, found %d", + LIBRT_THREADING_ABI_VERSION, + abi_version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + if (api_version() < LIBRT_THREADING_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.threading, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_THREADING_API_VERSION, + api_version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + // Provider API version is >= our expected version, which (by the API + // compatibility contract) means it has at least LIBRT_THREADING_API_LEN + // entries, so this copy is safe. + memcpy(LibRTThreading_API, capsule, sizeof(LibRTThreading_API)); + return 0; +} diff --git a/mypyc/lib-rt/threading/librt_threading_api.h b/mypyc/lib-rt/threading/librt_threading_api.h new file mode 100644 index 0000000000000..acb3414d79561 --- /dev/null +++ b/mypyc/lib-rt/threading/librt_threading_api.h @@ -0,0 +1,20 @@ +#ifndef LIBRT_THREADING_API_H +#define LIBRT_THREADING_API_H + +#include "librt_threading.h" + +int +import_librt_threading(void); + +extern void *LibRTThreading_API[LIBRT_THREADING_API_LEN]; + +#define LibRTThreading_ABIVersion (*(int (*)(void)) LibRTThreading_API[0]) +#define LibRTThreading_APIVersion (*(int (*)(void)) LibRTThreading_API[1]) +#define LibRTThreading_Lock_type_internal (*(PyTypeObject* (*)(void)) LibRTThreading_API[2]) +#define LibRTThreading_Lock_new_internal (*(PyObject* (*)(void)) LibRTThreading_API[3]) +#define LibRTThreading_Lock_acquire_internal (*(char (*)(PyObject *self)) LibRTThreading_API[4]) +#define LibRTThreading_Lock_release_internal (*(char (*)(PyObject *self)) LibRTThreading_API[5]) +#define LibRTThreading_Lock_locked_internal (*(char (*)(PyObject *self)) LibRTThreading_API[6]) +#define LibRTThreading_Lock_acquire_blocking_internal (*(char (*)(PyObject *self, char blocking)) LibRTThreading_API[7]) + +#endif // LIBRT_THREADING_API_H diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3cfd1756201ce..5c0865ed7772e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -756,6 +756,36 @@ static inline VecT vec_from_sequence( VecT v = vec_alloc(alloc_size, item_type); if (VEC_IS_ERROR(v)) return vec_error(); +#ifdef Py_GIL_DISABLED + if (is_list) { + // Other threads could be mutating the list concurrently, so use strong references + Py_ssize_t actual = 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(seq) && i < n; i++) { + PyObject *item = PyList_GetItemRef(seq, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + VEC_DECREF(v); + return vec_error(); + } + if (!VecT_ItemCheck(v, item, item_type)) { + Py_DECREF(item); + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + VEC_DECREF(v); + return vec_error(); + } + v.items[actual++] = item; + } + for (Py_ssize_t j = actual; j < alloc_size; j++) + v.items[j] = NULL; + vec_track_buffer(&v); + v.len = actual; + return v; + } + // Tuples are immutable, so fall through to the shared loop below +#endif for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = is_list ? PyList_GET_ITEM(seq, i) : PyTuple_GET_ITEM(seq, i); if (!VecT_ItemCheck(v, item, item_type)) { diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index b3e80261fec38..a6b09ba2a7b29 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -148,6 +148,30 @@ static inline VEC vec_from_sequence(PyObject *seq, int64_t cap, const int is_lis VEC v = vec_alloc(alloc_size); if (VEC_IS_ERROR(v)) return vec_error(); +#ifdef Py_GIL_DISABLED + if (is_list) { + // Other threads could be mutating the list concurrently, so use strong references + Py_ssize_t actual = 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(seq) && i < n; i++) { + PyObject *item = PyList_GetItemRef(seq, i); + if (unlikely(item == NULL)) { + // Race condition: list shrank between size read and get item + VEC_DECREF(v); + return vec_error(); + } + ITEM_C_TYPE x = UNBOX_ITEM(item); + Py_DECREF(item); + if (IS_UNBOX_ERROR(x)) { + VEC_DECREF(v); + return vec_error(); + } + v.items[actual++] = x; + } + v.len = actual; + return v; + } + // Tuples are immutable, so fall through to the shared loop below +#endif for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = is_list ? PyList_GET_ITEM(seq, i) : PyTuple_GET_ITEM(seq, i); ITEM_C_TYPE x = UNBOX_ITEM(item); diff --git a/mypyc/lower/registry.py b/mypyc/lower/registry.py index dec6a24b9417a..36262c4a011a3 100644 --- a/mypyc/lower/registry.py +++ b/mypyc/lower/registry.py @@ -26,4 +26,4 @@ def wrapper(f: LF) -> LF: # Import various modules that set up global state. -from mypyc.lower import int_ops, list_ops, misc_ops # noqa: F401 +from mypyc.lower import int_ops, list_ops, misc_ops, vec_ops # noqa: F401 diff --git a/mypyc/lower/vec_ops.py b/mypyc/lower/vec_ops.py new file mode 100644 index 0000000000000..768c8e0073af8 --- /dev/null +++ b/mypyc/lower/vec_ops.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from mypyc.ir.ops import Value +from mypyc.irbuild.ll_builder import LowLevelIRBuilder +from mypyc.irbuild.vec import vec_get_item_unsafe_lower +from mypyc.lower.registry import lower_primitive_op + + +@lower_primitive_op("vec_get_item_unsafe") +def vec_get_item_unsafe(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + base, index = args + return vec_get_item_unsafe_lower(builder, base, index, line, can_borrow=False) + + +@lower_primitive_op("vec_get_item_unsafe_borrow") +def vec_get_item_unsafe_borrow(builder: LowLevelIRBuilder, args: list[Value], line: int) -> Value: + base, index = args + return vec_get_item_unsafe_lower(builder, base, index, line, can_borrow=True) diff --git a/mypyc/primitives/librt_threading_ops.py b/mypyc/primitives/librt_threading_ops.py new file mode 100644 index 0000000000000..c060742dc2050 --- /dev/null +++ b/mypyc/primitives/librt_threading_ops.py @@ -0,0 +1,54 @@ +from mypyc.ir.deps import LIBRT_THREADING +from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import bool_rprimitive, lock_rprimitive, none_rprimitive +from mypyc.primitives.registry import function_op, method_op + +# Lock() +function_op( + name="librt.threading.Lock", + arg_types=[], + return_type=lock_rprimitive, + c_function_name="LibRTThreading_Lock_new_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.acquire() -- blocking acquire, returns True unless it raises +lock_acquire_op = method_op( + name="acquire", + arg_types=[lock_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_acquire_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.acquire(blocking) -- acquire with explicit blocking argument +method_op( + name="acquire", + arg_types=[lock_rprimitive, bool_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_acquire_blocking_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.release() +lock_release_op = method_op( + name="release", + arg_types=[lock_rprimitive], + return_type=none_rprimitive, + c_function_name="LibRTThreading_Lock_release_internal", + error_kind=ERR_MAGIC, + dependencies=[LIBRT_THREADING], +) + +# Lock.locked() +method_op( + name="locked", + arg_types=[lock_rprimitive], + return_type=bool_rprimitive, + c_function_name="LibRTThreading_Lock_locked_internal", + error_kind=ERR_NEVER, + dependencies=[LIBRT_THREADING], +) diff --git a/mypyc/primitives/librt_vecs_ops.py b/mypyc/primitives/librt_vecs_ops.py index e4852d5387069..901c779a2bfd2 100644 --- a/mypyc/primitives/librt_vecs_ops.py +++ b/mypyc/primitives/librt_vecs_ops.py @@ -1,13 +1,15 @@ from mypyc.ir.deps import LIBRT_VECS, VECS_EXTRA_OPS from mypyc.ir.ops import ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( + RTypeVar, RVec, bit_rprimitive, bytes_rprimitive, + int64_rprimitive, object_rprimitive, uint8_rprimitive, ) -from mypyc.primitives.registry import function_op +from mypyc.primitives.registry import custom_primitive_op, function_op # isinstance(obj, vec) isinstance_vec = function_op( @@ -28,3 +30,22 @@ error_kind=ERR_MAGIC, dependencies=[LIBRT_VECS, VECS_EXTRA_OPS], ) + +# Get vec item, assuming the index is valid (no bounds check) +vec_get_item_unsafe_op = custom_primitive_op( + name="vec_get_item_unsafe", + arg_types=[RVec(RTypeVar(0)), int64_rprimitive], + return_type=RTypeVar(0), + error_kind=ERR_NEVER, + type_params=[RTypeVar(0)], +) + +# Like vec_get_item_unsafe, but the result is a borrowed reference +vec_get_item_unsafe_borrow_op = custom_primitive_op( + name="vec_get_item_unsafe_borrow", + arg_types=[RVec(RTypeVar(0)), int64_rprimitive], + is_borrowed=True, + return_type=RTypeVar(0), + error_kind=ERR_NEVER, + type_params=[RTypeVar(0)], +) diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index efd40469606f3..8af7e9e71df7e 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -2,6 +2,7 @@ from __future__ import annotations +from mypyc.common import IS_FREE_THREADED from mypyc.ir.ops import ERR_FALSE, ERR_MAGIC, ERR_NEVER from mypyc.ir.rtypes import ( bit_rprimitive, @@ -108,30 +109,8 @@ priority=2, ) -# list[index] that produces a borrowed result -method_op( - name="__getitem__", - arg_types=[list_rprimitive, int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=3, -) - -# list[index] that produces a borrowed result and index is known to be short -method_op( - name="__getitem__", - arg_types=[list_rprimitive, short_int_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemShortBorrow", - error_kind=ERR_MAGIC, - is_borrowed=True, - priority=4, -) - # Version with native int index -method_op( +list_get_item_int64_op = method_op( name="__getitem__", arg_types=[list_rprimitive, int64_rprimitive], return_type=object_rprimitive, @@ -140,16 +119,39 @@ priority=5, ) -# Version with native int index -method_op( - name="__getitem__", - arg_types=[list_rprimitive, int64_rprimitive], - return_type=object_rprimitive, - c_function_name="CPyList_GetItemInt64Borrow", - is_borrowed=True, - error_kind=ERR_MAGIC, - priority=6, -) +if not IS_FREE_THREADED: + # list[index] that produces a borrowed result + method_op( + name="__getitem__", + arg_types=[list_rprimitive, int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=3, + ) + + # list[index] that produces a borrowed result and index is known to be short + method_op( + name="__getitem__", + arg_types=[list_rprimitive, short_int_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemShortBorrow", + error_kind=ERR_MAGIC, + is_borrowed=True, + priority=4, + ) + + # Version with native int index + method_op( + name="__getitem__", + arg_types=[list_rprimitive, int64_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyList_GetItemInt64Borrow", + is_borrowed=True, + error_kind=ERR_MAGIC, + priority=6, + ) # This is unsafe because it assumes that the index is a non-negative integer # that is in-bounds for the list. diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index e22a044d9bb27..060af0f88020c 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -41,7 +41,7 @@ from mypyc.ir.deps import Dependency from mypyc.ir.ops import PrimitiveDescription, StealsDescription -from mypyc.ir.rtypes import RType +from mypyc.ir.rtypes import RType, RTypeVar # Error kind for functions that return negative integer on exception. This # is only used for primitives. We translate it away during IR building. @@ -154,6 +154,7 @@ def method_op( is_pure=is_pure, experimental=experimental, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -204,6 +205,7 @@ def function_op( is_pure=False, experimental=experimental, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -253,6 +255,7 @@ def binary_op( is_pure=False, experimental=False, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -313,6 +316,7 @@ def custom_primitive_op( is_pure: bool = False, experimental: bool = False, dependencies: list[Dependency] | None = None, + type_params: list[RTypeVar] | None = None, ) -> PrimitiveDescription: """Define a primitive op that can't be automatically generated based on the AST. @@ -336,6 +340,7 @@ def custom_primitive_op( is_pure=is_pure, experimental=experimental, dependencies=dependencies, + type_params=type_params, ) @@ -380,6 +385,7 @@ def unary_op( is_pure=is_pure, experimental=False, dependencies=dependencies, + type_params=None, ) ops.append(desc) return desc @@ -405,6 +411,7 @@ def load_global_op(name: str, type: RType, src: str) -> LoadAddressDescription: import mypyc.primitives.int_ops import mypyc.primitives.librt_random_ops import mypyc.primitives.librt_strings_ops +import mypyc.primitives.librt_threading_ops import mypyc.primitives.librt_time_ops import mypyc.primitives.librt_vecs_ops import mypyc.primitives.list_ops diff --git a/mypyc/rt_expandtype.py b/mypyc/rt_expandtype.py new file mode 100644 index 0000000000000..8537e6777ccb5 --- /dev/null +++ b/mypyc/rt_expandtype.py @@ -0,0 +1,32 @@ +from mypyc.ir.rtypes import ( + RArray, + RInstance, + RPrimitive, + RStruct, + RTuple, + RType, + RTypeVar, + RUnion, + RVec, + RVoid, +) + + +def expand_rtype(typ: RType, type_args: list[RType]) -> RType: + if isinstance(typ, (RPrimitive, RInstance, RVoid)): + # Atomic types can't contain type variables + return typ + elif isinstance(typ, RTypeVar): + return type_args[typ.id] + elif isinstance(typ, RVec): + return RVec(expand_rtype(typ.item_type, type_args)) + elif isinstance(typ, RUnion): + return RUnion([expand_rtype(item, type_args) for item in typ.items]) + elif isinstance(typ, RTuple): + return RTuple([expand_rtype(item, type_args) for item in typ.types]) + elif isinstance(typ, RStruct): + assert False, "Generic RStruct type not supported" + elif isinstance(typ, RArray): + assert False, "Generic RArray type not supported" + else: + assert False, r"Unexpected type {typ!r}" diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index c271c9ae63d5e..b8d8b4727363c 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -98,7 +98,7 @@ L6: r5 = :: int return r5 -[case testListSum] +[case testListSum_withgil] from typing import List def sum(a: List[int], l: int) -> int: sum = 0 @@ -553,7 +553,7 @@ L3: r3 = :: i64 return r3 -[case testExceptionWithNativeAttributeGetAndSet] +[case testExceptionWithNativeAttributeGetAndSet_withgil] class C: def __init__(self, x: int) -> None: self.x = x @@ -578,6 +578,32 @@ L0: c.x = r1 return 1 +[case testExceptionWithNativeAttributeGetAndSet_nogil] +class C: + def __init__(self, x: int) -> None: + self.x = x + +def foo(c: C, x: int) -> None: + c.x = x - c.x +[out] +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def foo(c, x): + c :: __main__.C + x, r0, r1 :: int + r2 :: bool +L0: + r0 = c.x + r1 = CPyTagged_Subtract(x, r0) + dec_ref r0 :: int + c.x = r1 + return 1 + [case testExceptionWithOverlappingFloatErrorValue] def f() -> float: return 0.0 diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index a9d7ee2357b68..c6c231f0386be 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -1870,7 +1870,7 @@ L0: r5 = a.f(12, 6, r4) return 1 -[case testListComprehension] +[case testListComprehension_withgil] from typing import List def f() -> List[int]: @@ -1931,7 +1931,7 @@ L7: L8: return r0 -[case testDictComprehension] +[case testDictComprehension_withgil] from typing import Dict def f() -> Dict[int, int]: return {x: x*x for x in [1,2,3] if x != 2 if x != 3} @@ -1993,7 +1993,7 @@ L7: L8: return r0 -[case testLoopsMultipleAssign] +[case testLoopsMultipleAssign_withgil] from typing import List, Tuple def f(l: List[Tuple[int, int, int]]) -> List[int]: for x, y, z in l: @@ -2063,7 +2063,7 @@ L7: L8: return r10 -[case testProperty] +[case testProperty_withgil] class PropertyHolder: @property def value(self) -> int: @@ -2114,7 +2114,7 @@ L0: r1 = CPyTagged_Multiply(4, r0) return r1 -[case testNativeIndex] +[case testNativeIndex_withgil] from typing import List class A: def __getitem__(self, index: int) -> int: pass diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a7fdc09399009..8eaa63e4583b5 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -26,7 +26,7 @@ L0: a.x = 2; r0 = is_error return 1 -[case testUserClassInList] +[case testUserClassInList_withgil] class C: x: int @@ -103,7 +103,7 @@ L0: r0 = a.n return r0 -[case testOptionalMember] +[case testOptionalMember_withgil] from typing import Optional class Node: next: Optional[Node] @@ -1239,7 +1239,7 @@ L0: __mypyc_self__.s = r0 return 1 -[case testBorrowAttribute] +[case testBorrowAttribute_withgil] def f(d: D) -> int: return d.c.x @@ -1258,6 +1258,64 @@ L0: keep_alive d return r1 +[case testCannotBorrowAttribute_nogil] +def f(d: D) -> int: + return d.c.x + +class C: + x: int +class D: + c: C +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = d.c + r1 = r0.x + return r1 + +[case testCanBorrowFinalAttribute_nogil] +from typing import Final + +# On free-threaded builds native attribute reads are not borrowed (a concurrent +# store could free the old value), EXCEPT Final attributes, which can't be rebound. +# Here the intermediate 'd.c' is borrowed because 'c' is Final; contrast with +# testCannotBorrowAttribute_nogil, where the non-Final 'c' is read owned. The +# borrow decision uses the same ClassIR.is_final_attr predicate as codegen. +def f(d: D) -> int: + return d.c.xf + +class C: + def __init__(self, xf: int) -> None: + self.xf: Final = xf +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.xf + keep_alive d + return r1 +def C.__init__(self, xf): + self :: __main__.C + xf :: int +L0: + self.xf = xf + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + self.c = c + return 1 + [case testNoBorrowOverPropertyAccess] class C: d: D @@ -1287,7 +1345,7 @@ L0: r2 = r1.x return r2 -[case testBorrowResultOfCustomGetItemInIfStatement] +[case testBorrowResultOfCustomGetItemInIfStatement_withgil] from typing import List class C: diff --git a/mypyc/test-data/irbuild-final.test b/mypyc/test-data/irbuild-final.test new file mode 100644 index 0000000000000..0ede988a43206 --- /dev/null +++ b/mypyc/test-data/irbuild-final.test @@ -0,0 +1,252 @@ +[case testFinalAttrBorrowed] +from typing import Final + +class C: + def __init__(self, x: str) -> None: + self.f: Final = x + self.x = x + +def f(s: str) -> None: pass + +def calls(c: C) -> None: + f(c.f) + f(c.x) + c.f.startswith("a") + +def ret(c: C) -> str: + return c.f +[out] +def C.__init__(self, x): + self :: __main__.C + x :: str +L0: + self.f = x + self.x = x + return 1 +def f(s): + s :: str +L0: + return 1 +def calls(c): + c :: __main__.C + r0 :: str + r1 :: None + r2 :: str + r3 :: None + r4, r5 :: str + r6 :: i32 + r7 :: bool +L0: + r0 = borrow c.f + r1 = f(r0) + keep_alive c + r2 = c.x + r3 = f(r2) + r4 = borrow c.f + r5 = 'a' + r6 = CPyStr_Startswith(r4, r5) + r7 = truncate r6: i32 to builtins.bool + keep_alive c + return 1 +def ret(c): + c :: __main__.C + r0 :: str +L0: + r0 = c.f + return r0 + +[case testInheritedFinalAttrBorrowed] +from typing import Final + +class Base: + x: str + + def __init__(self, s: str) -> None: + self.s: Final = s + +class Deriv(Base): + def __init__(self, n: int) -> None: + self.n: Final = n + +def f(b: Base) -> str: + return b.s + "a" + +def g(d: Deriv) -> str: + return d.s * d.n + +def h(d: Deriv) -> str: + return d.x + "a" +[out] +def Base.__init__(self, s): + self :: __main__.Base + s :: str +L0: + self.s = s + return 1 +def Deriv.__init__(self, n): + self :: __main__.Deriv + n :: int +L0: + self.n = n + return 1 +def f(b): + b :: __main__.Base + r0, r1, r2 :: str +L0: + r0 = borrow b.s + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + keep_alive b + return r2 +def g(d): + d :: __main__.Deriv + r0 :: str + r1 :: int + r2 :: str +L0: + r0 = borrow d.s + r1 = borrow d.n + r2 = CPyStr_Multiply(r0, r1) + keep_alive d, d + return r2 +def h(d): + d :: __main__.Deriv + r0, r1, r2 :: str +L0: + r0 = d.x + r1 = 'a' + r2 = PyUnicode_Concat(r0, r1) + return r2 + +[case testFinalAttrExpressionKinds] +from typing import Final + +class C: + def __init__(self, s: str, n: int, l: list[str]) -> None: + self.f: Final = s + self.n: Final = n + self.l: Final = l + self.s = s + +def f() -> C: + return C("", 0, []) + +def assign(c: C) -> None: + a = c.f + c.s = c.f + +def indexing(c: C) -> str: + return c.l[c.n] + +def call_eq(c: C) -> bool: + return f().f == c.f +[out] +def C.__init__(self, s, n, l): + self :: __main__.C + s :: str + n :: int + l :: list +L0: + self.f = s + self.n = n + self.l = l + self.s = s + return 1 +def f(): + r0 :: str + r1 :: list + r2 :: __main__.C +L0: + r0 = '' + r1 = PyList_New(0) + r2 = C(r0, 0, r1) + return r2 +def assign(c): + c :: __main__.C + r0, a, r1 :: str + r2 :: bool +L0: + r0 = c.f + a = r0 + r1 = c.f + c.s = r1; r2 = is_error + return 1 +def indexing(c): + c :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = borrow c.l + r1 = borrow c.n + r2 = CPyList_GetItem(r0, r1) + r3 = cast(str, r2) + keep_alive c, c + return r3 +def call_eq(c): + c, r0 :: __main__.C + r1, r2 :: str + r3 :: bool +L0: + r0 = f() + r1 = borrow r0.f + r2 = borrow c.f + r3 = CPyStr_Equal(r1, r2) + keep_alive r0, c + return r3 + +[case testFinalAttrExpressionKinds2_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f(c: C) -> str: + return c.d.f + "a" + +def g(a: list[D], n: int) -> str: + return a[n].f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + self.f = s + return 1 +def f(c): + c :: __main__.C + r0 :: __main__.D + r1, r2, r3 :: str +L0: + r0 = borrow c.d + r1 = borrow r0.f + r2 = 'a' + r3 = PyUnicode_Concat(r1, r2) + keep_alive c, r0 + return r3 +def g(a, n): + a :: list + n :: int + r0 :: object + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CPyList_GetItemBorrow(a, n) + r1 = borrow cast(__main__.D, r0) + r2 = r1.f + keep_alive a, n, r0 + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + return r4 diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 9ec29182e89b6..0d4dad4f97e23 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -40,7 +40,7 @@ L0: y = r3 return 1 -[case testGenericAttrAndTypeApplication] +[case testGenericAttrAndTypeApplication_withgil] from typing import TypeVar, Generic T = TypeVar('T') class C(Generic[T]): diff --git a/mypyc/test-data/irbuild-glue-methods.test b/mypyc/test-data/irbuild-glue-methods.test index 87b3bf7fee588..bdbdb7e2afa68 100644 --- a/mypyc/test-data/irbuild-glue-methods.test +++ b/mypyc/test-data/irbuild-glue-methods.test @@ -91,7 +91,7 @@ L0: r0 = x.foo(y) return r0 -[case testPropertyDerivedGen] +[case testPropertyDerivedGen_withgil] from typing import Callable class BaseProperty: @property diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index 0fcadefd46c27..4cea7894052aa 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -1032,7 +1032,7 @@ L4: y = r3 return y -[case testBorrowOverI64Arithmetic] +[case testBorrowOverI64Arithmetic_withgil] from mypy_extensions import i64 def add_simple(c: C) -> i64: @@ -1084,7 +1084,7 @@ L0: keep_alive d, d return r4 -[case testBorrowOverI64Bitwise] +[case testBorrowOverI64Bitwise_withgil] from mypy_extensions import i64 def bitwise_simple(c: C) -> i64: @@ -1136,7 +1136,7 @@ L0: keep_alive d, d return r4 -[case testBorrowOverI64ListGetItem1] +[case testBorrowOverI64ListGetItem1_withgil] from mypy_extensions import i64 def f(n: i64) -> str: @@ -1168,7 +1168,38 @@ L0: keep_alive a, n, r3 return r5 -[case testBorrowOverI64ListGetItem2] +[case testBorrowOverI64ListGetItem1_nogil] +from mypy_extensions import i64 + +def f(n: i64) -> str: + a = [C()] + return a[n].s + +class C: + s: str +[out] +def f(n): + n :: i64 + r0 :: __main__.C + r1 :: list + r2 :: ptr + a :: list + r3 :: object + r4 :: __main__.C + r5 :: str +L0: + r0 = C() + r1 = PyList_New(1) + r2 = list_items r1 + buf_init_item r2, 0, r0 + keep_alive r1 + a = r1 + r3 = CPyList_GetItemInt64(a, n) + r4 = cast(__main__.C, r3) + r5 = r4.s + return r5 + +[case testBorrowOverI64ListGetItem2_withgil] from typing import List from mypy_extensions import i64 @@ -1194,6 +1225,31 @@ L1: L2: return 0 +[case testBorrowOverI64ListGetItem2_nogil] +from typing import List +from mypy_extensions import i64 + +def f(a: List[i64], n: i64) -> bool: + if a[n] == 0: + return True + return False +[out] +def f(a, n): + a :: list + n :: i64 + r0 :: object + r1 :: i64 + r2 :: bit +L0: + r0 = CPyList_GetItemInt64(a, n) + r1 = unbox(i64, r0) + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + [case testCoerceShortIntToI64] from mypy_extensions import i64 from typing import List @@ -2014,7 +2070,7 @@ L4: r6 = CPyFloat_FromTagged(r3) return r6 -[case testI64IsinstanceNarrowing] +[case testI64IsinstanceNarrowing_withgil] from typing import Union from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 36a9300350bd3..216a52ff2a25c 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -48,7 +48,7 @@ L2: L3: return r1 -[case testBorrowSpecialCaseWithIsinstance] +[case testBorrowSpecialCaseWithIsinstance_withgil] class C: s: str diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index f6cdbcaf3a96e..e70d81c96bf7a 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -26,7 +26,7 @@ L0: r1 = cast(list, r0) return r1 -[case testListOfListGet2] +[case testListOfListGet2_withgil] from typing import List def f(x: List[List[int]]) -> int: return x[0][1] @@ -45,6 +45,24 @@ L0: keep_alive x, r0 return r3 +[case testListOfListGet2_nogil] +from typing import List +def f(x: List[List[int]]) -> int: + return x[0][1] +[out] +def f(x): + x :: list + r0 :: object + r1 :: list + r2 :: object + r3 :: int +L0: + r0 = CPyList_GetItemShort(x, 0) + r1 = cast(list, r0) + r2 = CPyList_GetItemShort(r1, 2) + r3 = unbox(int, r2) + return r3 + [case testListSet] from typing import List def f(x: List[int]) -> None: @@ -361,7 +379,7 @@ L0: r2 = r1 >= 0 :: signed return 1 -[case testListBuiltFromGenerator] +[case testListBuiltFromGenerator_withgil] from typing import List def f(source: List[int]) -> None: a = list(x + 1 for x in source) @@ -430,7 +448,7 @@ L8: b = r11 return 1 -[case testGeneratorNext] +[case testGeneratorNext_withgil] from typing import List, Optional def test(x: List[int]) -> None: @@ -471,7 +489,7 @@ L5: res = r6 return 1 -[case testSimplifyListUnion] +[case testSimplifyListUnion_withgil] from typing import List, Union, Optional def narrow(a: Union[List[str], List[bytes], int]) -> int: @@ -905,7 +923,7 @@ L10: a = r3 return 1 -[case testListBuiltFromStars] +[case testListBuiltFromStars_withgil] from typing import Final abc: Final = "abc" diff --git a/mypyc/test-data/irbuild-optional.test b/mypyc/test-data/irbuild-optional.test index fbf7cb148b089..252f9489dc57f 100644 --- a/mypyc/test-data/irbuild-optional.test +++ b/mypyc/test-data/irbuild-optional.test @@ -237,7 +237,7 @@ L3: L4: return 1 -[case testUnionType] +[case testUnionType_withgil] from typing import Union class A: diff --git a/mypyc/test-data/irbuild-set.test b/mypyc/test-data/irbuild-set.test index 00bcff68d4ff5..ac01c58401118 100644 --- a/mypyc/test-data/irbuild-set.test +++ b/mypyc/test-data/irbuild-set.test @@ -53,7 +53,7 @@ L0: r0 = PySet_New(l) return r0 -[case testNewSetFromIterable2] +[case testNewSetFromIterable2_withgil] def f(x: int) -> int: return x @@ -273,7 +273,7 @@ L4: e = r0 return 1 -[case testNewSetFromIterable3] +[case testNewSetFromIterable3_withgil] def f1(x: int) -> int: return x diff --git a/mypyc/test-data/irbuild-statements.test b/mypyc/test-data/irbuild-statements.test index df7eb5fc7449a..8c5c95c8223ec 100644 --- a/mypyc/test-data/irbuild-statements.test +++ b/mypyc/test-data/irbuild-statements.test @@ -218,7 +218,7 @@ L5: L6: return 1 -[case testForList] +[case testForList_withgil] from typing import List def f(ls: List[int]) -> int: @@ -255,6 +255,43 @@ L3: L4: return y +[case testForList_nogil] +from typing import List + +def f(ls: List[int]) -> int: + y = 0 + for x in ls: + y = y + x + return y +[out] +def f(ls): + ls :: list + y :: int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x, r5 :: int + r6 :: native_int +L0: + y = 0 + r0 = 0 +L1: + r1 = var_object_size ls + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = CPyList_GetItemInt64(ls, r0) + r4 = unbox(int, r3) + x = r4 + r5 = CPyTagged_Add(y, x) + y = r5 +L3: + r6 = r0 + 1 + r0 = r6 + goto L1 +L4: + return y + [case testForDictBasic] from typing import Dict @@ -575,7 +612,7 @@ L0: z = r6 return 1 -[case testMultipleAssignmentUnpackFromSequence] +[case testMultipleAssignmentUnpackFromSequence_withgil] from typing import List, Tuple def f(l: List[int], t: Tuple[int, ...]) -> None: @@ -612,6 +649,43 @@ L0: y = r9 return 1 +[case testMultipleAssignmentUnpackFromSequence_nogil] +from typing import List, Tuple + +def f(l: List[int], t: Tuple[int, ...]) -> None: + x: object + y: int + x, y = l + x, y = t +[out] +def f(l, t): + l :: list + t :: tuple + r0 :: i32 + r1 :: bit + r2, r3, x :: object + r4, y :: int + r5 :: i32 + r6 :: bit + r7, r8 :: object + r9 :: int +L0: + r0 = CPySequence_CheckUnpackCount(l, 2) + r1 = r0 >= 0 :: signed + r2 = CPyList_GetItemInt64(l, 0) + r3 = CPyList_GetItemInt64(l, 1) + x = r2 + r4 = unbox(int, r3) + y = r4 + r5 = CPySequence_CheckUnpackCount(t, 2) + r6 = r5 >= 0 :: signed + r7 = CPySequenceTuple_GetItemUnsafe(t, 0) + r8 = CPySequenceTuple_GetItemUnsafe(t, 1) + x = r7 + r9 = unbox(int, r8) + y = r9 + return 1 + [case testAssert] from typing import Optional @@ -862,7 +936,7 @@ L0: r6 = r5 >= 0 :: signed return 1 -[case testForEnumerate] +[case testForEnumerate_withgil] from typing import List, Iterable def f(a: List[int]) -> None: @@ -930,7 +1004,7 @@ L4: L5: return 1 -[case testForZip] +[case testForZip_withgil] from typing import List, Iterable, Sequence def f(a: List[int], b: Sequence[bool]) -> None: diff --git a/mypyc/test-data/irbuild-threading.test b/mypyc/test-data/irbuild-threading.test new file mode 100644 index 0000000000000..652ccedcb9d75 --- /dev/null +++ b/mypyc/test-data/irbuild-threading.test @@ -0,0 +1,115 @@ +[case testLockBasics] +from librt.threading import Lock + +def lock_create() -> Lock: + return Lock() + +def lock_acquire(lk: Lock) -> bool: + return lk.acquire() + +def lock_release(lk: Lock) -> None: + lk.release() + +def lock_locked(lk: Lock) -> bool: + return lk.locked() +[out] +def lock_create(): + r0 :: librt.threading.Lock +L0: + r0 = LibRTThreading_Lock_new_internal() + return r0 +def lock_acquire(lk): + lk :: librt.threading.Lock + r0 :: bool +L0: + r0 = LibRTThreading_Lock_acquire_internal(lk) + return r0 +def lock_release(lk): + lk :: librt.threading.Lock + r0 :: None +L0: + r0 = LibRTThreading_Lock_release_internal(lk) + return 1 +def lock_locked(lk): + lk :: librt.threading.Lock + r0 :: bool +L0: + r0 = LibRTThreading_Lock_locked_internal(lk) + return r0 + +[case testLockAcquireRelease] +from librt.threading import Lock + +def acquire_release() -> None: + lk = Lock() + lk.acquire() + lk.release() +[out] +def acquire_release(): + r0, lk :: librt.threading.Lock + r1 :: bool + r2 :: None +L0: + r0 = LibRTThreading_Lock_new_internal() + lk = r0 + r1 = LibRTThreading_Lock_acquire_internal(lk) + r2 = LibRTThreading_Lock_release_internal(lk) + return 1 + +[case testLockAcquireBlocking] +from librt.threading import Lock + +def lock_acquire_blocking(lk: Lock, b: bool) -> bool: + return lk.acquire(b) +[out] +def lock_acquire_blocking(lk, b): + lk :: librt.threading.Lock + b, r0 :: bool +L0: + r0 = LibRTThreading_Lock_acquire_blocking_internal(lk, b) + return r0 + +[case testLockWith] +from librt.threading import Lock + +def with_lock(lk: Lock) -> None: + with lk: + a: list[int] = [] +[out] +def with_lock(lk): + lk :: librt.threading.Lock + r0 :: bool + r1, a :: list + r2, r3, r4 :: tuple[object, object, object] + r5 :: None + r6 :: bit +L0: + r0 = LibRTThreading_Lock_acquire_internal(lk) +L1: + r1 = PyList_New(0) + a = r1 +L2: +L3: + r2 = :: tuple[object, object, object] + r3 = r2 + goto L5 +L4: (handler for L1) + r4 = CPy_CatchError() + r3 = r4 +L5: + r5 = LibRTThreading_Lock_release_internal(lk) + if is_error(r3) goto L7 else goto L6 +L6: + CPy_Reraise() + unreachable +L7: + goto L11 +L8: (handler for L5, L6) + if is_error(r3) goto L10 else goto L9 +L9: + CPy_RestoreExcInfo(r3) +L10: + r6 = CPy_KeepPropagating() + unreachable +L11: + return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 9f6b0f5d07390..808b5b9e0d4ab 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -281,7 +281,7 @@ L5: L6: return r6 -[case testTupleBuiltFromList] +[case testTupleBuiltFromList_withgil] def f(val: int) -> bool: return val % 2 == 0 @@ -1018,7 +1018,7 @@ L5: a = r42 return 1 -[case testTupleBuiltFromStars] +[case testTupleBuiltFromStars_withgil] from typing import Final abc: Final = "abc" diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index aeab3ed9f2a8b..176e2616a4856 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -64,11 +64,7 @@ def f(v, i): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: i64 + r5, r6 :: i64 L0: r0 = v.len r1 = i < r0 :: unsigned @@ -86,15 +82,10 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[i64] v, r5 + return r6 [case testVecI64GetItem_32bit] -# The IR is quite verbose, but it's acceptable since 32-bit targets are not common any more from librt.vecs import vec from mypy_extensions import i64 @@ -110,13 +101,7 @@ def f(v, i): r3 :: i64 r4 :: bit r5 :: bool - r6 :: i64 - r7, r8 :: bit - r9 :: native_int - r10 :: ptr - r11 :: native_int - r12 :: ptr - r13 :: i64 + r6, r7 :: i64 L0: r0 = v.len r1 = extend signed r0: native_int to i64 @@ -135,24 +120,8 @@ L3: L4: r6 = i L5: - r7 = r6 < 2147483648 :: signed - if r7 goto L6 else goto L8 :: bool -L6: - r8 = r6 >= -2147483648 :: signed - if r8 goto L7 else goto L8 :: bool -L7: - r9 = truncate r6: i64 to native_int - goto L9 -L8: - CPyInt32_Overflow() - unreachable -L9: - r10 = v.items - r11 = r9 * 8 - r12 = r10 + r11 - r13 = load_mem r12 :: i64* - keep_alive v - return r13 + r7 = vec_get_item_unsafe[i64] v, r6 + return r7 [case testVecI64Append] from librt.vecs import vec, append @@ -365,7 +334,7 @@ L3: L4: return r1 -[case testVecI64FastComprehensionFromList] +[case testVecI64FastComprehensionFromList_nogil] from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -395,7 +364,7 @@ L1: r4 = r2 < r3 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = list_get_item_unsafe l, r2 + r5 = CPyList_GetItemInt64(l, r2) r6 = unbox(i64, r5) x = r6 r7 = x + 1 @@ -411,7 +380,7 @@ L3: L4: return r1 -[case testVecI64FastComprehensionFromVec] +[case testVecI64FastComprehensionFromVec_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -426,14 +395,11 @@ def f(n, v): r1 :: vec[i64] r2, r3 :: native_int r4 :: bit - r5 :: ptr - r6 :: native_int + r5, x, r6 :: i64 r7 :: ptr - r8, x, r9 :: i64 - r10 :: ptr - r11 :: native_int - r12 :: ptr - r13 :: native_int + r8 :: native_int + r9 :: ptr + r10 :: native_int L0: r0 = v.len r1 = VecI64Api.alloc(r0, r0) @@ -443,21 +409,17 @@ L1: r4 = r2 < r3 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = v.items - r6 = r2 * 8 - r7 = r5 + r6 - r8 = load_mem r7 :: i64* - x = r8 - keep_alive v - r9 = x + 1 - r10 = r1.items - r11 = r2 * 8 - r12 = r10 + r11 - set_mem r12, r9 :: i64* + r5 = vec_get_item_unsafe[i64] v, r2 + x = r5 + r6 = x + 1 + r7 = r1.items + r8 = r2 * 8 + r9 = r7 + r8 + set_mem r9, r6 :: i64* keep_alive r1 L3: - r13 = r2 + 1 - r2 = r13 + r10 = r2 + 1 + r2 = r10 goto L1 L4: return r1 @@ -499,7 +461,7 @@ L3: L4: return r1 -[case testVecI64ForLoop] +[case testVecI64ForLoop_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -514,11 +476,8 @@ def f(v): t :: i64 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, x, r7 :: i64 - r8 :: native_int + r3, x, r4 :: i64 + r5 :: native_int L0: t = 0 r0 = 0 @@ -527,17 +486,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 8 - r5 = r3 + r4 - r6 = load_mem r5 :: i64* - x = r6 - keep_alive v - r7 = t + 1 - t = r7 + r3 = vec_get_item_unsafe[i64] v, r0 + x = r3 + r4 = t + 1 + t = r4 L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return t @@ -601,11 +556,7 @@ def f(v): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: i64 + r5, r6 :: i64 L0: r0 = v.len r1 = 0 < r0 :: unsigned @@ -623,12 +574,8 @@ L3: L4: r5 = 0 L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[i64] v, r5 + return r6 [case testVecI64Slicing_64bit] from librt.vecs import vec @@ -720,20 +667,16 @@ def inplace(v, n, m): r2 :: i64 r3 :: bit r4 :: bool - r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9, r10 :: i64 - r11 :: native_int - r12 :: bit + r5, r6, r7 :: i64 + r8 :: native_int + r9 :: bit + r10 :: i64 + r11 :: bit + r12 :: bool r13 :: i64 - r14 :: bit - r15 :: bool - r16 :: i64 - r17 :: ptr - r18 :: i64 - r19 :: ptr + r14 :: ptr + r15 :: i64 + r16 :: ptr L0: r0 = v.len r1 = n < r0 :: unsigned @@ -751,32 +694,28 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: i64* - keep_alive v - r10 = r9 + m - r11 = v.len - r12 = n < r11 :: unsigned - if r12 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe[i64] v, r5 + r7 = r6 + m + r8 = v.len + r9 = n < r8 :: unsigned + if r9 goto L9 else goto L6 :: bool L6: - r13 = n + r11 - r14 = r13 < r11 :: unsigned - if r14 goto L8 else goto L7 :: bool + r10 = n + r8 + r11 = r10 < r8 :: unsigned + if r11 goto L8 else goto L7 :: bool L7: - r15 = raise IndexError + r12 = raise IndexError unreachable L8: - r16 = r13 + r13 = r10 goto L10 L9: - r16 = n + r13 = n L10: - r17 = v.items - r18 = r16 * 8 - r19 = r17 + r18 - set_mem r19, r10 :: i64* + r14 = v.items + r15 = r13 * 8 + r16 = r14 + r15 + set_mem r16, r7 :: i64* keep_alive v return 1 diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index c0d4325e38fcc..22037d8597a73 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -125,10 +125,7 @@ def get_item_bool(v, i): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: bool + r6 :: bool L0: r0 = v.len r1 = i < r0 :: unsigned @@ -146,12 +143,8 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 1 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.bool* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[bool] v, r5 + return r6 [case testVecMiscPop] from librt.vecs import vec, pop @@ -209,7 +202,7 @@ L0: r0 = VecFloatApi.slice(v, x, y) return r0 -[case testVecMiscForLoop] +[case testVecMiscForLoop_64bit] from librt.vecs import vec, remove from mypy_extensions import i64, i16 @@ -225,11 +218,8 @@ def for_bool(v): s :: i16 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, x, r7 :: i16 - r8 :: native_int + r3, x, r4 :: i16 + r5 :: native_int L0: s = 0 r0 = 0 @@ -238,17 +228,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 2 - r5 = r3 + r4 - r6 = load_mem r5 :: i16* - x = r6 - keep_alive v - r7 = s + x - s = r7 + r3 = vec_get_item_unsafe[i16] v, r0 + x = r3 + r4 = s + x + s = r4 L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return s @@ -270,10 +256,7 @@ def get_item_nested(v, i): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i32] + r6 :: vec[i32] L0: r0 = v.len r1 = i < r0 :: unsigned @@ -291,12 +274,8 @@ L3: L4: r5 = i L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[i32]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[i32]] v, r5 + return r6 [case testVecMiscNestedPop_64bit] from librt.vecs import vec, pop diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 1fe42a880d5b0..dd49d9475812f 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -206,10 +206,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[str] + r6 :: vec[str] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -227,12 +224,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[str]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[str]] v, r5 + return r6 [case testVecNestedI64GetItem_64bit] from librt.vecs import vec @@ -250,10 +243,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] + r6 :: vec[i64] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -271,12 +261,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[i64]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[i64]] v, r5 + return r6 [case testVecNestedI64GetItemWithBorrow_64bit] from librt.vecs import vec @@ -294,20 +280,13 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] - r10 :: native_int - r11 :: bit - r12 :: i64 - r13 :: bit - r14 :: bool - r15 :: i64 - r16 :: ptr - r17 :: i64 - r18 :: ptr - r19 :: i64 + r6 :: vec[i64] + r7 :: native_int + r8 :: bit + r9 :: i64 + r10 :: bit + r11 :: bool + r12, r13 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned @@ -325,32 +304,94 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = borrow load_mem r8 :: vec[i64]* - r10 = r9.len - r11 = n < r10 :: unsigned - if r11 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe_borrow[vec[i64]] v, r5 + r7 = r6.len + r8 = n < r7 :: unsigned + if r8 goto L9 else goto L6 :: bool +L6: + r9 = n + r7 + r10 = r9 < r7 :: unsigned + if r10 goto L8 else goto L7 :: bool +L7: + r11 = raise IndexError + unreachable +L8: + r12 = r9 + goto L10 +L9: + r12 = n +L10: + r13 = vec_get_item_unsafe[i64] r6, r12 + keep_alive v, r5 + return r13 + +[case testVecNestedStrGetItemWithBorrow_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +class C: + v: vec[vec[str]] + + def f(self, n: i64) -> str: + # The intermediate vec is borrowed, but the result must be owned. + return self.v[n][n] +[out] +def C.f(self, n): + self :: __main__.C + n :: i64 + r0 :: vec[vec[str]] + r1 :: native_int + r2 :: bit + r3 :: i64 + r4 :: bit + r5 :: bool + r6 :: i64 + r7 :: vec[str] + r8 :: native_int + r9 :: bit + r10 :: i64 + r11 :: bit + r12 :: bool + r13 :: i64 + r14 :: str +L0: + r0 = borrow self.v + r1 = r0.len + r2 = n < r1 :: unsigned + if r2 goto L4 else goto L1 :: bool +L1: + r3 = n + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L3 else goto L2 :: bool +L2: + r5 = raise IndexError + unreachable +L3: + r6 = r3 + goto L5 +L4: + r6 = n +L5: + r7 = vec_get_item_unsafe_borrow[vec[str]] r0, r6 + r8 = r7.len + r9 = n < r8 :: unsigned + if r9 goto L9 else goto L6 :: bool L6: - r12 = n + r10 - r13 = r12 < r10 :: unsigned - if r13 goto L8 else goto L7 :: bool + r10 = n + r8 + r11 = r10 < r8 :: unsigned + if r11 goto L8 else goto L7 :: bool L7: - r14 = raise IndexError + r12 = raise IndexError unreachable L8: - r15 = r12 + r13 = r10 goto L10 L9: - r15 = n + r13 = n L10: - r16 = r9.items - r17 = r15 * 8 - r18 = r16 + r17 - r19 = load_mem r18 :: i64* - keep_alive v, r9 - return r19 + r14 = vec_get_item_unsafe[str] r7, r13 + keep_alive self, r0, r6 + return r14 [case testVecDoublyNestedGetItem_64bit] from librt.vecs import vec @@ -368,10 +409,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[vec[str]] + r6 :: vec[vec[str]] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -389,12 +427,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = load_mem r8 :: vec[vec[str]]* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[vec[vec[str]]] v, r5 + return r6 [case testVecNestedCreateWithCap_64bit] from librt.vecs import vec diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 63ad14bc2d7a2..ce162bd0806c7 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -215,10 +215,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: str + r6 :: str L0: r0 = v.len r1 = n < r0 :: unsigned @@ -236,12 +233,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.str* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[str] v, r5 + return r6 [case testVecTOptionalGetItem_64bit] from librt.vecs import vec @@ -260,10 +253,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: union[str, None] + r6 :: union[str, None] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -281,12 +271,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: union* - keep_alive v - return r9 + r6 = vec_get_item_unsafe[union[str, None]] v, r5 + return r6 [case testNewTPopLast] from typing import Tuple @@ -524,3 +510,62 @@ L0: r1 = r0 r2 = VecTApi.from_iterable(r1, a, 0) return r2 + +[case testVecTBorrowGetItem_64bit] +from typing import Final + +from librt.vecs import vec +from mypy_extensions import i64 + +class A: + def __init__(self, x: str) -> None: + # Final to allow borrowing on free-threaded builds + self.x: Final = x + +def f(v: vec[A], n: i64) -> int: + return len(v[n].x) +[out] +def A.__init__(self, x): + self :: __main__.A + x :: str +L0: + self.x = x + return 1 +def f(v, n): + v :: vec[__main__.A] + n :: i64 + r0 :: native_int + r1 :: bit + r2 :: i64 + r3 :: bit + r4 :: bool + r5 :: i64 + r6 :: __main__.A + r7 :: str + r8 :: native_int + r9 :: bit + r10 :: short_int +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L4 else goto L1 :: bool +L1: + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool +L2: + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = vec_get_item_unsafe_borrow[__main__.A] v, r5 + r7 = r6.x + keep_alive v, r5 + r8 = CPyStr_Size_size_t(r7) + r9 = r8 >= 0 :: signed + r10 = r8 << 1 + return r10 diff --git a/mypyc/test-data/lowering-int.test b/mypyc/test-data/lowering-int.test index c2bcba54e444d..72c4a82dcaea3 100644 --- a/mypyc/test-data/lowering-int.test +++ b/mypyc/test-data/lowering-int.test @@ -332,7 +332,7 @@ L4: L5: return 4 -[case testLowerIntForLoop_64bit] +[case testLowerIntForLoop_withgil_64bit] from __future__ import annotations def f(l: list[int]) -> None: diff --git a/mypyc/test-data/lowering-vec.test b/mypyc/test-data/lowering-vec.test new file mode 100644 index 0000000000000..374d336b98d8e --- /dev/null +++ b/mypyc/test-data/lowering-vec.test @@ -0,0 +1,155 @@ +[case testLowerVecI64GetItem_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(i: i64) -> i64: + v = vec[i64]() + return v[i] +[out] +def f(i): + i :: i64 + r0, v :: vec[i64] + r1 :: native_int + r2 :: bit + r3 :: i64 + r4 :: bit + r5 :: bool + r6 :: i64 + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10, r11 :: i64 +L0: + r0 = VecI64Api.alloc(0, 0) + if is_error(r0) goto L8 (error at f:5) else goto L1 +L1: + v = r0 + r1 = v.len + r2 = i < r1 :: unsigned + if r2 goto L6 else goto L2 :: bool +L2: + r3 = i + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L5 else goto L9 :: bool +L3: + r5 = raise IndexError + if not r5 goto L8 (error at f:6) else goto L4 :: bool +L4: + unreachable +L5: + r6 = r3 + goto L7 +L6: + r6 = i +L7: + r7 = v.items + r8 = r6 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: i64* + dec_ref v + return r10 +L8: + r11 = :: i64 + return r11 +L9: + dec_ref v + goto L3 + +[case testLowerVecNestedGetItem_64bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(i: i64, j: i64) -> str: + v = vec[vec[str]]([]) + return v[i][j] +[out] +def f(i, j): + i, j :: i64 + r0 :: object + r1 :: ptr + r2 :: vec[vec[str]] + r3 :: ptr + v :: vec[vec[str]] + r4 :: native_int + r5 :: bit + r6 :: i64 + r7 :: bit + r8 :: bool + r9 :: i64 + r10 :: ptr + r11 :: i64 + r12 :: ptr + r13 :: vec[str] + r14 :: native_int + r15 :: bit + r16 :: i64 + r17 :: bit + r18 :: bool + r19 :: i64 + r20 :: ptr + r21 :: i64 + r22 :: ptr + r23, r24 :: str +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecNestedApi.alloc(0, 0, r1, 1) + if is_error(r2) goto L14 (error at f:5) else goto L1 +L1: + r3 = r2.items + v = r2 + r4 = v.len + r5 = i < r4 :: unsigned + if r5 goto L6 else goto L2 :: bool +L2: + r6 = i + r4 + r7 = r6 < r4 :: unsigned + if r7 goto L5 else goto L15 :: bool +L3: + r8 = raise IndexError + if not r8 goto L14 (error at f:6) else goto L4 :: bool +L4: + unreachable +L5: + r9 = r6 + goto L7 +L6: + r9 = i +L7: + r10 = v.items + r11 = r9 * 16 + r12 = r10 + r11 + r13 = borrow load_mem r12 :: vec[str]* + r14 = r13.len + r15 = j < r14 :: unsigned + if r15 goto L12 else goto L8 :: bool +L8: + r16 = j + r14 + r17 = r16 < r14 :: unsigned + if r17 goto L11 else goto L16 :: bool +L9: + r18 = raise IndexError + if not r18 goto L14 (error at f:6) else goto L10 :: bool +L10: + unreachable +L11: + r19 = r16 + goto L13 +L12: + r19 = j +L13: + r20 = r13.items + r21 = r19 * 8 + r22 = r20 + r21 + r23 = load_mem r22 :: builtins.str* + dec_ref v + return r23 +L14: + r24 = :: str + return r24 +L15: + dec_ref v + goto L3 +L16: + dec_ref v + goto L9 diff --git a/mypyc/test-data/opt-copy-propagation.test b/mypyc/test-data/opt-copy-propagation.test index 49b80f4385fc4..9fa5edb579fac 100644 --- a/mypyc/test-data/opt-copy-propagation.test +++ b/mypyc/test-data/opt-copy-propagation.test @@ -185,7 +185,7 @@ L1: L2: return 2 -[case testIRTransformRegisterOps1] +[case testIRTransformRegisterOps1_withgil] from __future__ import annotations from typing import cast diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 918c84ee3b0ad..bedfabac791d4 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -592,6 +592,28 @@ L0: dec_ref d return 1 +[case testBorrowListObjectWithLiteralIndex_nogil] +from typing import cast + +def f() -> str: + o = cast(object, []) + return cast(list[str], o)[0] +[out] +def f(): + r0 :: list + o :: object + r1 :: list + r2 :: object + r3 :: str +L0: + r0 = PyList_New(0) + o = r0 + r1 = borrow cast(list, o) + r2 = CPyList_GetItemShort(r1, 0) + r3 = cast(str, r2) + dec_ref o + return r3 + [case testUnaryBranchSpecialCase] def f(x: bool) -> int: if x: @@ -629,7 +651,7 @@ L0: r4 = (r2, r3) return r4 -[case testDecomposeTuple] +[case testDecomposeTuple_withgil] from typing import Tuple class C: @@ -948,7 +970,7 @@ L0: dec_ref r4 return r5 -[case testBorrowAttribute] +[case testBorrowAttribute_withgil] def g() -> int: d = D() return d.c.x @@ -981,7 +1003,99 @@ L0: r1 = r0.x return r1 -[case testBorrowAttributeTwice] +[case testBorrowAttribute_nogil] +from typing import Final + +def g() -> int: + d = D(C(1)) + return d.c.x + +def f(d: D) -> int: + return d.c.x + +class C: + def __init__(self, x: int) -> None: + self.x: Final = x +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def g(): + r0 :: __main__.C + r1, d :: __main__.D + r2 :: __main__.C + r3 :: int +L0: + r0 = C(2) + r1 = D(r0) + dec_ref r0 + d = r1 + r2 = borrow d.c + r3 = r2.x + dec_ref d + return r3 +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + return r1 +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + inc_ref c + self.c = c + return 1 + +[case testBorrowInheritedFinalAttribute_nogil] +from typing import Final + +def f(d: D) -> int: + return d.c.x + +class B: + def __init__(self, x: int) -> None: + self.x: Final = x +class C(B): + pass +class D: + def __init__(self, c: C) -> None: + self.c: Final = c +[out] +def f(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int +L0: + r0 = borrow d.c + r1 = r0.x + return r1 +def B.__init__(self, x): + self :: __main__.B + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def D.__init__(self, c): + self :: __main__.D + c :: __main__.C +L0: + inc_ref c + self.c = c + return 1 + +[case testBorrowAttributeTwice_withgil] def f(e: E) -> int: return e.d.c.x @@ -1003,7 +1117,7 @@ L0: r2 = r1.x return r2 -[case testBorrowAttributeIsNone] +[case testBorrowAttributeIsNone_withgil] from typing import Optional def f(c: C) -> bool: @@ -1036,7 +1150,7 @@ L0: r2 = r0 == r1 return r2 -[case testBorrowAttributeNarrowOptional] +[case testBorrowAttributeNarrowOptional_withgil] from typing import Optional def f(c: C) -> bool: @@ -1071,7 +1185,7 @@ L1: L2: return 0 -[case testBorrowLenArgument] +[case testBorrowLenArgument_withgil] from typing import List def f(x: C) -> int: @@ -1091,7 +1205,7 @@ L0: r2 = r1 << 1 return r2 -[case testBorrowIsinstanceArgument] +[case testBorrowIsinstanceArgument_withgil] from typing import List def f(x: C) -> bool: @@ -1130,7 +1244,7 @@ L1: L2: return 1 -[case testBorrowListGetItem1] +[case testBorrowListGetItem1_withgil] from typing import List def literal_index(x: C) -> str: @@ -1178,7 +1292,58 @@ L0: r2 = cast(str, r1) return r2 -[case testBorrowListGetItem2] +[case testBorrowListGetItem1_nogil] +from typing import List + +def literal_index(x: C) -> str: + return x.a[0] + +def negative_index(x: C) -> str: + return x.a[-1] + +def lvar_index(x: C, n: int) -> str: + return x.a[n] + +class C: + a: List[str] + +[out] +def literal_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItemShort(r0, 0) + dec_ref r0 + r2 = cast(str, r1) + return r2 +def negative_index(x): + x :: __main__.C + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItemShort(r0, -2) + dec_ref r0 + r2 = cast(str, r1) + return r2 +def lvar_index(x, n): + x :: __main__.C + n :: int + r0 :: list + r1 :: object + r2 :: str +L0: + r0 = x.a + r1 = CPyList_GetItem(r0, n) + dec_ref r0 + r2 = cast(str, r1) + return r2 + +[case testBorrowListGetItem2_withgil] from typing import List def attr_before_index(x: C) -> str: @@ -1228,6 +1393,60 @@ L0: r2 = r1.n return r2 +[case testBorrowListGetItem2_nogil] +from typing import List + +def attr_before_index(x: C) -> str: + return x.a[x.n] + +def attr_after_index(a: List[C], i: int) -> int: + return a[i].n + +def attr_after_index_literal(a: List[C]) -> int: + return a[0].n + +class C: + a: List[str] + n: int +[out] +def attr_before_index(x): + x :: __main__.C + r0 :: list + r1 :: int + r2 :: object + r3 :: str +L0: + r0 = x.a + r1 = x.n + r2 = CPyList_GetItem(r0, r1) + dec_ref r0 + dec_ref r1 :: int + r3 = cast(str, r2) + return r3 +def attr_after_index(a, i): + a :: list + i :: int + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItem(a, i) + r1 = cast(__main__.C, r0) + r2 = r1.n + dec_ref r1 + return r2 +def attr_after_index_literal(a): + a :: list + r0 :: object + r1 :: __main__.C + r2 :: int +L0: + r0 = CPyList_GetItemShort(a, 0) + r1 = cast(__main__.C, r0) + r2 = r1.n + dec_ref r1 + return r2 + [case testCannotBorrowListGetItem] from typing import List @@ -1257,7 +1476,7 @@ def f(): L0: return 0 -[case testBorrowListGetItemKeepAlive] +[case testBorrowListGetItemKeepAlive_withgil] from typing import List def f() -> str: @@ -1287,7 +1506,7 @@ L0: dec_ref a return r5 -[case testBorrowSetAttrObject] +[case testBorrowSetAttrObject_withgil] from typing import Optional def f(x: Optional[C]) -> None: @@ -1327,7 +1546,7 @@ L0: r0.b = 0; r1 = is_error return 1 -[case testBorrowIntEquality] +[case testBorrowIntEquality_withgil] def add(c: C) -> bool: return c.x == c.y @@ -1345,7 +1564,7 @@ L0: r2 = int_eq r0, r1 return r2 -[case testBorrowIntLessThan] +[case testBorrowIntLessThan_withgil] def add(c: C) -> bool: return c.x < c.y @@ -1363,7 +1582,7 @@ L0: r2 = int_lt r0, r1 return r2 -[case testBorrowIntCompareFinal] +[case testBorrowIntCompareFinal_withgil] from typing import Final X: Final = 10 @@ -1383,7 +1602,7 @@ L0: r1 = int_eq r0, 20 return r1 -[case testBorrowIntArithmetic] +[case testBorrowIntArithmetic_withgil] def add(c: C) -> int: return c.x + c.y @@ -1411,7 +1630,39 @@ L0: r2 = CPyTagged_Subtract(r0, r1) return r2 -[case testBorrowIntComparisonInIf] +[case testBorrowIntArithmetic_nogil] +def add(c: C) -> int: + return c.x + c.y + +def sub(c: C) -> int: + return c.x - c.y + +class C: + x: int + y: int +[out] +def add(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = c.x + r1 = c.y + r2 = CPyTagged_Add(r0, r1) + dec_ref r0 :: int + dec_ref r1 :: int + return r2 +def sub(c): + c :: __main__.C + r0, r1, r2 :: int +L0: + r0 = c.x + r1 = c.y + r2 = CPyTagged_Subtract(r0, r1) + dec_ref r0 :: int + dec_ref r1 :: int + return r2 + +[case testBorrowIntComparisonInIf_withgil] def add(c: C, n: int) -> bool: if c.x == c.y: return True @@ -1435,7 +1686,7 @@ L1: L2: return 0 -[case testBorrowIntInPlaceOp] +[case testBorrowIntInPlaceOp_withgil] def add(c: C, n: int) -> None: c.x += n @@ -1606,12 +1857,9 @@ def f(v): t :: i64 r0, r1 :: native_int r2 :: bit - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6, s :: str - r7 :: None - r8 :: native_int + r3, s :: str + r4 :: None + r5 :: native_int L0: t = 0 r0 = 0 @@ -1620,16 +1868,13 @@ L1: r2 = r0 < r1 :: signed if r2 goto L2 else goto L4 :: bool L2: - r3 = v.items - r4 = r0 * 8 - r5 = r3 + r4 - r6 = load_mem r5 :: builtins.str* - s = r6 - r7 = g(s) + r3 = vec_get_item_unsafe[str] v, r0 + s = r3 + r4 = g(s) dec_ref s L3: - r8 = r0 + 1 - r0 = r8 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return t @@ -1684,11 +1929,7 @@ def C.f(self, x): r3 :: i64 r4 :: bit r5 :: bool - r6 :: i64 - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: i64 + r6, r7 :: i64 L0: r0 = borrow self.v r1 = r0.len @@ -1707,11 +1948,8 @@ L3: L4: r6 = x L5: - r7 = r0.items - r8 = r6 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: i64* - return r10 + r7 = vec_get_item_unsafe[i64] r0, r6 + return r7 [case testVecI64LenBorrowVec_64bit] from librt.vecs import vec @@ -1771,10 +2009,7 @@ def f(v, n): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: str + r6 :: str L0: r0 = v.len r1 = n < r0 :: unsigned @@ -1792,11 +2027,8 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 8 - r8 = r6 + r7 - r9 = load_mem r8 :: builtins.str* - return r9 + r6 = vec_get_item_unsafe[str] v, r5 + return r6 [case testVecNestedGetItem_64bit] from librt.vecs import vec @@ -1817,10 +2049,7 @@ def f(v, n): r6 :: bit r7 :: bool r8 :: i64 - r9 :: ptr - r10 :: i64 - r11 :: ptr - r12, vv :: vec[str] + r9, vv :: vec[str] L0: r0 = load_address PyUnicode_Type r1 = r0 @@ -1841,12 +2070,9 @@ L3: L4: r8 = n L5: - r9 = r2.items - r10 = r8 * 16 - r11 = r9 + r10 - r12 = load_mem r11 :: vec[str]* + r9 = vec_get_item_unsafe[vec[str]] r2, r8 dec_ref r2 - vv = r12 + vv = r9 dec_ref vv return 1 L6: @@ -1869,20 +2095,13 @@ def f(v, n, m): r3 :: bit r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 - r8 :: ptr - r9 :: vec[i64] - r10 :: native_int - r11 :: bit - r12 :: i64 - r13 :: bit - r14 :: bool - r15 :: i64 - r16 :: ptr - r17 :: i64 - r18 :: ptr - r19 :: i64 + r6 :: vec[i64] + r7 :: native_int + r8 :: bit + r9 :: i64 + r10 :: bit + r11 :: bool + r12, r13 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned @@ -1900,31 +2119,25 @@ L3: L4: r5 = n L5: - r6 = v.items - r7 = r5 * 16 - r8 = r6 + r7 - r9 = borrow load_mem r8 :: vec[i64]* - r10 = r9.len - r11 = m < r10 :: unsigned - if r11 goto L9 else goto L6 :: bool + r6 = vec_get_item_unsafe_borrow[vec[i64]] v, r5 + r7 = r6.len + r8 = m < r7 :: unsigned + if r8 goto L9 else goto L6 :: bool L6: - r12 = m + r10 - r13 = r12 < r10 :: unsigned - if r13 goto L8 else goto L7 :: bool + r9 = m + r7 + r10 = r9 < r7 :: unsigned + if r10 goto L8 else goto L7 :: bool L7: - r14 = raise IndexError + r11 = raise IndexError unreachable L8: - r15 = r12 + r12 = r9 goto L10 L9: - r15 = m + r12 = m L10: - r16 = r9.items - r17 = r15 * 8 - r18 = r16 + r17 - r19 = load_mem r18 :: i64* - return r19 + r13 = vec_get_item_unsafe[i64] r6, r12 + return r13 [case testVecPop] from librt.vecs import vec, pop, append @@ -2153,3 +2366,717 @@ L0: r3 = CPyBytes_Concat(r2, c) dec_ref r2 return r3 + +[case testBorrowedFinalAttribute] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, b: bool) -> str: + a = c.s if b else "x" + return a + +def g(c: C) -> C: + return C(c.s) +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, b): + c :: __main__.C + b :: bool + r0, r1, r2, a :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = borrow c.s + inc_ref r0 + r1 = r0 + goto L3 +L2: + r2 = 'x' + inc_ref r2 + r1 = r2 +L3: + a = r1 + return a +def g(c): + c :: __main__.C + r0 :: str + r1 :: __main__.C +L0: + r0 = borrow c.s + r1 = C(r0) + return r1 + +[case testBorrowedFinalAttribute2_withgil] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: int) -> str: + a = [c] + return a[n].s + "a" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: int + r0 :: list + r1 :: ptr + a :: list + r2 :: object + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = PyList_New(1) + r1 = list_items r0 + inc_ref c + buf_init_item r1, 0, c + a = r0 + r2 = CPyList_GetItemBorrow(a, n) + r3 = borrow cast(__main__.C, r2) + r4 = r3.s + dec_ref a + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 + +[case testBorrowedFinalAttribute3_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d = d + +class CC: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.f: Final = s + +def f() -> str: + c = C(D("x")) + return c.d.f + "a" + +def g(d: D) -> str: + c = CC(d) + return c.d.f + "a" +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def CC.__init__(self, d): + self :: __main__.CC + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.f = s + return 1 +def f(): + r0 :: str + r1 :: __main__.D + r2, c :: __main__.C + r3 :: __main__.D + r4, r5, r6 :: str +L0: + r0 = 'x' + r1 = D(r0) + r2 = C(r1) + dec_ref r1 + c = r2 + r3 = borrow c.d + r4 = r3.f + dec_ref c + r5 = 'a' + r6 = PyUnicode_Concat(r4, r5) + dec_ref r4 + return r6 +def g(d): + d :: __main__.D + r0, c :: __main__.CC + r1 :: __main__.D + r2, r3, r4 :: str +L0: + r0 = CC(d) + c = r0 + r1 = borrow c.d + r2 = borrow r1.f + r3 = 'a' + r4 = PyUnicode_Concat(r2, r3) + dec_ref c + return r4 + +[case testBorrowedFinalAttribute4] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + + def method(self) -> int: + return self.d.foo() + +class D: + def foo(self) -> int: + return 1 +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def C.method(self): + self :: __main__.C + r0 :: __main__.D + r1 :: int +L0: + r0 = borrow self.d + r1 = r0.foo() + return r1 +def D.foo(self): + self :: __main__.D +L0: + return 2 + +[case testBorrowedFinalAttribute5_64bit] +from typing import Final +from librt.vecs import vec +from mypy_extensions import i64 + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def g(c: C, n: i64) -> str: + a = vec[C]([c]) + return a[n].s + "a" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def g(c, n): + c :: __main__.C + n :: i64 + r0 :: object + r1 :: ptr + r2 :: vec[__main__.C] + r3, r4 :: ptr + a :: vec[__main__.C] + r5 :: native_int + r6 :: bit + r7 :: i64 + r8 :: bit + r9 :: bool + r10 :: i64 + r11 :: __main__.C + r12, r13, r14 :: str +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = VecTApi.alloc(1, 1, r1) + r3 = r2.items + inc_ref c + set_mem r3, c :: __main__.C* + r4 = r3 + 8 + a = r2 + r5 = a.len + r6 = n < r5 :: unsigned + if r6 goto L4 else goto L1 :: bool +L1: + r7 = n + r5 + r8 = r7 < r5 :: unsigned + if r8 goto L3 else goto L6 :: bool +L2: + r9 = raise IndexError + unreachable +L3: + r10 = r7 + goto L5 +L4: + r10 = n +L5: + r11 = vec_get_item_unsafe_borrow[__main__.C] a, r10 + r12 = r11.s + dec_ref a + r13 = 'a' + r14 = PyUnicode_Concat(r12, r13) + dec_ref r12 + return r14 +L6: + dec_ref a + goto L2 + +[case testBorrowedFinalAttributeReassignViaWalrus] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C, d: C) -> str: + return c.s + (c := d).s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(c, d): + c, d :: __main__.C + r0, r1, r2 :: str +L0: + r0 = c.s + inc_ref d + c = d + dec_ref c + r1 = borrow d.s + r2 = PyUnicode_Concat(r0, r1) + dec_ref r0 + return r2 + +[case testBorrowedFinalAttributeConditionalMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(b: bool) -> str: + return (C("x").s if b else "y") + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(b): + b :: bool + r0 :: str + r1 :: __main__.C + r2, r3, r4, r5, r6 :: str +L0: + if b goto L1 else goto L2 :: bool +L1: + r0 = 'x' + r1 = C(r0) + r2 = borrow r1.s + inc_ref r2 + r3 = r2 + dec_ref r1 + goto L3 +L2: + r4 = 'y' + inc_ref r4 + r3 = r4 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r3, r5) + dec_ref r3 + return r6 + +[case testBorrowedFinalAttributeAndMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s and C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L2 else goto L1 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeOrMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> str: + return (s or C("x").s) + "z" +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s :: str + r0 :: bit + r1, r2 :: str + r3 :: __main__.C + r4, r5, r6 :: str +L0: + r0 = CPyStr_IsTrue(s) + if r0 goto L1 else goto L2 :: bool +L1: + inc_ref s + r1 = s + goto L3 +L2: + r2 = 'x' + r3 = C(r2) + r4 = borrow r3.s + inc_ref r4 + r1 = r4 + dec_ref r3 +L3: + r5 = 'z' + r6 = PyUnicode_Concat(r1, r5) + dec_ref r1 + return r6 + +[case testBorrowedFinalAttributeChainedComparisonMerge] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(s: str) -> bool: + return s == "x" == C("x").s +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(s): + s, r0 :: str + r1, r2 :: bool + r3 :: str + r4 :: __main__.C + r5 :: str + r6 :: bool +L0: + r0 = 'x' + r1 = CPyStr_EqualLiteral(s, r0, 1) + if r1 goto L2 else goto L1 :: bool +L1: + r2 = r1 + goto L3 +L2: + r3 = 'x' + r4 = C(r3) + r5 = borrow r4.s + r6 = CPyStr_EqualLiteral(r5, r0, 1) + r2 = r6 + dec_ref r4 +L3: + return r2 + +[case testBorrowedFinalInListComprehension_withgil] +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(a: list[C]) -> list[str]: + return [x.s for x in a] +[out] +def C.__init__(self, s): + self :: __main__.C + s :: str +L0: + inc_ref s + self.s = s + return 1 +def f(a): + a :: list + r0 :: native_int + r1 :: list + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x :: __main__.C + r7 :: str + r8 :: native_int +L0: + r0 = var_object_size a + r1 = PyList_New(r0) + r2 = 0 +L1: + r3 = var_object_size a + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = list_get_item_unsafe a, r2 + r6 = cast(__main__.C, r5) + x = r6 + r7 = x.s + dec_ref x + CPyList_SetItemUnsafe(r1, r2, r7) +L3: + r8 = r2 + 1 + r2 = r8 + goto L1 +L4: + return r1 + +[case testBorrowedFinalInSetComprehension_withgil] +from typing import Final + +class C: + def __init__(self, d: D) -> None: + self.d: Final = d + +class D: + def __init__(self, s: str) -> None: + self.s: Final = s + +def use(s: str) -> bool: + return True + +def f(a: list[C]) -> set[C]: + return {x for x in a if use(x.d.s)} +[out] +def C.__init__(self, d): + self :: __main__.C + d :: __main__.D +L0: + inc_ref d + self.d = d + return 1 +def D.__init__(self, s): + self :: __main__.D + s :: str +L0: + inc_ref s + self.s = s + return 1 +def use(s): + s :: str +L0: + return 1 +def f(a): + a :: list + r0 :: set + r1, r2 :: native_int + r3 :: bit + r4 :: object + r5, x :: __main__.C + r6 :: __main__.D + r7 :: str + r8 :: bool + r9 :: i32 + r10 :: bit + r11 :: native_int +L0: + r0 = PySet_New(0) + r1 = 0 +L1: + r2 = var_object_size a + r3 = r1 < r2 :: signed + if r3 goto L2 else goto L5 :: bool +L2: + r4 = list_get_item_unsafe a, r1 + r5 = cast(__main__.C, r4) + x = r5 + r6 = borrow x.d + r7 = borrow r6.s + r8 = use(r7) + if r8 goto L3 else goto L6 :: bool +L3: + r9 = PySet_Add(r0, x) + dec_ref x + r10 = r9 >= 0 :: signed +L4: + r11 = r1 + 1 + r1 = r11 + goto L1 +L5: + return r0 +L6: + dec_ref x + goto L4 + +[case testBorrowedFinalAttributeThroughCast] +from typing import Final, cast + +class C: + def __init__(self, x: object) -> None: + self.x: Final = x + +def make() -> object: + return C(1) + +def g(a: object) -> object: + return a + +def h() -> object: + return g(cast(C, make()).x) +[out] +def C.__init__(self, x): + self :: __main__.C + x :: object +L0: + inc_ref x + self.x = x + return 1 +def make(): + r0 :: object + r1 :: __main__.C +L0: + r0 = object 1 + r1 = C(r0) + return r1 +def g(a): + a :: object +L0: + inc_ref a + return a +def h(): + r0 :: object + r1 :: __main__.C + r2, r3 :: object +L0: + r0 = make() + r1 = borrow cast(__main__.C, r0) + r2 = borrow r1.x + r3 = g(r2) + dec_ref r0 + return r3 + +[case testBorrowedFinalAttributeCastChain] +from typing import Final, cast + +class Inner: + def __init__(self, x: object) -> None: + self.x: Final = x + +class Outer: + def __init__(self, inner: object) -> None: + self.inner: Final = inner + +def g(a: object) -> object: + return a + +def make() -> object: + return Outer(Inner(1)) + +def h() -> object: + # cast( Inner, ).x + o = cast(Outer, make()) + return g(cast(Inner, o.inner).x) +[out] +def Inner.__init__(self, x): + self :: __main__.Inner + x :: object +L0: + inc_ref x + self.x = x + return 1 +def Outer.__init__(self, inner): + self :: __main__.Outer + inner :: object +L0: + inc_ref inner + self.inner = inner + return 1 +def g(a): + a :: object +L0: + inc_ref a + return a +def make(): + r0 :: object + r1 :: __main__.Inner + r2 :: __main__.Outer +L0: + r0 = object 1 + r1 = Inner(r0) + r2 = Outer(r1) + dec_ref r1 + return r2 +def h(): + r0 :: object + r1, o :: __main__.Outer + r2 :: object + r3 :: __main__.Inner + r4, r5 :: object +L0: + r0 = make() + r1 = cast(__main__.Outer, r0) + o = r1 + r2 = borrow o.inner + r3 = borrow cast(__main__.Inner, r2) + r4 = borrow r3.x + r5 = g(r4) + dec_ref o + return r5 diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 2733a31f3af2a..4c9fca54f4a2f 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -407,6 +407,12 @@ import asyncio import gc import platform +from testutil import is_gil_disabled + +def immediate_deallocation_checks_are_reliable() -> bool: + # Free-threaded builds can defer deallocation, so weakref liveness is not deterministic. + return not is_gil_disabled() + async def sleep() -> None: # Non-zero value hangs on Windows. time = 0 if platform.system() == "Windows" else 0.0001 @@ -512,6 +518,106 @@ async def stolen(n: int) -> int: async def test_stolen() -> None: await assert_no_leaks(lambda: stolen(200), 16) +async def retain_arg(c: set[int]) -> None: + async def nested() -> int: + await sleep() + return len(c) + + await nested() + +async def test_completed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + + c = {1} + ref = weakref.ref(c) + coro = retain_arg(c) + c = {2} + await coro + assert ref() is None + +async def retain_arg_then_raise(c: set[int]) -> None: + async def nested() -> int: + await sleep() + return len(c) + + await nested() + raise ValueError + +async def test_failed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + + c = {1} + ref = weakref.ref(c) + coro = retain_arg_then_raise(c) + c = {2} + try: + await coro + except ValueError: + pass + assert ref() is None + +async def close_retain_arg(c: set[int]) -> None: + await sleep() + len(c) + +async def test_closed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import weakref + from typing import Any, cast + + c = {1} + ref = weakref.ref(c) + coro = cast(Any, close_retain_arg(c)) + coro.send(None) + c = {2} + coro.close() + assert ref() is None + +async def interpreted_leaf(c: set[int]) -> int: + await sleep() + return len(c) + +async def compiled_leaf(c: set[int]) -> int: + await sleep() + return len(c) + +async def compiled_awaits_interpreted(c: set[int]) -> int: + import interpreted + + return await interpreted.interpreted_leaf(c) + +async def interpreted_awaits_compiled(fn, c: set[int]) -> int: + return await fn(c) + +async def test_completed_mixed_coroutine_releases_environment() -> None: + if not immediate_deallocation_checks_are_reliable(): + return + + import interpreted + import weakref + + c = {1} + ref = weakref.ref(c) + coro = compiled_awaits_interpreted(c) + c = {2} + assert await coro == 1 + assert ref() is None + + c = {3} + ref = weakref.ref(c) + coro = interpreted.interpreted_awaits_compiled(compiled_leaf, c) + c = {4} + assert await coro == 1 + assert ref() is None + [file asyncio/__init__.pyi] async def sleep(t: float) -> None: ... @@ -1416,6 +1522,7 @@ def run(x: object) -> object: ... [case testAsyncIntrospection] import asyncio +import gc import inspect import sys import weakref @@ -1423,6 +1530,8 @@ import weakref from functools import wraps from typing import Any, Callable, TypeVar, cast +from testutil import is_gil_disabled + def identity(val: int) -> int: return val @@ -1589,6 +1698,37 @@ def test_nested() -> None: assert is_coroutine(nested_wrapped_async) assert asyncio.run(nested_wrapped_async()) == 4 +def test_async_function_wrapper_code_refcount() -> None: + if is_gil_disabled(): + # On free-threaded builds the code object might be immortal, so the ref count test doesn't work. + return + code = getattr(identity_async, "__code__") + getrefcount = getattr(sys, "getrefcount") + # getrefcount sees the local code variable plus the wrapper-owned reference. + assert getrefcount(code) == 2, getrefcount(code) + +def test_nested_async_function_wrapper_code_refcount() -> None: + if is_gil_disabled(): + # On free-threaded builds the code object might be immortal, so the ref count test doesn't work. + return + def make_nested() -> Any: + async def nested_refcounted() -> int: + return 1 + + return nested_refcounted + + getrefcount = getattr(sys, "getrefcount") + fn = make_nested() + code = getattr(fn, "__code__") + before = getrefcount(code) + assert asyncio.run(fn()) == 1 + + del fn + gc.collect() + after = getrefcount(code) + assert before == after + 1, (before, after) + assert after == 1, after + [file asyncio/__init__.pyi] def run(x: object) -> object: ... @@ -1940,3 +2080,189 @@ async def async_iter(vals: list[int]) -> AsyncIterator[int]: yield v [typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwait] +from typing import Final +from testutil import async_val + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the await, which is a + # suspension point in the middle of the expression. + return c.s + await async_val("hello") + +[file driver.py] +from native import f, C +from testutil import run_generator + +for i in range(50): + c = C("value" + str(i)) + yields, val = run_generator(f(c), inputs=["world"]) + assert yields == ("hello",), yields + assert val == "value" + str(i) + "world", repr(val) + +[case testCoroutineClosureOutlivesCompletedCoroutine] +import asyncio +from typing import Callable + +async def make_reader() -> Callable[[], int]: + payload = [7] + await asyncio.sleep(0) + + def read_payload() -> int: + return payload[0] + + return read_payload + +async def make_arg_reader(payload: list[int]) -> Callable[[], int]: + await asyncio.sleep(0) + + def read_payload() -> int: + return payload[0] + + return read_payload + +async def make_indirect_reader() -> Callable[[], int]: + def leaf() -> int: + return 17 + + def reader() -> int: + return leaf() + + await asyncio.sleep(0) + return reader + +def test_closure_returned_from_completed_coroutine_keeps_capture() -> None: + read_payload = asyncio.run(make_reader()) + assert read_payload() == 7 + + payload = [13] + read_arg_payload = asyncio.run(make_arg_reader(payload)) + payload = [0] + assert read_arg_payload() == 13 + + indirect_reader = asyncio.run(make_indirect_reader()) + assert indirect_reader() == 17 + +async def test_closure_returned_from_awaited_coroutine_keeps_capture() -> None: + read_payload = await make_reader() + assert read_payload() == 7 + + payload = [13] + read_arg_payload = await make_arg_reader(payload) + payload = [0] + assert read_arg_payload() == 13 + + indirect_reader = await make_indirect_reader() + assert indirect_reader() == 17 + +[file asyncio/__init__.pyi] +from typing import Awaitable, TypeVar + +T = TypeVar("T") + +def run(x: Awaitable[T]) -> T: ... +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[case testNestedCoroutineOutlivesCompletedCoroutine] +import asyncio +from typing import Awaitable + +async def make_nested_coroutine() -> Awaitable[int]: + payload = [11] + await asyncio.sleep(0) + + async def read_payload() -> int: + await asyncio.sleep(0) + return payload[0] + + return read_payload() + +def test_nested_coroutine_returned_from_completed_coroutine_keeps_capture() -> None: + read_payload = asyncio.run(make_nested_coroutine()) + assert asyncio.run(read_payload) == 11 + +[file asyncio/__init__.pyi] +from typing import Awaitable, TypeVar + +T = TypeVar("T") + +def run(x: Awaitable[T]) -> T: ... +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAsyncComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +class AsyncRange: + def __init__(self, n: int) -> None: + self.n = n + self.i = 0 + + def __aiter__(self) -> "AsyncRange": + return self + + async def __anext__(self) -> str: + if self.i >= self.n: + raise StopAsyncIteration + i = self.i + self.i += 1 + # Suspend in the middle of iteration. + await asyncio.sleep(0) + return "item" + str(i) + +async def f(c: C) -> str: + # c.s is a final attr; it must not be borrowed across the async comprehension, + # which suspends via an implicit await on __anext__ that isn't represented as + # an AwaitExpr node in the AST. + return c.s + "".join([x async for x in AsyncRange(3)]) + +async def test_async_comprehension_borrow() -> None: + for i in range(50): + c = C("value" + str(i)) + assert await f(c) == "value" + str(i) + "item0item1item2" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] + +[case testBorrowedFinalAttrAcrossAwaitAfterComprehension] +import asyncio +from typing import Final + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +async def sink() -> str: + await asyncio.sleep(0) + return "z" + +async def f(c: C, xs: list[int]) -> str: + # A (sync) comprehension enters a nested borrow scope earlier in the same + # top-level expression. It must not clear the enclosing expression's suspend + # flag, or c.s (a final attr) gets borrowed across the await below, which + # produces invalid C (the borrow does not survive the suspension point). + return str([x for x in xs]) + (c.s + await sink()) + +async def test_borrow_final_attr_across_await_after_comprehension() -> None: + for i in range(50): + c = C("v" + str(i)) + assert await f(c, [1, 2, 3]) == "[1, 2, 3]" + "v" + str(i) + "z" + +[file asyncio/__init__.pyi] +async def sleep(t: float) -> None: ... + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 7722cf26ca910..56ad3673e2896 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2825,6 +2825,98 @@ def test_final_attribute() -> None: assert C.b['x'] == 'y' assert C.a is C.b +[case testFinalInstanceAttributeCannotBeRebound] +from typing import Any, Final + +from testutil import assertRaises + +class C: + def __init__(self, x: int) -> None: + self.x: Final[int] = x + +class D(C): + def __init__(self, x: int, y: int) -> None: + super().__init__(x) + self.y: Final[int] = y + +def rebind_via_any(o: Any, value: int) -> None: + o.x = value + +def test_rebind_via_any() -> None: + c = C(1) + with assertRaises(AttributeError): + rebind_via_any(c, 2) + assert c.x == 1 + +def test_rebind_via_setattr() -> None: + c = C(1) + with assertRaises(AttributeError): + setattr(c, "x", 3) + assert c.x == 1 + +def test_rebind_inherited_via_setattr() -> None: + d = D(1, 2) + # Inherited Final attribute can't be modified. + with assertRaises(AttributeError): + setattr(d, "x", 3) + # The subclass's own Final attribute can't be modified either. + with assertRaises(AttributeError): + setattr(d, "y", 4) + assert d.x == 1 + assert d.y == 2 + +[case testFinalRefcountedAttributeRead] +# Exercises reading Final reference-counted ('PyObject *') attributes, which on +# free-threaded builds use a faster read path (CPy_GetAttrRefFinal) that skips the +# concurrent-writer guard, since a Final attribute is never rebound. +from typing import Final + +class C: + def __init__(self, s: str, items: list[int]) -> None: + self.s: Final = s + self.items: Final = items + + def read_s(self) -> str: + return self.s + + def read_items(self) -> list[int]: + return self.items + + def chained_len(self) -> int: + # borrowed intermediate read of a Final attr, then a call on it + return len(self.items) + +class D(C): + def __init__(self, s: str, items: list[int], t: str) -> None: + super().__init__(s, items) + self.t: Final = t + + def read_inherited(self) -> str: + return self.s + +def test_read_final_object_attrs() -> None: + c = C("hello", [1, 2, 3]) + # Read via getter (property access) and via native method bodies. + assert c.s == "hello" + assert c.read_s() == "hello" + assert c.items == [1, 2, 3] + assert c.read_items() == [1, 2, 3] + assert c.chained_len() == 3 + # Reading repeatedly must not corrupt the refcount / free the value. + for _ in range(1000): + assert c.read_s() == "hello" + assert c.read_items() == [1, 2, 3] + assert c.s == "hello" + assert c.read_items() is c.items + +def test_read_inherited_final_attr() -> None: + d = D("a", [4, 5], "b") + assert d.s == "a" + assert d.t == "b" + assert d.read_s() == "a" + assert d.read_inherited() == "a" + assert d.read_items() == [4, 5] + [case testClassDerivedFromIntEnum] from enum import IntEnum, auto @@ -6006,3 +6098,134 @@ for _ in range(100): check(foo) after = sys.getrefcount(foo.obj) assert after - init == 0, f"Leaked {after - init} refs" + +[case testNativeAttrSetterRefcountLeak] +# Setting a native attribute from interpreted code goes through the generated +# getset descriptor setter. For refcounted unboxed types (heap-boxed ints, +# tuples with refcounted items) the setter must take exactly one reference to +# the stored value, not two. +from dataclasses import dataclass +from typing import Tuple + +@dataclass +class IntField: + v: int + +@dataclass +class TupleField: + v: Tuple[int, int] + +[file driver.py] +import sys +from native import IntField, TupleField + +# A heap-boxed int (>= 2**62) is stored as a refcounted PyObject*, unlike small +# inline-tagged ints, so an over-incref strands a real reference. Compute the +# values at runtime (not as folded literals): on free-threaded builds code +# constants are immortal, and getrefcount could not observe a leak on them. +shift = 70 +BIG = 1 << shift + +def check_no_leak(make, value) -> None: + base = sys.getrefcount(value) + objs = [make(value) for _ in range(100)] + alive = sys.getrefcount(value) + # Each live instance must hold exactly one reference to the field value. + assert alive - base == 100, f"expected 100 live refs, got {alive - base}" + del objs + after = sys.getrefcount(value) + assert after == base, f"leaked {after - base} refs" + +# The dataclass-generated __init__ stores self.v = v via the descriptor setter. +check_no_leak(IntField, BIG) + +# Tuple[int, int] is stored as an unboxed RTuple; its boxed-int elements are +# refcounted, so the setter must not over-incref them either. Use the same int +# in both slots so each instance holds exactly two references to it. +ELEM = 1 << (shift + 1) +base = sys.getrefcount(ELEM) +objs = [TupleField((ELEM, ELEM)) for _ in range(100)] +alive = sys.getrefcount(ELEM) +assert alive - base == 200, f"expected 200 live refs, got {alive - base}" +del objs +assert sys.getrefcount(ELEM) == base, f"tuple field leaked {sys.getrefcount(ELEM) - base} refs" + +# Re-assigning through the setter must release the previous value too. +o = IntField(BIG) +base = sys.getrefcount(BIG) +o.v = BIG +o.v = BIG +assert sys.getrefcount(BIG) == base, "reassignment leaked refs" + +[case testBorrowedFinalAttributeInLambdaAndNestedFunction] +from typing import Final, Callable + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def make_lambda() -> Callable[[], int]: + # The lambda body creates a C, borrows its final attribute, then calls + # spend() (which allocates heavily) before reading the borrow. + return lambda: use(C(Box(111)).b, spend(Box(0))) + +def make_nested() -> Callable[[], int]: + def inner() -> int: + return use(C(Box(111)).b, spend(Box(0))) + return inner + +def test_borrowed_final_attribute_in_lambda() -> None: + f = make_lambda() + for _ in range(1000): + assert f() == 111 + +def test_borrowed_final_attribute_in_nested_function() -> None: + g = make_nested() + for _ in range(1000): + assert g() == 111 + +[case testBorrowedFinalAttributeInComprehension] +from typing import Final + +class Box: + def __init__(self, n: int) -> None: + self.n = n + +class C: + def __init__(self, b: Box) -> None: + self.b: Final = b + +def use(x: Box, y: Box) -> int: + return x.n + y.n + +def spend(x: Box) -> Box: + # Allocate a lot so a freed Box slot is likely to be reused, turning any + # use-after-free of a borrowed attribute into an observable wrong result. + junk = [Box(i) for i in range(100)] + return Box(x.n + junk[50].n - 50) + +def comp(ns: list[int]) -> list[int]: + # Each iteration allocates a fresh C, borrows its final attribute .b, then + # calls spend() (which allocates heavily) before reading the borrow. The + # borrow container must be kept alive for the current iteration only; if it + # is scoped to the whole (comprehension) expression, the refcount pass + # dec_refs an uninitialized temporary on the loop back-edge and leaks/uses + # freed memory. + return [use(C(Box(111)).b, spend(Box(i))) for i in ns] + +def test_borrowed_final_attribute_in_comprehension() -> None: + for _ in range(1000): + assert comp([1, 2, 3, 4, 5]) == [112, 113, 114, 115, 116] diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index cf1dac7c57333..f4cef93ea8f3e 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1,7 +1,7 @@ # Test cases for generators and yield (compile and run) [case testYield] -from typing import Generator, Iterable, Union, Tuple, Dict +from typing import Callable, Dict, Generator, Iterable, Iterator, Tuple, Union def yield_three_times() -> Iterable[int]: yield 1 @@ -87,6 +87,19 @@ def return_tuple() -> Generator[int, None, Tuple[int, int]]: yield 0 return 1, 2 +def call_lambda(fn: Callable[[], None]) -> None: + fn() + +def get_iterator() -> Iterator[str]: + call_lambda(lambda: None) + yield "" + +def yield_twice_via_generator_reuse() -> Iterator[str]: + identified_imports = get_iterator() + yield from identified_imports + for identified_import in identified_imports: + yield identified_import + [file driver.py] from native import ( yield_three_times, @@ -99,6 +112,7 @@ from native import ( A, return_tuple, yield_dict_methods, + yield_twice_via_generator_reuse, ) from testutil import run_generator from collections import defaultdict @@ -114,6 +128,7 @@ assert run_generator(A(0).generator()) == ((0,), None) assert run_generator(return_tuple()) == ((0,), (1, 2)) assert run_generator(yield_dict_methods({}, {}, {})) == ((), None) assert run_generator(yield_dict_methods({1: 2}, {3: 4}, {5: 6})) == ((1, 3, 4, 6), None) +assert run_generator(yield_twice_via_generator_reuse()) == (("",), None) dd = defaultdict(int, {0: 1}) assert run_generator(yield_dict_methods(dd, dd, dd)) == ((0, 0, 1, 1), None) @@ -944,3 +959,46 @@ from typing import Optional, Union def test_compiledGeneratorEmptyTuple() -> None: jobs: Generator[Optional[str], None, None] = (_ for _ in ()) assert list(jobs) == [] + +[case testBorrowedFinalAttrAcrossYield] +from typing import Final, Generator + +class C: + def __init__(self, s: str) -> None: + self.s: Final = s + +def f(c: C) -> Generator[str, str, None]: + # c.s is a final attr; it must not be borrowed across the yield, which is + # a suspension point in the middle of the expression. + x = c.s + (yield "first") + yield x + +def test_borrow_across_yield() -> None: + for i in range(50): + c = C("value" + str(i)) + g = f(c) + assert next(g) == "first" + assert g.send("world") == "value" + str(i) + "world" + +def yield_from_gen(c: C) -> Generator[str, str, str]: + r = yield "inner" + return c.s + r + +def g2(c: C) -> Generator[str, str, str]: + # Borrow a final attr across a yield from expression. + x = c.s + (yield from yield_from_gen(c)) + return x + +def test_borrow_across_yield_from() -> None: + for i in range(50): + c = C("v" + str(i)) + gen = g2(c) + assert next(gen) == "inner" + try: + gen.send("!") + except StopIteration as e: + assert e.value == "v" + str(i) + "v" + str(i) + "!", e.value + else: + assert False + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-librt-strings.test b/mypyc/test-data/run-librt-strings.test index 7efff12667d87..333b5962c4c99 100644 --- a/mypyc/test-data/run-librt-strings.test +++ b/mypyc/test-data/run-librt-strings.test @@ -1449,8 +1449,9 @@ from testutil import assertRaises def test_codepoint_classifiers() -> None: - # Negative values are not codepoints. - for bad in (i32(-1), i32(-113)): + # Out-of-range values are not codepoints: negative, just past the maximum + # valid code point (0x10FFFF), and the largest i32. + for bad in (i32(-1), i32(-113), i32(0x110000), i32(0x7FFFFFFF)): assert not isspace(bad) assert not isdigit(bad) assert not isalnum(bad) @@ -1485,6 +1486,10 @@ def test_codepoint_classifiers_via_any() -> None: assert f(ord(false_input)) is False # Negative values are valid i32, just not codepoints. assert f(-1) is False + # Values within i32 range but past the maximum code point (0x10FFFF) + # are not codepoints either. + assert f(0x110000) is False + assert f(0x7FFFFFFF) is False # Inputs outside i32 range raise OverflowError through the wrapper. with assertRaises(OverflowError, "codepoint out of i32 range"): f(1 << 40) @@ -1509,8 +1514,9 @@ def _expect(c: str, method: str) -> int: def test_codepoint_case_conversion() -> None: - # Negative inputs return unchanged. - for bad in (i32(-1), i32(-113)): + # Out-of-range inputs return unchanged: negative, just past the maximum + # valid code point (0x10FFFF), and the largest i32. + for bad in (i32(-1), i32(-113), i32(0x110000), i32(0x7FFFFFFF)): assert toupper(bad) == bad assert tolower(bad) == bad # Agree with str.upper / str.lower across the full Unicode range @@ -1534,6 +1540,10 @@ def test_codepoint_case_conversion_via_any() -> None: assert f(in_cp) == out_cp # Negative values are valid i32, returned unchanged. assert f(-1) == -1 + # Values within i32 range but past the maximum code point (0x10FFFF) + # are returned unchanged. + assert f(0x110000) == 0x110000 + assert f(0x7FFFFFFF) == 0x7FFFFFFF # Inputs outside i32 range raise OverflowError through the wrapper. with assertRaises(OverflowError, "codepoint out of i32 range"): f(1 << 40) diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 40ca1b6e005f9..d06fc9ce10b0b 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -171,6 +171,68 @@ def test_list_build() -> None: l3.append('a') assert l3 == ['a'] +class C: + def __init__(self, s: str) -> None: + self.s = s + +a = [C("a"), C("b"), C("c")] + +def test_get_item() -> None: + zero = int() + assert a[zero].s == "a" + assert a[zero + 1].s == "b" + assert a[zero + 2].s == "c" + assert a[zero - 1].s == "c" + assert a[zero - 2].s == "b" + assert a[zero - 3].s == "a" + with assertRaises(IndexError): + a[zero + 3] + with assertRaises(IndexError): + a[zero - 4] + # TODO: Raise IndexError? + with assertRaises(OverflowError): + a[zero + 2**90] + with assertRaises(OverflowError): + a[zero - 2**90] + +def test_get_item_literal() -> None: + assert a[0].s == "a" + assert a[1].s == "b" + assert a[2].s == "c" + assert a[-1].s == "c" + assert a[-2].s == "b" + assert a[-3].s == "a" + with assertRaises(IndexError): + a[3] + with assertRaises(IndexError): + a[-4] + # TODO: Raise IndexError? + with assertRaises(OverflowError): + a[2**90] + with assertRaises(OverflowError): + a[-2**90] + +def test_set_item() -> None: + l = a.copy() + zero = int() + zero2 = int() + l[zero] = C("x") + assert l[zero2].s == "x" + l[zero + 2] = C("y") + assert l[zero2 + 2].s == "y" + l[zero - 1] = C("t") + assert l[zero2 - 1].s == "t" + l[zero - 3] = C("u") + assert l[zero2 - 3].s == "u" + with assertRaises(IndexError): + a[zero + 3] = C("z") + with assertRaises(IndexError): + a[zero - 4] = C("z") + with assertRaises(OverflowError): + a[zero + 2**90] = C("z") + with assertRaises(OverflowError): + a[zero - 2**90] = C("z") + def test_append() -> None: l = [1, 2] l.append(10) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index e586a3cba22e9..4cf391d312dff 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -1246,6 +1246,39 @@ import native [rechecked other_a] +-- Test that importing a skipped module does not force rechecks. +[case testIncrementalCompilationFollowImportsSkip] +from other_dep import f +assert f() == 1 + +[file other_dep.py] +import skipped + +def f() -> int: + return 1 + +def g() -> int: + return 2 + +# Files under "skipped" are not typechecked because of the "--follow_imports = skip" option. +[file skipped/__init__.py] +x = 1 + +[file skipped/other_child.py] +import skipped + +def child() -> int: + return 1 + +[file native.py.2] +from other_dep import g +assert g() == 2 + +[file driver.py] +import native + +[rechecked native] + [case testSeparateCompilationWithUndefinedAttribute] from other_a import A diff --git a/mypyc/test-data/run-python37.test b/mypyc/test-data/run-python37.test index 61d428c17a441..1245ad516a2d9 100644 --- a/mypyc/test-data/run-python37.test +++ b/mypyc/test-data/run-python37.test @@ -74,8 +74,15 @@ class Person5: friends: Set[str] = field(default_factory=set) parents: FrozenSet[str] = frozenset() +@dataclass +class HasFactoryDefault: + x: dict[str, int] = field(default_factory=dict) + +class InheritsDataclassInit(HasFactoryDefault): + pass + [file other.py] -from native import Person1, Person1b, Person2, Person3, Person4, Person5, testBool +from native import Person1, Person1b, Person2, Person3, Person4, Person5, testBool, InheritsDataclassInit i1 = Person1(age = 5, name = 'robot') assert i1.age == 5 assert i1.name == 'robot' @@ -125,6 +132,9 @@ assert Person1.__annotations__ == {'age': int, 'name': str} assert Person2.__annotations__ == {'age': int, 'name': str} assert Person5.__annotations__ == {'weight': float, 'friends': set, 'parents': frozenset} +v = InheritsDataclassInit() +assert isinstance(v.x, dict) +assert v.x == {} [file driver.py] import sys diff --git a/mypyc/test-data/run-threading.test b/mypyc/test-data/run-threading.test new file mode 100644 index 0000000000000..8af8755b76c44 --- /dev/null +++ b/mypyc/test-data/run-threading.test @@ -0,0 +1,156 @@ +# Test cases for librt.threading (compile and run) + +[case testLockBasics_librt] +from typing import Any +from testutil import assertRaises +from librt.threading import Lock + +class BadBool: + def __bool__(self) -> bool: + raise RuntimeError("bad bool") + +def test_lock_basic() -> None: + lock = Lock() + assert not lock.locked() + assert lock.acquire() + assert lock.locked() + lock.release() + assert not lock.locked() + +def test_lock_context_manager() -> None: + lock = Lock() + with lock as acquired: + assert acquired is True + assert lock.locked() + assert not lock.locked() + +def test_lock_non_blocking() -> None: + lock = Lock() + assert lock.acquire() + assert not lock.acquire(False) + lock.release() + assert lock.acquire(False) + lock.release() + +def test_contention() -> None: + import threading + lock = Lock() + counter = [0] + n_threads = 4 + n_increments = 10000 + + def worker() -> None: + for _ in range(n_increments): + lock.acquire() + counter[0] += 1 + lock.release() + + threads = [threading.Thread(target=worker) for _ in range(n_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + assert counter[0] == n_threads * n_increments + +def test_cross_thread_release() -> None: + # threading.Lock is unowned: a lock acquired on one thread may be + # released from another, after which another thread can acquire it. + import threading + lock = Lock() + assert lock.acquire() + + started = threading.Event() + released = threading.Event() + + def releaser() -> None: + started.set() + # Hand the lock off from a different thread than the acquirer. + lock.release() + released.set() + + t = threading.Thread(target=releaser) + t.start() + started.wait() + # This may block until releaser releases the lock on the other thread. + assert lock.acquire() + t.join() + assert released.is_set() + lock.release() + assert not lock.locked() + +def test_context_manager_exception() -> None: + lock = Lock() + try: + with lock: + assert lock.locked() + raise ValueError("test") + except ValueError: + pass + assert not lock.locked() + +def test_acquire_blocking_true() -> None: + lock = Lock() + assert lock.acquire(True) + assert lock.locked() + lock.release() + +def test_lock_constructor_errors() -> None: + lock_type: Any = Lock + make_type: Any = type + with assertRaises(TypeError): + lock_type(1) + with assertRaises(TypeError): + lock_type(foo=1) + with assertRaises(TypeError): + make_type("LockSubclass", (lock_type,), {}) + +def test_lock_acquire_argument_errors() -> None: + lock: Any = Lock() + with assertRaises(TypeError): + lock.acquire(True, False) + with assertRaises(TypeError): + lock.acquire(foo=True) + +def test_lock_acquire_blocking_truthiness() -> None: + lock: Any = Lock() + assert lock.acquire(blocking=True) + assert lock.locked() + assert not lock.acquire(blocking=False) + lock.release() + + assert lock.acquire(None) + assert lock.locked() + assert not lock.acquire(None) + lock.release() + + assert lock.acquire(1) + lock.release() + assert lock.acquire(0) + lock.release() + +def test_lock_acquire_blocking_bool_error() -> None: + lock: Any = Lock() + with assertRaises(RuntimeError, "bad bool"): + lock.acquire(BadBool()) + with assertRaises(RuntimeError, "bad bool"): + lock.acquire(blocking=BadBool()) + +def test_lock_exit_manual_call() -> None: + lock: Any = Lock() + lock.acquire() + assert lock.__exit__(None, None, None) is None + assert not lock.locked() + + lock.acquire() + assert lock.__exit__() is None + assert not lock.locked() + +def test_release_unlocked() -> None: + lock = Lock() + with assertRaises(RuntimeError): + lock.release() + # Also after acquire + release + lock.acquire() + lock.release() + with assertRaises(RuntimeError): + lock.release() diff --git a/mypyc/test/librt_cache.py b/mypyc/test/librt_cache.py index f80059ed64786..bf57b9f54196e 100644 --- a/mypyc/test/librt_cache.py +++ b/mypyc/test/librt_cache.py @@ -162,8 +162,15 @@ def get_librt_path(experimental: bool = True, opt_level: str = "0") -> str: os.makedirs(cache_root, exist_ok=True) + binary_suffix = ".pyd" if sys.platform == "win32" else ".so" + with filelock.FileLock(lock_file, timeout=300): # 5 min timeout - if os.path.exists(marker): + # Reuse the cache only if the build completed *and* the compiled + # binaries still exist. A repo-wide clean of .so/.pyd files can delete + # the cached binaries while leaving the marker behind. + if os.path.exists(marker) and any( + f.endswith(binary_suffix) for f in os.listdir(os.path.join(build_dir, "librt")) + ): return build_dir # Clean up any partial build diff --git a/mypyc/test/test_emit.py b/mypyc/test/test_emit.py index 285488e03c9ae..b2199b2dcb3b2 100644 --- a/mypyc/test/test_emit.py +++ b/mypyc/test/test_emit.py @@ -42,9 +42,23 @@ def test_reg(self) -> None: def test_object_annotation(self) -> None: assert self.emitter.object_annotation("hello, world", "line;") == " /* 'hello, world' */" - assert self.emitter.object_annotation(list(range(30)), "line;") == """\ - /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29] */""" + assert self.emitter.object_annotation(42, "line;") == " /* 42 */" + assert self.emitter.object_annotation((1, "x", None), "line;") == " /* (1, 'x', None) */" + # Annotations containing illegal C comment characters are dropped. + assert self.emitter.object_annotation("a /* b */ c", "line;") == "" + + def test_object_annotation_frozenset_is_deterministic(self) -> None: + assert ( + self.emitter.object_annotation(frozenset({"self", "cls"}), "line;") + == self.emitter.object_annotation(frozenset({"cls", "self"}), "line;") + == " /* frozenset({'cls', 'self'}) */" + ) + assert ( + self.emitter.object_annotation((frozenset({"b", "a"}),), "line;") + == self.emitter.object_annotation((frozenset({"a", "b"}),), "line;") + == " /* (frozenset({'a', 'b'}),) */" + ) + assert self.emitter.object_annotation(frozenset(), "line;") == " /* frozenset() */" def test_emit_line(self) -> None: emitter = self.emitter @@ -55,11 +69,9 @@ def test_emit_line(self) -> None: assert emitter.fragments == ["line;\n", "a {\n", " f();\n", "}\n"] emitter = Emitter(self.context, {}) emitter.emit_line("CPyStatics[0];", ann="hello, world") - emitter.emit_line("CPyStatics[1];", ann=list(range(30))) + emitter.emit_line("CPyStatics[1];", ann=42) assert emitter.fragments[0] == "CPyStatics[0]; /* 'hello, world' */\n" - assert emitter.fragments[1] == """\ -CPyStatics[1]; /* [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29] */\n""" + assert emitter.fragments[1] == "CPyStatics[1]; /* 42 */\n" def test_emit_undefined_value_for_simple_type(self) -> None: emitter = self.emitter diff --git a/mypyc/test/test_exceptions.py b/mypyc/test/test_exceptions.py index e9d4348eb64c5..d842ea1a85eda 100644 --- a/mypyc/test/test_exceptions.py +++ b/mypyc/test/test_exceptions.py @@ -11,7 +11,7 @@ from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase from mypyc.analysis.blockfreq import frequently_executed_blocks -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -34,6 +34,14 @@ class TestExceptionTransform(MypycDataSuite): def run_case(self, testcase: DataDrivenTestCase) -> None: """Perform a runtime checking transformation test case.""" + + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: diff --git a/mypyc/test/test_expand_rtype.py b/mypyc/test/test_expand_rtype.py new file mode 100644 index 0000000000000..90acc61f625d9 --- /dev/null +++ b/mypyc/test/test_expand_rtype.py @@ -0,0 +1,48 @@ +import unittest + +from mypyc.ir.class_ir import ClassIR +from mypyc.ir.rtypes import ( + RInstance, + RTuple, + RTypeVar, + RUnion, + RVec, + int_rprimitive, + str_rprimitive, + void_rtype, +) +from mypyc.rt_expandtype import expand_rtype + + +class TestExpandRType(unittest.TestCase): + def test_trivial(self) -> None: + assert expand_rtype(str_rprimitive, []) == str_rprimitive + assert expand_rtype(str_rprimitive, [int_rprimitive]) == str_rprimitive + assert expand_rtype(void_rtype, []) == void_rtype + + def test_instance(self) -> None: + inst = RInstance(ClassIR("A", "__main__")) + assert expand_rtype(inst, [int_rprimitive]) == inst + + def test_simple_expansion(self) -> None: + assert expand_rtype(RTypeVar(0), [str_rprimitive]) == str_rprimitive + + def test_tuple_expansion(self) -> None: + assert expand_rtype( + RTuple([RTypeVar(0), RTypeVar(1)]), [str_rprimitive, int_rprimitive] + ) == RTuple([str_rprimitive, int_rprimitive]) + + def test_union_expansion(self) -> None: + assert expand_rtype( + RUnion([RTypeVar(0), RTypeVar(1)]), [str_rprimitive, int_rprimitive] + ) == RUnion([str_rprimitive, int_rprimitive]) + + def test_vec_expansion(self) -> None: + assert expand_rtype(RVec(RTypeVar(0)), [str_rprimitive]) == RVec(str_rprimitive) + + def test_nested_expansion(self) -> None: + typ = RUnion([RTuple([RVec(RTypeVar(0)), RTypeVar(1)]), RVec(RVec(RTypeVar(0)))]) + expected = RUnion( + [RTuple([RVec(str_rprimitive), int_rprimitive]), RVec(RVec(str_rprimitive))] + ) + assert expand_rtype(typ, [str_rprimitive, int_rprimitive]) == expected diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 7e3993e267e74..6608e6db8e7b4 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -36,6 +36,7 @@ "irbuild-statements.test", "irbuild-nested.test", "irbuild-classes.test", + "irbuild-final.test", "irbuild-optional.test", "irbuild-any.test", "irbuild-generics.test", @@ -61,6 +62,7 @@ "irbuild-librt-strings.test", "irbuild-librt-random.test", "irbuild-base64.test", + "irbuild-threading.test", "irbuild-time.test", "irbuild-match.test", ] @@ -86,6 +88,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if "_withgil" in testcase.name and IS_FREE_THREADED: # Test case should only run on a non-free-threaded build. return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) diff --git a/mypyc/test/test_literals.py b/mypyc/test/test_literals.py index a8c17d10d30d0..f1dc7f2434414 100644 --- a/mypyc/test/test_literals.py +++ b/mypyc/test/test_literals.py @@ -10,6 +10,7 @@ _encode_int_values, _encode_str_values, format_str_literal, + literal_sort_key, ) @@ -88,3 +89,50 @@ def test_tuple_literal(self) -> None: "7", # Second tuple (length=4) "0", # Third tuple (length=0) ] + + def test_frozenset_literal_index_is_deterministic(self) -> None: + # Index assignment for members must not depend on frozenset iteration + # order (which is hash-seed dependent), so that generated code is + # reproducible. + lit1 = Literals() + lit1.record_literal(frozenset({"self", "cls"})) + lit2 = Literals() + lit2.record_literal(frozenset({"cls", "self"})) + for s in ("self", "cls"): + assert lit1.literal_index(s) == lit2.literal_index(s) + # Members are recorded in sorted order. + assert lit1.literal_index("cls") == 3 + assert lit1.literal_index("self") == 4 + + def test_frozenset_encoding_is_deterministic(self) -> None: + lit1 = Literals() + lit1.record_literal(frozenset({"self", "cls"})) + lit2 = Literals() + lit2.record_literal(frozenset({"cls", "self"})) + assert lit1.encoded_frozenset_values() == lit2.encoded_frozenset_values() + + def test_literal_sort_key_is_total_over_types(self) -> None: + # Heterogeneous, individually unorderable items must still be sorted. + values = ["x", b"y", 1, None, (1, 2), frozenset({1, 2})] + values_reversed = list(reversed(values)) + assert sorted(values, key=literal_sort_key) == sorted( + values_reversed, key=literal_sort_key + ) + + def test_literal_sort_key_with_frozenset(self) -> None: + assert literal_sort_key(frozenset({"a", "b"})) == literal_sort_key(frozenset({"b", "a"})) + assert literal_sort_key((frozenset({"a", "b"}),)) == literal_sort_key( + (frozenset({"b", "a"}),) + ) + assert literal_sort_key(frozenset({"a", frozenset({"b", "c"})})) == literal_sort_key( + frozenset({frozenset({"c", "b"}), "a"}) + ) + + def test_nested_frozenset_literal_index_is_deterministic(self) -> None: + lit1 = Literals() + lit1.record_literal(frozenset({frozenset({"a", "b"}), frozenset({"c", "d"})})) + lit2 = Literals() + lit2.record_literal(frozenset({frozenset({"d", "c"}), frozenset({"b", "a"})})) + for s in ("a", "b", "c", "d"): + assert lit1.literal_index(s) == lit2.literal_index(s) + assert lit1.encoded_frozenset_values() == lit2.encoded_frozenset_values() diff --git a/mypyc/test/test_lowering.py b/mypyc/test/test_lowering.py index e27b4e77eea8a..75351cd50e4a7 100644 --- a/mypyc/test/test_lowering.py +++ b/mypyc/test/test_lowering.py @@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions from mypyc.test.testutil import ( @@ -28,7 +28,7 @@ class TestLowering(MypycDataSuite): - files = ["lowering-int.test", "lowering-list.test"] + files = ["lowering-int.test", "lowering-list.test", "lowering-vec.test"] base_path = test_temp_dir def run_case(self, testcase: DataDrivenTestCase) -> None: @@ -36,6 +36,12 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) diff --git a/mypyc/test/test_optimizations.py b/mypyc/test/test_optimizations.py index 6ca53b134ba91..c6a40d08e2e67 100644 --- a/mypyc/test/test_optimizations.py +++ b/mypyc/test/test_optimizations.py @@ -7,7 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.func_ir import FuncIR from mypyc.ir.pprint import format_func from mypyc.options import CompilerOptions @@ -33,6 +33,12 @@ class OptimizationSuite(MypycDataSuite): base_path = test_temp_dir def run_case(self, testcase: DataDrivenTestCase) -> None: + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) try: diff --git a/mypyc/test/test_refcount.py b/mypyc/test/test_refcount.py index 7efa861e784fb..896ba14e2e68b 100644 --- a/mypyc/test/test_refcount.py +++ b/mypyc/test/test_refcount.py @@ -11,7 +11,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.common import TOP_LEVEL_NAME +from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME from mypyc.ir.pprint import format_func from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, @@ -40,6 +40,12 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + if "_withgil" in testcase.name and IS_FREE_THREADED: + # Test case should only run on a non-free-threaded build. + return + if "_nogil" in testcase.name and not IS_FREE_THREADED: + # Test case should only run on a free-threaded build. + return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index e7be5fcf8425a..868f83654b47c 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -82,6 +82,7 @@ "run-base64.test", "run-librt-time.test", "run-librt-random.test", + "run-threading.test", "run-match.test", "run-vecs-i64-interp.test", "run-vecs-misc-interp.test", @@ -234,6 +235,10 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. options.per_module_options["unchecked.*"] = {"follow_imports": "error"} + # Avoid checking modules/packages named 'skipped', to provide a way + # to test interacting with code ignored by follow_imports=skip. + options.per_module_options["skipped"] = {"follow_imports": "skip"} + options.per_module_options["skipped.*"] = {"follow_imports": "skip"} source = build.BuildSource("native.py", "native", None) sources = [source] diff --git a/mypyc/transform/spill.py b/mypyc/transform/spill.py index d92dd661e7eb4..c2d4d4311375d 100644 --- a/mypyc/transform/spill.py +++ b/mypyc/transform/spill.py @@ -2,6 +2,9 @@ from __future__ import annotations +from collections.abc import Collection +from typing import cast + from mypyc.analysis.dataflow import AnalysisResult, analyze_live_regs, get_cfg from mypyc.common import TEMP_ATTR_NAME from mypyc.ir.class_ir import ClassIR @@ -13,6 +16,7 @@ GetAttr, IncRef, LoadErrorValue, + Op, Register, SetAttr, Value, @@ -31,6 +35,19 @@ def insert_spills(ir: FuncIR, env: ClassIR) -> None: ir.blocks = spill_regs(ir.blocks, env, entry_live, live, ir.arg_regs[0]) +def sort_values(values: Collection[Op], blocks: list[BasicBlock]) -> list[Op]: + if len(values) > 1: + order = {} + i = 0 + for block in blocks: + for op in block.ops: + order[op] = i + i += 1 + return sorted(values, key=lambda v: order[v]) + else: + return list(values) + + def spill_regs( blocks: list[BasicBlock], env: ClassIR, @@ -48,7 +65,9 @@ def spill_regs( env_reg = self_reg spill_locs = {} - for i, val in enumerate(to_spill): + # Sort values to make the order deterministic. All the spilled values are + # known to be Op instances, so the cast is safe. + for i, val in enumerate(sort_values(cast(set[Op], to_spill), blocks)): name = f"{TEMP_ATTR_NAME}2_{i}" env.attributes[name] = val.type if val.type.error_overlap: diff --git a/pyproject.toml b/pyproject.toml index a89bac5c356eb..aa9432c06e48b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,12 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.11.0; platform_python_implementation != 'PyPy'", + "librt>=0.13.0; platform_python_implementation != 'PyPy'", # the following is from build-requirements.txt "types-psutil", "types-setuptools", # required to work around a mypyc import bug - "ast-serialize>=0.5.0,<1.0.0", + "ast-serialize>=0.6.0,<1.0.0", ] build-backend = "setuptools.build_meta" @@ -58,8 +58,8 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.11.0; platform_python_implementation != 'PyPy'", - "ast-serialize>=0.5.0,<1.0.0", + "librt>=0.13.0; platform_python_implementation != 'PyPy'", + "ast-serialize>=0.6.0,<1.0.0", ] dynamic = ["version"] diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index e887b4c575528..cd40f2cf29a65 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -839,6 +839,34 @@ def bar() -> None: [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] +[case testUntypedAsyncFunctionAndMethodReturnCoroutine] +# flags: --show-error-codes +async def foo(): + pass + +class C: + async def method(self): + pass + +def f(c: C) -> None: + a = foo() + b = c.method() + reveal_type(foo) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, Any]" + reveal_type(a) # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? \ + # N: Revealed type is "typing.Coroutine[Any, Any, Any]" + foo() # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? + reveal_type(c.method) # N: Revealed type is "def () -> typing.Coroutine[Any, Any, Any]" + reveal_type(b) # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? \ + # N: Revealed type is "typing.Coroutine[Any, Any, Any]" + c.method() # E: Value of type "Coroutine[Any, Any, Any]" must be used [unused-coroutine] \ + # N: Are you missing an await? + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + [case testAsyncForOutsideCoroutine] async def g(): yield 0 diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 54b3afadc8b32..5ac0318421167 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2609,6 +2609,27 @@ class Y(X): [builtins fixtures/tuple.pyi] +[case testDunderReplaceDoesNotBlockPlainIssubclassNarrowing] +# https://github.com/python/mypy/issues/21635 +# flags: --python-version 3.13 +from dataclasses import dataclass + +@dataclass +class A: ... +@dataclass +class M: ... +@dataclass +class B(A): ... +@dataclass +class C(M, A): ... + +cls: type[A] = C +if issubclass(cls, M): + reveal_type(cls) # N: Revealed type is "type[__main__.]" + n: int = 'foo' # E: Incompatible types in assignment (expression has type "str", variable has type "int") +[builtins fixtures/isinstancelist.pyi] + + [case testFrozenWithFinal] from dataclasses import dataclass from typing import Final diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index dd4687181ca41..acdee2a10430a 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -59,7 +59,7 @@ async def f(): # E: Function is missing a return type annotation \ # N: Use "-> None" if function does not return a value pass [builtins fixtures/async_await.pyi] -[typing fixtures/typing-medium.pyi] +[typing fixtures/typing-async.pyi] [case testAsyncUnannotatedArgument] # flags: --disallow-untyped-defs diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 7ff9fbef06a53..18931dd9f152f 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -8204,3 +8204,19 @@ reveal_type(x) tmp/a.py:2: note: Revealed type is "builtins.int" [out2] tmp/a.py:2: note: Revealed type is "builtins.str" + +[case testIncrementalFollowImportsSkipStubWithSkippedRuntimeDependency] +# flags: --follow-imports=skip +import pkg.foo +[file pkg/__init__.py] + +[file pkg/foo.pyi] +from pkg import helper + +x = 1 +[file pkg/helper.py] +y = 2 +[rechecked] +[stale] +[out2] +[out3] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 7bab7baa6cebd..29f4cd47929d9 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -3335,6 +3335,24 @@ def narrow_dict(x: Literal['a', 'b', 'c'], t: dict[Literal['a', 'b'], int]): [builtins fixtures/primitives.pyi] +[case testNarrowCustomEqEnumInLiteralContainer] +# flags: --strict-equality --warn-unreachable +# https://github.com/python/mypy/issues/21703 +from enum import Enum + +class E(Enum): + foo = 1 + bar = 2 + + def __eq__(self, other: object) -> bool: + return True + +def f(x: int) -> None: + if x in [E.foo, E.bar]: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/list.pyi] + + [case testNarrowingLiteralInLiteralContainer] # flags: --strict-equality --warn-unreachable from typing import Literal diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index bfbd2e631f5d8..79c6f630f4d98 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -426,6 +426,42 @@ class A: pass class B: pass [builtins fixtures/tuple.pyi] +[case testMultipleAssignmentWithOverloadReinferredAsHomogeneousTuple] +from typing import Any, TypeVar, overload + +T = TypeVar("T") + +@overload +def f(value: T) -> tuple[T, int]: ... +@overload +def f(*values: object) -> tuple[Any, ...]: ... +def f(*values: object, **kwargs: object) -> object: ... + +def g() -> None: + first: str + first, second = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(second) # N: Revealed type is "builtins.int" + + last: str + _, last = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMultipleAssignmentWithOverloadReinferredAsNonTuple] +from typing import TypeVar, overload + +T = TypeVar("T") + +@overload +def f(value: T) -> tuple[T]: ... +@overload +def f(*values: object) -> int: ... +def f(*values: object, **kwargs: object) -> object: ... + +def g() -> None: + value: str + (value,) = f(1) # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[builtins fixtures/tuple.pyi] + [case testMultipleAssignmentWithSquareBracketTuples] # flags: --no-strict-optional from typing import Tuple diff --git a/test-data/unit/check-typevar-unbound.test b/test-data/unit/check-typevar-unbound.test index 79d326f9fedf3..2dffaee6f2fe7 100644 --- a/test-data/unit/check-typevar-unbound.test +++ b/test-data/unit/check-typevar-unbound.test @@ -13,6 +13,9 @@ def g() -> U: # E: A function returning TypeVar should receive at least one argu # N: Consider using the upper bound "int" instead ... +def g2() -> U: # type: ignore[type-var] + ... + V = TypeVar('V', int, str) def h() -> V: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index ab2c5f1c43e74..6850eabf09b6a 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -69,6 +69,19 @@ x = 1 -> m.f -> m, m.f +[case testAccessReexportedModuleAttribute] +import pkg +pkg.f() +[file pkg/__init__.py] +from pkg.sub import * +[file pkg/sub.py] +def f() -> None: pass +[out] + -> m + -> m + -> pkg + -> m + [case testImport] import n [file n.py] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index bc02f49e5d835..ef2e8c0b343bf 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -11839,3 +11839,134 @@ def bar() -> str: return "a" main:2: note: Revealed type is "None | builtins.int" == main:2: note: Revealed type is "None | builtins.str" + +[case testStarExportedNameDeleted1] +import pkg + +pkg.loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "loads" + +[case testStarExportedNameDeleted2] +import pkg + +pkg.C() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +class C: pass + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "C" + +[case testStarExportedNameDeleted3] +from pkg import loads + +loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:1: error: Module "pkg" has no attribute "loads" + +[case testStarExportedNameDeleted4] +from pkg import sub + +a = sub.x + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +x = 1 + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: "object" has no attribute "x" + +[case testStarExportedNameDeleted5] +from pkg import sub + +x: sub.N = 1 + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +N = int + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "sub.N" is not defined + +[case testStarExportedNameDeleted6] +from pkg import * + +loads() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +def loads() -> None: ... + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "loads" is not defined + +[case testStarExportedNameDeleted7] +from pkg import * + +C() + +[file pkg/__init__.pyi] +from .sub import * + +[file pkg/sub.pyi] +class C: pass + +[file pkg/sub.pyi.2] +[out] +== +main:3: error: Name "C" is not defined + +[case testStarExportedNameDeleted8] +from a import * + +C() + +[file a.py] +from b import * + +[file b.py] +from c import * + +[file c.py] +class C: pass + +[file c.py.2] +[out] +== +main:3: error: Name "C" is not defined diff --git a/test-requirements.txt b/test-requirements.txt index 254c6b8aaaa28..73d5966040c4a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in # -ast-serialize==0.5.0 +ast-serialize==0.6.0 # via -r mypy-requirements.txt attrs==26.1.0 # via -r test-requirements.in @@ -25,7 +25,7 @@ identify==2.6.19 # via pre-commit iniconfig==2.3.0 # via pytest -librt==0.11.0 ; platform_python_implementation != "PyPy" +librt==0.13.0 ; platform_python_implementation != "PyPy" # via -r mypy-requirements.txt lxml==6.1.0 ; python_version < "3.15" # via -r test-requirements.in