From 36b59cba6c78b0ebdc43abf6229397ec90a464ff Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 16:29:25 +0100 Subject: [PATCH 01/10] wip reworked setting __protocol_attrs__ --- Lib/test/test_typing.py | 44 ++++++++++++++++++++++- Lib/typing.py | 77 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 111 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2b5f34b4b92e0c..03432f74cdb8a2 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3907,7 +3907,7 @@ def __iter__(self): def close(self): return 0 - self.assertIsSubclass(B, Custom) + self.assertIsSubclass(B, Custom, msg=Custom.__protocol_attrs__) self.assertNotIsSubclass(A, Custom) @runtime_checkable @@ -4091,6 +4091,48 @@ def method(self) -> None: ... self.assertIsInstance(Foo(), ProtocolWithMixedMembers) self.assertNotIsInstance(42, ProtocolWithMixedMembers) + def test_protocol_special_members(self): + # See https://github.com/python/cpython/issues/112319 + + T_co = TypeVar("T_co", covariant=True) + + @runtime_checkable + class GenericIterable(Protocol[T_co]): + def __class_getitem__(cls, item): ... + def __iter__(self): ... + + self.assertIsInstance([1,2,3], GenericIterable) + self.assertNotIsInstance("123", GenericIterable) # str is not a generic type! + + class TakesKWARGS(Protocol): + def __init__(self, **kwargs): ... # NOTE: For static checking. + + assert TakesKWARGS.__protocol_attrs__ == {"__init__"} + + + def test_protocol_special_attributes(self): + class Documented(Protocol): + """Matches classes that have a docstring.""" + __doc__: str # NOTE: For static checking, undocumented classes have __doc__ = None. + + assert Documented.__protocol_attrs__ == {"__doc__"} + + @runtime_checkable + class Slotted(Protocol): + """Matches classes that have a __slots__ attribute.""" + __slots__: tuple + + class Unslotted: + pass + + class WithSLots: + __slots__ = ("foo", "bar") + + assert Slotted.__protocol_attrs__ == {"__slots__"} + self.assertNotIsInstance(Unslotted(), Slotted) + self.assertIsInstance(WithSLots(), Slotted) + + class GenericTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index a96c7083eb785e..11669324969cfa 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1675,13 +1675,66 @@ class _TypingEllipsis: _SPECIAL_NAMES = frozenset({ '__abstractmethods__', '__annotations__', '__dict__', '__doc__', - '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', + '__module__', '__slots__', '__match_args__', '__qualname__', +}) +_SPECIAL_CALLABLE_NAMES = frozenset({ + '__init__', '__new__', '__subclasshook__','__class_getitem__', '__weakref__', }) # These special attributes will be not collected as protocol members. EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS | _SPECIAL_NAMES | {'_MutableMapping__marker'} +EXCLUDED_MEMBERS = EXCLUDED_ATTRIBUTES | _SPECIAL_CALLABLE_NAMES + + +def _get_local_members(namespace): + """Collect the specified attributes from a classes' namespace.""" + annotations = namespace.get("__annotations__", {}) + attrs = (set(namespace) | set(annotations)) - EXCLUDED_MEMBERS + # exclude special "_abc_" attributes + return {attr for attr in attrs if not attr.startswith('_abc_')} + + +def _get_local_protocol_members(namespace): + """Collect the specified attributes from the protocols' namespace.""" + # annotated attributes are always considered protocol members + annotations = namespace.get("__annotations__", {}) + # only namespace members outside the excluded set are considered protocol members + return (set(namespace) - EXCLUDED_ATTRIBUTES) | set(annotations) + + + +def _get_parent_members(cls): + """Collect protocol members from parents of arbitrary class object. + + This includes names actually defined in the class dictionary, as well + as names that appear in annotations. Special names (above) are skipped. + """ + attrs = set() + for base in cls.__mro__[1:-1]: # without self and object + if base.__name__ in {'Protocol', 'Generic'}: + continue + elif getattr(base, "_is_protocol", False): + attrs |= getattr(base, "__protocol_attrs__", set()) + else: # get from annotations + attrs |= _get_local_members(base.__dict__) + return attrs + + +def _get_cls_members(cls): + """Collect protocol members from an arbitrary class object. + + This includes names actually defined in the class dictionary, as well + as names that appear in annotations. Special names (above) are skipped. + """ + attrs = set() + for base in cls.__mro__[:-1]: # without object + if base.__name__ in {'Protocol', 'Generic'}: + continue + elif getattr(base, "_is_protocol", False): + attrs |= getattr(base, "__protocol_attrs__", set()) + else: + attrs |= _get_local_members(base.__dict__) + return attrs def _get_protocol_attrs(cls): @@ -1696,11 +1749,12 @@ def _get_protocol_attrs(cls): continue annotations = getattr(base, '__annotations__', {}) for attr in (*base.__dict__, *annotations): - if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: + if attr not in EXCLUDED_MEMBERS and not attr.startswith('_abc_'): attrs.add(attr) return attrs + def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) @@ -1804,10 +1858,14 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): ) return super().__new__(mcls, name, bases, namespace, **kwargs) - def __init__(cls, *args, **kwargs): - super().__init__(*args, **kwargs) - if getattr(cls, "_is_protocol", False): - cls.__protocol_attrs__ = _get_protocol_attrs(cls) + def __init__(cls, name, bases, namespace, **kwds): + super().__init__(name, bases, namespace, **kwds) + if getattr(cls, "_is_protocol", False) and cls.__name__ != "Protocol": + cls.__protocol_attrs__ = ( + _get_local_protocol_members(namespace) | _get_parent_members(cls) + ) + # local_attrs = _get_local_protocol_members(namespace) + # cls.__protocol_attrs__ = local_attrs.union(*map(_get_cls_members, bases)) # PEP 544 prohibits using issubclass() # with protocols that have non-method members. cls.__callable_proto_members_only__ = all( @@ -1829,7 +1887,8 @@ def __subclasscheck__(cls, other): and cls.__dict__.get("__subclasshook__") is _proto_hook ): raise TypeError( - "Protocols with non-method members don't support issubclass()" + f"Protocols with non-method members don't support issubclass()." + f" Non-method members: {cls.__protocol_attrs__}." ) if not getattr(cls, '_is_runtime_protocol', False): raise TypeError( From b9d150d1db8dbfb8ce351a59b63f568548fb925c Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 16:49:27 +0100 Subject: [PATCH 02/10] removed remnant code --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 03432f74cdb8a2..377f42af504920 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3907,7 +3907,7 @@ def __iter__(self): def close(self): return 0 - self.assertIsSubclass(B, Custom, msg=Custom.__protocol_attrs__) + self.assertIsSubclass(B, Custom) self.assertNotIsSubclass(A, Custom) @runtime_checkable From f7f9a90ac523b44e1d9d1c66b0c22b6c34c9da42 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 17:29:53 +0100 Subject: [PATCH 03/10] Update Lib/typing.py Co-authored-by: Alex Waygood --- Lib/typing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 11669324969cfa..9ed820a62c57ea 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1754,7 +1754,6 @@ def _get_protocol_attrs(cls): return attrs - def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) From a55ac9bfb5da876f4838a64e3c3dcb742ad44bc3 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 17:30:01 +0100 Subject: [PATCH 04/10] Update Lib/typing.py Co-authored-by: Alex Waygood --- Lib/typing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 9ed820a62c57ea..6c970ea8e19fbe 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1702,7 +1702,6 @@ def _get_local_protocol_members(namespace): return (set(namespace) - EXCLUDED_ATTRIBUTES) | set(annotations) - def _get_parent_members(cls): """Collect protocol members from parents of arbitrary class object. From 3686d11bff5a9b9c8fcc0c8fc74369c71a5d658e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 23 Nov 2023 16:37:39 +0000 Subject: [PATCH 05/10] Apply suggestions from code review --- Lib/test/test_typing.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 377f42af504920..10e3a83edb524c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4107,15 +4107,14 @@ def __iter__(self): ... class TakesKWARGS(Protocol): def __init__(self, **kwargs): ... # NOTE: For static checking. - assert TakesKWARGS.__protocol_attrs__ == {"__init__"} - + self.assertEqual(TakesKWARGS.__protocol_attrs__, {"__init__"}) def test_protocol_special_attributes(self): class Documented(Protocol): """Matches classes that have a docstring.""" __doc__: str # NOTE: For static checking, undocumented classes have __doc__ = None. - assert Documented.__protocol_attrs__ == {"__doc__"} + self.assertEqual(Documented.__protocol_attrs__, {"__doc__"}) @runtime_checkable class Slotted(Protocol): @@ -4125,13 +4124,12 @@ class Slotted(Protocol): class Unslotted: pass - class WithSLots: + class WithSlots: __slots__ = ("foo", "bar") - assert Slotted.__protocol_attrs__ == {"__slots__"} + self.assertEqual(Slotted.__protocol_attrs__, {"__slots__"}) self.assertNotIsInstance(Unslotted(), Slotted) - self.assertIsInstance(WithSLots(), Slotted) - + self.assertIsInstance(WithSlots(), Slotted) class GenericTests(BaseTestCase): From d9c4c3c8c3214f3ade1b0c143fa359e5996a1374 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 17:40:42 +0100 Subject: [PATCH 06/10] reverted typeerror context --- Lib/typing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 6c970ea8e19fbe..a314e9efbafcbb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1885,8 +1885,7 @@ def __subclasscheck__(cls, other): and cls.__dict__.get("__subclasshook__") is _proto_hook ): raise TypeError( - f"Protocols with non-method members don't support issubclass()." - f" Non-method members: {cls.__protocol_attrs__}." + "Protocols with non-method members don't support issubclass()" ) if not getattr(cls, '_is_runtime_protocol', False): raise TypeError( From 51bf22145e9d7990b3d70c513be7b28357a439f1 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 18:05:46 +0100 Subject: [PATCH 07/10] reverted order in and check --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index a314e9efbafcbb..9b416c1f0a1893 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1748,7 +1748,7 @@ def _get_protocol_attrs(cls): continue annotations = getattr(base, '__annotations__', {}) for attr in (*base.__dict__, *annotations): - if attr not in EXCLUDED_MEMBERS and not attr.startswith('_abc_'): + if not attr.startswith('_abc_') and attr not in EXCLUDED_MEMBERS: attrs.add(attr) return attrs From 36706a649168e733b32975ec0641204b89f44378 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 18:07:38 +0100 Subject: [PATCH 08/10] Update Lib/typing.py Co-authored-by: Alex Waygood --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 9b416c1f0a1893..0d0e8c229d20cf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1689,7 +1689,7 @@ class _TypingEllipsis: def _get_local_members(namespace): """Collect the specified attributes from a classes' namespace.""" annotations = namespace.get("__annotations__", {}) - attrs = (set(namespace) | set(annotations)) - EXCLUDED_MEMBERS + attrs = (namespace.keys() | annotations.keys()) - EXCLUDED_MEMBERS # exclude special "_abc_" attributes return {attr for attr in attrs if not attr.startswith('_abc_')} From 342afdaa9ee8a745e576fa13a313cf5ccc818b03 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 23 Nov 2023 18:07:45 +0100 Subject: [PATCH 09/10] Update Lib/typing.py Co-authored-by: Alex Waygood --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 0d0e8c229d20cf..35865933bd199c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1699,7 +1699,7 @@ def _get_local_protocol_members(namespace): # annotated attributes are always considered protocol members annotations = namespace.get("__annotations__", {}) # only namespace members outside the excluded set are considered protocol members - return (set(namespace) - EXCLUDED_ATTRIBUTES) | set(annotations) + return (namespace.keys() - EXCLUDED_ATTRIBUTES) | annotations.keys() def _get_parent_members(cls): From 14c2c9156fa1d1a6f0e14bfd39c3004cf781d669 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 23 Nov 2023 17:43:29 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...-11-23-17-43-28.gh-issue-112319.MdsRx7.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-23-17-43-28.gh-issue-112319.MdsRx7.rst diff --git a/Misc/NEWS.d/next/Library/2023-11-23-17-43-28.gh-issue-112319.MdsRx7.rst b/Misc/NEWS.d/next/Library/2023-11-23-17-43-28.gh-issue-112319.MdsRx7.rst new file mode 100644 index 00000000000000..f84b9a3c117e51 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-23-17-43-28.gh-issue-112319.MdsRx7.rst @@ -0,0 +1,22 @@ +:class:`Protocol` classes now allow specification of previously excluded attributes and methods. + +Example:: + + class Documented(Protocol): + """A Protocol for documented classes.""" + __doc__: str + + class Slotted(Protocol): + """A Protocol for classes with __slots__.""" + __slots__: tuple[str, ...] + + @runtime_checkable + class GenericIterable(Protocol): + """An iterable that must also be a generic type.""" + def __class_getitem__(cls, item): ... + def __iter__(self): ... + + assert isinstance(["a", "b", "c"], GenericIterable) # ✅ + assert not isinstance("abc", GenericIterable) # ✅ + +Patch by Randolf Scholz.