From c320bce6b332bca6f124a53478bb3fd456566d58 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 13 Apr 2025 21:10:10 -0700 Subject: [PATCH 1/3] gh-132426: Add get_annotate_from_class_namespace replacing get_annotate_function As noted on the issue, making get_annotate_function() support both types and mappings is problematic because one object may be both. So let's add a new one that works with any mapping. This leaves get_annotate_function() not very useful, so remove it. --- Doc/library/annotationlib.rst | 77 ++++++++++++++++++- Doc/reference/datamodel.rst | 19 +---- Lib/annotationlib.py | 16 ++-- Lib/test/test_annotationlib.py | 60 +++++++++------ Lib/typing.py | 4 +- ...-04-13-21-11-11.gh-issue-132426.SZno1d.rst | 3 + 6 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-13-21-11-11.gh-issue-132426.SZno1d.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 7a6d44069ed005..102c4237290b36 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -40,7 +40,7 @@ The :func:`get_annotations` function is the main entry point for retrieving annotations. Given a function, class, or module, it returns an annotations dictionary in the requested format. This module also provides functionality for working directly with the :term:`annotate function` -that is used to evaluate annotations, such as :func:`get_annotate_function` +that is used to evaluate annotations, such as :func:`get_annotate_from_class_namespace` and :func:`call_annotate_function`, as well as the :func:`call_evaluate_function` function for working with :term:`evaluate functions `. @@ -300,10 +300,10 @@ Functions .. versionadded:: 3.14 -.. function:: get_annotate_function(obj) +.. function:: get_annotate_from_class_namespace(namespace) - Retrieve the :term:`annotate function` for *obj*. Return :const:`!None` - if *obj* does not have an annotate function. *obj* may be a class, function, + Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*. + Return :const:`!None` if the namespace does not contain an annotate function. *obj* may be a class, function, module, or a namespace dictionary for a class. The last case is useful during class creation, e.g. in the ``__new__`` method of a metaclass. @@ -407,3 +407,72 @@ Functions .. versionadded:: 3.14 + +Recipes +------- + +Using annotations in a metaclass +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A :ref:`metaclass ` may want to inspect or even modify the annotations +in a class body during class creation. Doing so requires retrieving annotations +from the class namespace dictionary. For classes created with +``from __future__ import annotations``, the annotations will be in the ``__annotations__`` +key of the dictionary. For other classes with annotations, +:func:`get_annotate_from_class_namespace` can be used to get the +annotate function, and :func:`call_annotate_function` can be used to call it and +retrieve the annotations. Using the :attr:`~Format.FORWARDREF` format will usually +be best, because this allows the annotations to refer to names that cannot yet be +resolved when the class is created. + +To modify the annotations, it is best to create a wrapper annotate function +that calls the original annotate function, makes any necessary adjustments, and +returns the result. + +Below is an example of a metaclass that filters out all :class:`typing.ClassVar` +annotations from the class and puts them in a separate attribute:: + + import annotationlib + import typing + + class ClassVarSeparator(type): + def __new__(mcls, name, bases, ns): + if "__annotations__" in ns: # from __future__ import annotations + annotations = ns["__annotations__"] + classvar_keys = { + key for key, value in annotations.items() + # Use string comparison for simplicity; a more robust solution + # could use annotationlib.ForwardRef.evaluate + if value.startswith("ClassVar") + } + classvars = {key: annotations[key] for key in classvar_keys} + ns["__annotations__"] = { + key: value for key, value in annotations.items() + if key not in classvar_keys + } + wrapped_annotate = None + elif annotate := annotationlib.get_annotate_from_class_namespace(ns): + annotations = annotationlib.call_annotate_function( + annotate, format=annotationlib.Format.FORWARDREF + ) + classvar_keys = { + key for key, value in annotations.items() + if typing.get_origin(value) is typing.ClassVar + } + classvars = {key: annotations[key] for key in classvar_keys} + + def wrapped_annotate(format): + annos = annotationlib.call_annotate_function(annotate, format, owner=typ) + return {key: value for key, value in annos.items() if key not in classvar_keys} + + else: # no annotations + classvars = {} + wrapped_annotate = None + typ = super().__new__(mcls, name, bases, ns) + + if wrapped_annotate is not None: + # Wrap the original __annotate__ with a wrapper that removes ClassVars + typ.__annotate__ = wrapped_annotate + typ.classvars = classvars # Store the ClassVars in a separate attribute + return typ + diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 66b836eaf0008a..475ea09d760bbd 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1228,15 +1228,9 @@ Special attributes :attr:`__annotations__ attributes `. For best practices on working with :attr:`~object.__annotations__`, - please see :mod:`annotationlib`. - - .. caution:: - - Accessing the :attr:`!__annotations__` attribute of a class - object directly may yield incorrect results in the presence of - metaclasses. In addition, the attribute may not exist for - some classes. Use :func:`annotationlib.get_annotations` to - retrieve class annotations safely. + please see :mod:`annotationlib`. It is recommended to use + :func:`annotationlib.get_annotations` instead of accessing this + attribute directly. .. versionchanged:: 3.14 Annotations are now :ref:`lazily evaluated `. @@ -1247,13 +1241,6 @@ Special attributes if the class has no annotations. See also: :attr:`__annotate__ attributes `. - .. caution:: - - Accessing the :attr:`!__annotate__` attribute of a class - object directly may yield incorrect results in the presence of - metaclasses. Use :func:`annotationlib.get_annotate_function` to - retrieve the annotate function safely. - .. versionadded:: 3.14 * - .. attribute:: type.__type_params__ diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index e1c96298426283..77e6ed514bc661 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -12,7 +12,7 @@ "ForwardRef", "call_annotate_function", "call_evaluate_function", - "get_annotate_function", + "get_annotate_from_class_namespace", "get_annotations", "annotations_to_string", "value_to_string", @@ -619,7 +619,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): raise ValueError(f"Invalid format: {format!r}") -def get_annotate_function(obj): +def get_annotate_from_class_namespace(obj): """Get the __annotate__ function for an object. obj may be a function, class, or module, or a user-defined type with @@ -627,12 +627,10 @@ def get_annotate_function(obj): Returns the __annotate__ function or None. """ - if isinstance(obj, dict): - try: - return obj["__annotate__"] - except KeyError: - return obj.get("__annotate_func__", None) - return getattr(obj, "__annotate__", None) + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) def get_annotations( @@ -827,7 +825,7 @@ def annotations_to_string(annotations): def _get_and_call_annotate(obj, format): - annotate = get_annotate_function(obj) + annotate = getattr(obj, "__annotate__", None) if annotate is not None: ann = call_annotate_function(annotate, format, owner=obj) if not isinstance(ann, dict): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 9b3619afea2d45..4aa519409a310a 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1,5 +1,6 @@ """Tests for the annotations module.""" +import textwrap import annotationlib import builtins import collections @@ -11,7 +12,6 @@ Format, ForwardRef, get_annotations, - get_annotate_function, annotations_to_string, value_to_string, ) @@ -1079,13 +1079,13 @@ class Y(metaclass=Meta): b: float self.assertEqual(get_annotations(Meta), {"a": int}) - self.assertEqual(get_annotate_function(Meta)(Format.VALUE), {"a": int}) + self.assertEqual(Meta.__annotate__(Format.VALUE), {"a": int}) self.assertEqual(get_annotations(X), {}) - self.assertIs(get_annotate_function(X), None) + self.assertIs(X.__annotate__, None) self.assertEqual(get_annotations(Y), {"b": float}) - self.assertEqual(get_annotate_function(Y)(Format.VALUE), {"b": float}) + self.assertEqual(Y.__annotate__(Format.VALUE), {"b": float}) def test_unannotated_meta(self): class Meta(type): @@ -1098,13 +1098,13 @@ class Y(X): pass self.assertEqual(get_annotations(Meta), {}) - self.assertIs(get_annotate_function(Meta), None) + self.assertIs(Meta.__annotate__, None) self.assertEqual(get_annotations(Y), {}) - self.assertIs(get_annotate_function(Y), None) + self.assertIs(Y.__annotate__, None) self.assertEqual(get_annotations(X), {"a": str}) - self.assertEqual(get_annotate_function(X)(Format.VALUE), {"a": str}) + self.assertEqual(X.__annotate__(Format.VALUE), {"a": str}) def test_ordering(self): # Based on a sample by David Ellis @@ -1142,7 +1142,7 @@ class D(metaclass=Meta): for c in classes: with self.subTest(c=c): self.assertEqual(get_annotations(c), c.expected_annotations) - annotate_func = get_annotate_function(c) + annotate_func = getattr(c, "__annotate__", None) if c.expected_annotations: self.assertEqual( annotate_func(Format.VALUE), c.expected_annotations @@ -1151,25 +1151,39 @@ class D(metaclass=Meta): self.assertIs(annotate_func, None) -class TestGetAnnotateFunction(unittest.TestCase): - def test_static_class(self): - self.assertIsNone(get_annotate_function(object)) - self.assertIsNone(get_annotate_function(int)) - - def test_unannotated_class(self): - class C: - pass +class TestGetAnnotateFromClassNamespace(unittest.TestCase): + def test_with_metaclass(self): + class Meta(type): + def __new__(mcls, name, bases, ns): + annotate = annotationlib.get_annotate_from_class_namespace(ns) + expected = ns["expected_annotate"] + with self.subTest(name=name): + if expected: + self.assertIsNotNone(annotate) + else: + self.assertIsNone(annotate) + return super().__new__(mcls, name, bases, ns) + + class HasAnnotations(metaclass=Meta): + expected_annotate = True + a: int - self.assertIsNone(get_annotate_function(C)) + class NoAnnotations(metaclass=Meta): + expected_annotate = False - D = type("D", (), {}) - self.assertIsNone(get_annotate_function(D)) + class CustomAnnotate(metaclass=Meta): + expected_annotate = True + def __annotate__(format): + return {} - def test_annotated_class(self): - class C: - a: int + code = """ + from __future__ import annotations - self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) + class HasFutureAnnotations(metaclass=Meta): + expected_annotate = False + a: int + """ + exec(textwrap.dedent(code), {"Meta": Meta}) class TestToSource(unittest.TestCase): diff --git a/Lib/typing.py b/Lib/typing.py index e5d14b03a4fc94..5ad8c18da91d1f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2894,7 +2894,7 @@ def __new__(cls, typename, bases, ns): types = ns["__annotations__"] field_names = list(types) annotate = _make_eager_annotate(types) - elif (original_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None: + elif (original_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: types = _lazy_annotationlib.call_annotate_function( original_annotate, _lazy_annotationlib.Format.FORWARDREF) field_names = list(types) @@ -3080,7 +3080,7 @@ def __new__(cls, name, bases, ns, total=True): if "__annotations__" in ns: own_annotate = None own_annotations = ns["__annotations__"] - elif (own_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None: + elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: own_annotations = _lazy_annotationlib.call_annotate_function( own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict ) diff --git a/Misc/NEWS.d/next/Library/2025-04-13-21-11-11.gh-issue-132426.SZno1d.rst b/Misc/NEWS.d/next/Library/2025-04-13-21-11-11.gh-issue-132426.SZno1d.rst new file mode 100644 index 00000000000000..1f2b9a2df936c6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-13-21-11-11.gh-issue-132426.SZno1d.rst @@ -0,0 +1,3 @@ +Add :func:`annotationlib.get_annotate_from_class_namespace` as a helper for +accessing annotations in metaclasses, and remove +``annotationlib.get_annotate_function``. From cddee44a8005343817a89d9ffe2ec543b4551ec2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 13 Apr 2025 21:15:07 -0700 Subject: [PATCH 2/3] update more docs --- Doc/library/annotationlib.rst | 11 +++++------ Lib/annotationlib.py | 8 +++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 102c4237290b36..5f0790aab9a747 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -303,12 +303,9 @@ Functions .. function:: get_annotate_from_class_namespace(namespace) Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*. - Return :const:`!None` if the namespace does not contain an annotate function. *obj* may be a class, function, - module, or a namespace dictionary for a class. The last case is useful during - class creation, e.g. in the ``__new__`` method of a metaclass. - - This is usually equivalent to accessing the :attr:`~object.__annotate__` - attribute of *obj*, but access through this public function is preferred. + Return :const:`!None` if the namespace does not contain an annotate function. + This is useful in :ref:`metaclasses ` to retrieve the annotate function; + see :ref:`below ` for an example. .. versionadded:: 3.14 @@ -411,6 +408,8 @@ Functions Recipes ------- +.. _annotationlib-metaclass: + Using annotations in a metaclass ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 77e6ed514bc661..d3ae34cc3c2219 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -620,12 +620,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def get_annotate_from_class_namespace(obj): - """Get the __annotate__ function for an object. + """Retrieve the annotate function from a class namespace dictionary. - obj may be a function, class, or module, or a user-defined type with - an `__annotate__` attribute. - - Returns the __annotate__ function or None. + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. """ try: return obj["__annotate__"] From e9fd0ebaaeed6721e65536b08b2b70a6f28f805e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 17 Apr 2025 09:48:50 -0700 Subject: [PATCH 3/3] reword docs, thanks Adam --- Doc/library/annotationlib.rst | 9 ++++++--- Doc/reference/datamodel.rst | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index b55aa930ead135..b9932a9e4cca1f 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -304,8 +304,9 @@ Functions Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*. Return :const:`!None` if the namespace does not contain an annotate function. - This is useful in :ref:`metaclasses ` to retrieve the annotate function; - see :ref:`below ` for an example. + This is primarily useful before the class has been fully created (e.g., in a metaclass); + after the class exists, the annotate function can be retrieved with ``cls.__annotate__``. + See :ref:`below ` for an example using this function in a metaclass. .. versionadded:: 3.14 @@ -429,7 +430,9 @@ that calls the original annotate function, makes any necessary adjustments, and returns the result. Below is an example of a metaclass that filters out all :class:`typing.ClassVar` -annotations from the class and puts them in a separate attribute:: +annotations from the class and puts them in a separate attribute: + +.. code-block:: python import annotationlib import typing diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 475ea09d760bbd..29aa927bf28fcb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1228,7 +1228,7 @@ Special attributes :attr:`__annotations__ attributes `. For best practices on working with :attr:`~object.__annotations__`, - please see :mod:`annotationlib`. It is recommended to use + please see :mod:`annotationlib`. Where possible, use :func:`annotationlib.get_annotations` instead of accessing this attribute directly.